Compare commits
85 Commits
62a4a20529
...
v0.14.0
| Author | SHA1 | Date | |
|---|---|---|---|
| 143c6e9b3a | |||
| 48e4784fcd | |||
| 78af5069c1 | |||
| 18963266d7 | |||
| d039a019cb | |||
| d1c0281053 | |||
| 870e4ca4e6 | |||
| fd82524977 | |||
| 3f76c75919 | |||
| fe764f72d8 | |||
| 26ab566f38 | |||
| 18ca21fa45 | |||
| e32fdc22f4 | |||
| b02b802427 | |||
| 301192b974 | |||
| f2aeb5eb30 | |||
| a27ee8f7da | |||
| 4d1895c223 | |||
| d7fe70520a | |||
| 7325a34c07 | |||
| 03c44923c8 | |||
| dc0adf5751 | |||
| eae21797f7 | |||
| ea6c7d5f0f | |||
| 389ef26db3 | |||
| 8a5c0de162 | |||
| 80aecb646d | |||
| b7b4000aab | |||
| 11ecb923ba | |||
| f46d5a4591 | |||
| 743e78d514 | |||
| 3743221aa5 | |||
| a8863d49d0 | |||
| c5f733af45 | |||
| c09d2c6e20 | |||
| 6c5dec761c | |||
| 74b91fae99 | |||
| b7ff37118c | |||
| ef012ea1e5 | |||
| 04d5d695e0 | |||
| 78bff91395 | |||
| c63ca5352f | |||
| 3f157150e4 | |||
| f1e1a45a05 | |||
| 996295d5c1 | |||
| 58a5bbe790 | |||
| dd89be3b03 | |||
| ed7c27324d | |||
| 05fe13d80b | |||
| 0eb009ad5a | |||
| 3fde1f6cbb | |||
| 481e2d494c | |||
| eb830bb613 | |||
| dd6d4591ae | |||
| de6443f3ae | |||
| 5a63b649f9 | |||
| c7e22c0dbe | |||
| e69a847cf2 | |||
| 21815ad7c0 | |||
| a051edd82d | |||
| ee0ad99e17 | |||
| bc5a2da0f2 | |||
| 089e824a92 | |||
| 93e8874813 | |||
| cc0b6183bf | |||
| bf472dad8a | |||
| a6e0ffea8f | |||
| e459c7d54b | |||
| 71f6683fda | |||
| 7827ca043b | |||
| b595dca8c7 | |||
| d5f34642d7 | |||
| b1866946d1 | |||
| 05082c6fdf | |||
| 51b072965c | |||
| d5940856be | |||
| 0d163f8165 | |||
| 1889915a81 | |||
| 3ca01ae0b0 | |||
| 6e283d16c3 | |||
| 41b2021d8c | |||
| 3add7024db | |||
| a148a2b1f4 | |||
| 7310ef34a4 | |||
| 4b8169da28 |
4
.gitignore
vendored
4
.gitignore
vendored
@@ -22,6 +22,9 @@
|
||||
go.work
|
||||
go.work.sum
|
||||
|
||||
# Local ReleaseKit executable
|
||||
.releasekit/
|
||||
|
||||
# env file
|
||||
.env
|
||||
|
||||
@@ -67,4 +70,3 @@ Icon
|
||||
.AppleDesktop
|
||||
Network Trash Folder
|
||||
Temporary Items
|
||||
.apdisk
|
||||
|
||||
@@ -37,6 +37,10 @@ boundary and constraints that framework work must preserve.
|
||||
|
||||
## Release Guidance
|
||||
|
||||
Consumers upgrading to `v0.12.0` should read the
|
||||
[v0.12.0 release notes](docs/releases/v0.12.0.md) for the intentional ConfigKit
|
||||
configuration and public construction API migration.
|
||||
|
||||
The [v0.11.2 release notes](docs/releases/v0.11.2.md) cover policy alignment
|
||||
and copied-policy attribution updates.
|
||||
|
||||
|
||||
237
assembled_workflow_benchmark_test.go
Normal file
237
assembled_workflow_benchmark_test.go
Normal file
@@ -0,0 +1,237 @@
|
||||
package promptkit_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"testing/fstest"
|
||||
|
||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
var (
|
||||
assembledPreparedSink *promptkit.PreparedRun
|
||||
assembledResultSink *promptkit.RunResult
|
||||
assembledDetailsSink promptkit.PreparedRun
|
||||
assembledHandleSink *promptkit.PreparedExecution
|
||||
)
|
||||
|
||||
type assembledWorkflowClient struct{ content string }
|
||||
|
||||
func (client assembledWorkflowClient) Generate(
|
||||
context.Context,
|
||||
promptkit.GenerateRequest,
|
||||
) (*promptkit.GenerateResponse, error) {
|
||||
return &promptkit.GenerateResponse{Content: client.content}, nil
|
||||
}
|
||||
|
||||
type assembledWorkflowFixture struct {
|
||||
engine *promptkit.Engine
|
||||
request promptkit.RunRequest
|
||||
expectedMessage string
|
||||
response string
|
||||
}
|
||||
|
||||
func assembledWorkflowSchema(b *testing.B) []byte {
|
||||
b.Helper()
|
||||
properties := make(map[string]any, 256)
|
||||
for index := range 256 {
|
||||
properties[fmt.Sprintf("property-%03d", index)] = map[string]any{"type": "string"}
|
||||
}
|
||||
data, err := json.Marshal(map[string]any{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"type": "object",
|
||||
"properties": properties,
|
||||
"additionalProperties": false,
|
||||
})
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
return data
|
||||
}
|
||||
|
||||
func assembledWorkflowExtras(entries int) map[string]any {
|
||||
providers := make(map[string]any, entries)
|
||||
for index := range entries {
|
||||
providers[fmt.Sprintf("provider-%03d", index)] = map[string]any{
|
||||
"enabled": true,
|
||||
"labels": []any{fmt.Sprintf("tier-%d", index%4), fmt.Sprintf("region-%d", index%3)},
|
||||
"limits": map[string]any{"burst": index + 1, "period": "minute"},
|
||||
"routes": []any{map[string]any{"weight": index + 1, "fallback": true}},
|
||||
}
|
||||
}
|
||||
return map[string]any{"provider_options": providers}
|
||||
}
|
||||
|
||||
func newAssembledWorkflowFixture(b *testing.B, name string) assembledWorkflowFixture {
|
||||
b.Helper()
|
||||
message := "Hello {{.name}}"
|
||||
response := "ok"
|
||||
output := map[string]any{"format": "text", "validation_mode": "none"}
|
||||
profile := map[string]any{"endpoint": "http://example.test/v1", "model": "benchmark-model"}
|
||||
options := []promptkit.Option{promptkit.WithLLMClient(assembledWorkflowClient{content: response})}
|
||||
var expectedMessage string
|
||||
|
||||
switch name {
|
||||
case "small":
|
||||
expectedMessage = "Hello Ada"
|
||||
case "large_message":
|
||||
prefix := strings.Repeat("x", 256*1024)
|
||||
message = prefix + "{{.name}}"
|
||||
expectedMessage = prefix + "Ada"
|
||||
case "mutable_payload":
|
||||
message = `{"subject":"{{.name}}"}`
|
||||
expectedMessage = `{"subject":"Ada"}`
|
||||
response = "{}"
|
||||
profile["extra_params"] = assembledWorkflowExtras(64)
|
||||
output = map[string]any{
|
||||
"format": "json", "validation_mode": "json_schema", "schema_path": "workflow-schema.json",
|
||||
}
|
||||
options[0] = promptkit.WithLLMClient(assembledWorkflowClient{content: response})
|
||||
options = append(options, promptkit.WithSchemaFS(fstest.MapFS{
|
||||
"workflow-schema.json": &fstest.MapFile{Data: assembledWorkflowSchema(b)},
|
||||
}, "."))
|
||||
default:
|
||||
b.Fatalf("unknown assembled workflow fixture %q", name)
|
||||
}
|
||||
|
||||
config := map[string]any{
|
||||
"profiles": map[string]any{"profile": profile},
|
||||
"prompts": map[string]any{"workflow": map[string]any{"v1": map[string]any{
|
||||
"default_profile": "profile",
|
||||
"messages": []any{map[string]any{"role": "user", "content": message}},
|
||||
"output": output,
|
||||
}}},
|
||||
}
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
|
||||
Source: configkit.Mapping("assembled workflow benchmark", config),
|
||||
}}}, options...)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
request := promptkit.RunRequest{
|
||||
PromptID: "workflow", PromptVersion: "v1", Vars: map[string]string{"name": "Ada"},
|
||||
}
|
||||
return assembledWorkflowFixture{
|
||||
engine: engine, request: request, expectedMessage: expectedMessage, response: response,
|
||||
}
|
||||
}
|
||||
|
||||
func verifyAssembledWorkflowFixture(b *testing.B, fixture assembledWorkflowFixture) {
|
||||
b.Helper()
|
||||
ctx := context.Background()
|
||||
prepared, err := fixture.engine.Prepare(ctx, fixture.request)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
if prepared.PromptID != "workflow" || prepared.PromptVersion != "v1" || len(prepared.Messages) != 1 ||
|
||||
prepared.Messages[0].Content != fixture.expectedMessage {
|
||||
b.Fatalf("prepared fixture output is invalid: %+v", prepared)
|
||||
}
|
||||
handle, err := fixture.engine.PrepareExecution(ctx, fixture.request)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
details := handle.Details()
|
||||
if details.PromptID != "workflow" || len(details.Messages) != 1 || details.Messages[0].Content != fixture.expectedMessage {
|
||||
handle.Discard()
|
||||
b.Fatalf("prepared execution details are invalid: %+v", details)
|
||||
}
|
||||
handle.Discard()
|
||||
result, err := fixture.engine.Run(ctx, fixture.request)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
if result.RawOutput != fixture.response {
|
||||
b.Fatalf("run output=%q, want %q", result.RawOutput, fixture.response)
|
||||
}
|
||||
if fixture.response == "{}" && result.Validation.Status != promptkit.ValidationPassed {
|
||||
b.Fatalf("structured response validation=%+v, want passed", result.Validation)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkEnginePreparation(b *testing.B) {
|
||||
for _, name := range []string{"small", "large_message", "mutable_payload"} {
|
||||
b.Run(name, func(b *testing.B) {
|
||||
fixture := newAssembledWorkflowFixture(b, name)
|
||||
verifyAssembledWorkflowFixture(b, fixture)
|
||||
ctx := context.Background()
|
||||
b.Run("prepare", func(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for b.Loop() {
|
||||
prepared, err := fixture.engine.Prepare(ctx, fixture.request)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
assembledPreparedSink = prepared
|
||||
}
|
||||
b.StopTimer()
|
||||
assembledPreparedSink = nil
|
||||
})
|
||||
b.Run("prepared_execution_creation_and_discard", func(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for b.Loop() {
|
||||
handle, err := fixture.engine.PrepareExecution(ctx, fixture.request)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
handle.Discard()
|
||||
assembledHandleSink = handle
|
||||
}
|
||||
b.StopTimer()
|
||||
assembledHandleSink = nil
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkEngineRun(b *testing.B) {
|
||||
for _, name := range []string{"small", "large_message", "mutable_payload"} {
|
||||
b.Run(name, func(b *testing.B) {
|
||||
fixture := newAssembledWorkflowFixture(b, name)
|
||||
verifyAssembledWorkflowFixture(b, fixture)
|
||||
ctx := context.Background()
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for b.Loop() {
|
||||
result, err := fixture.engine.Run(ctx, fixture.request)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
assembledResultSink = result
|
||||
}
|
||||
b.StopTimer()
|
||||
assembledResultSink = nil
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkPreparedExecutionDetails(b *testing.B) {
|
||||
for _, name := range []string{"small", "large_message", "mutable_payload"} {
|
||||
b.Run(name, func(b *testing.B) {
|
||||
fixture := newAssembledWorkflowFixture(b, name)
|
||||
verifyAssembledWorkflowFixture(b, fixture)
|
||||
handle, err := fixture.engine.PrepareExecution(context.Background(), fixture.request)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
defer handle.Discard()
|
||||
initial := handle.Details()
|
||||
if len(initial.Messages) != 1 || initial.Messages[0].Content != fixture.expectedMessage {
|
||||
b.Fatalf("prepared details fixture is invalid: %+v", initial)
|
||||
}
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for b.Loop() {
|
||||
assembledDetailsSink = handle.Details()
|
||||
}
|
||||
b.StopTimer()
|
||||
assembledDetailsSink = promptkit.PreparedRun{}
|
||||
})
|
||||
}
|
||||
}
|
||||
243
configuration_diff_contract_test.go
Normal file
243
configuration_diff_contract_test.go
Normal file
@@ -0,0 +1,243 @@
|
||||
package promptkit_test
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
func TestConfigurationInspectionDiffsCapturedValues(t *testing.T) {
|
||||
left := comparisonInspection(t, "earlier-model")
|
||||
right := comparisonInspection(t, "later-model")
|
||||
|
||||
differences, err := left.Diff(right)
|
||||
if err != nil {
|
||||
t.Fatalf("compare captured configurations: %v", err)
|
||||
}
|
||||
if len(differences) != 1 {
|
||||
t.Fatalf("difference count = %d, want 1: %+v", len(differences), differences)
|
||||
}
|
||||
difference := differences[0]
|
||||
if got, want := difference.Path.String(), "/profiles/offline/model"; got != want {
|
||||
t.Fatalf("difference path = %q, want %q", got, want)
|
||||
}
|
||||
if !strings.Contains(difference.Left.YAML, "earlier-model") || !strings.Contains(difference.Right.YAML, "later-model") {
|
||||
t.Fatalf("difference snapshots = (%q, %q)", difference.Left.YAML, difference.Right.YAML)
|
||||
}
|
||||
|
||||
reversed, err := right.Diff(left)
|
||||
if err != nil {
|
||||
t.Fatalf("compare reversed configurations: %v", err)
|
||||
}
|
||||
if len(reversed) != 1 || reversed[0].Path.String() != difference.Path.String() ||
|
||||
!strings.Contains(reversed[0].Left.YAML, "later-model") ||
|
||||
!strings.Contains(reversed[0].Right.YAML, "earlier-model") {
|
||||
t.Fatalf("reversed difference = %+v", reversed)
|
||||
}
|
||||
|
||||
equal := comparisonInspection(t, "earlier-model")
|
||||
if equal, err := left.Diff(equal); err != nil || len(equal) != 0 {
|
||||
t.Fatalf("compare equal configurations = (%+v, %v), want no differences", equal, err)
|
||||
}
|
||||
if equal, err := left.Diff(left); err != nil || len(equal) != 0 {
|
||||
t.Fatalf("compare view with itself = (%+v, %v), want no differences", equal, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigurationInspectionDiffRejectsZeroViews(t *testing.T) {
|
||||
valid := comparisonInspection(t, "current-model")
|
||||
var zero promptkit.ConfigurationInspection
|
||||
|
||||
for _, comparison := range []struct {
|
||||
name string
|
||||
left, right promptkit.ConfigurationInspection
|
||||
}{
|
||||
{name: "zero left", left: zero, right: valid},
|
||||
{name: "zero right", left: valid, right: zero},
|
||||
{name: "both zero", left: zero, right: zero},
|
||||
} {
|
||||
t.Run(comparison.name, func(t *testing.T) {
|
||||
differences, err := comparison.left.Diff(comparison.right)
|
||||
if !errors.Is(err, promptkit.ErrInvalidConfig) || len(differences) != 0 {
|
||||
t.Fatalf("Diff() = (%+v, %v), want ErrInvalidConfig and no records", differences, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigurationInspectionDiffIgnoresSourceHistory(t *testing.T) {
|
||||
left := comparisonInspectionFrom(t, "earlier source", "same-model", nil)
|
||||
right := comparisonInspectionFrom(t, "later source", "same-model", nil)
|
||||
if !hasSourceLabel(left, "earlier source") || !hasSourceLabel(right, "later source") {
|
||||
t.Fatal("comparison fixtures did not retain their distinct source labels")
|
||||
}
|
||||
|
||||
differences, err := left.Diff(right)
|
||||
if err != nil || len(differences) != 0 {
|
||||
t.Fatalf("compare equal effective values with different sources = (%+v, %v), want no differences", differences, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigurationInspectionDiffReportsPresenceAndSequences(t *testing.T) {
|
||||
left := comparisonInspectionFrom(t, "before", "same-model", map[string]any{
|
||||
"sequence": []any{"earlier", "unchanged"},
|
||||
})
|
||||
right := comparisonInspectionFrom(t, "after", "same-model", map[string]any{
|
||||
"nullable": nil,
|
||||
"sequence": []any{"later", "unchanged"},
|
||||
})
|
||||
|
||||
differences, err := left.Diff(right)
|
||||
if err != nil {
|
||||
t.Fatalf("compare presence and sequence values: %v", err)
|
||||
}
|
||||
if len(differences) != 2 {
|
||||
t.Fatalf("difference count = %d, want 2: %+v", len(differences), differences)
|
||||
}
|
||||
byPath := differencesByPath(differences)
|
||||
nullable, ok := byPath["/backends/local/extra_params/nullable"]
|
||||
if !ok || nullable.Left.Present || !nullable.Right.Present || strings.TrimSpace(nullable.Right.YAML) != "null" {
|
||||
t.Fatalf("absent-to-null difference = (%+v, %t)", nullable, ok)
|
||||
}
|
||||
sequence, ok := byPath["/backends/local/extra_params/sequence"]
|
||||
if !ok || !sequence.Left.Present || !sequence.Right.Present ||
|
||||
!strings.Contains(sequence.Left.YAML, "earlier") || !strings.Contains(sequence.Right.YAML, "later") {
|
||||
t.Fatalf("atomic sequence difference = (%+v, %t)", sequence, ok)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigurationInspectionDiffProtectsSensitiveValuesAndReturnedRecords(t *testing.T) {
|
||||
left := comparisonInspectionFrom(t, "before", "same-model", map[string]any{
|
||||
"token": "left-secret-sentinel",
|
||||
"visible": "before-value",
|
||||
})
|
||||
right := comparisonInspectionFrom(t, "after", "same-model", map[string]any{
|
||||
"token": "right-secret-sentinel",
|
||||
"visible": "after-value",
|
||||
})
|
||||
|
||||
differences, err := left.Diff(right)
|
||||
if err != nil {
|
||||
t.Fatalf("compare sensitive values: %v", err)
|
||||
}
|
||||
if len(differences) != 2 {
|
||||
t.Fatalf("difference count = %d, want 2: %+v", len(differences), differences)
|
||||
}
|
||||
byPath := differencesByPath(differences)
|
||||
secret, ok := byPath["/backends/local/extra_params/token"]
|
||||
if !ok || !secret.Left.Redacted || !secret.Right.Redacted || !secret.HiddenChanged {
|
||||
t.Fatalf("sensitive difference = (%+v, %t)", secret, ok)
|
||||
}
|
||||
if strings.Contains(secret.Left.YAML, "left-secret-sentinel") || strings.Contains(secret.Left.YAML, "right-secret-sentinel") ||
|
||||
strings.Contains(secret.Right.YAML, "left-secret-sentinel") || strings.Contains(secret.Right.YAML, "right-secret-sentinel") {
|
||||
t.Fatalf("sensitive snapshots disclosed a value: %+v", secret)
|
||||
}
|
||||
visible, ok := byPath["/backends/local/extra_params/visible"]
|
||||
if !ok || visible.Left.Redacted || visible.Right.Redacted || visible.HiddenChanged ||
|
||||
!strings.Contains(visible.Left.YAML, "before-value") || !strings.Contains(visible.Right.YAML, "after-value") {
|
||||
t.Fatalf("public sibling difference = (%+v, %t)", visible, ok)
|
||||
}
|
||||
|
||||
differences[0].Path = configkit.Keys("caller", "mutation")
|
||||
differences[0].Left.YAML = "caller mutation"
|
||||
differences[0].Right.YAML = "caller mutation"
|
||||
again, err := left.Diff(right)
|
||||
if err != nil {
|
||||
t.Fatalf("repeat comparison after mutating returned records: %v", err)
|
||||
}
|
||||
if len(again) != 2 {
|
||||
t.Fatalf("repeat difference count = %d, want 2: %+v", len(again), again)
|
||||
}
|
||||
againByPath := differencesByPath(again)
|
||||
if _, exists := againByPath["/caller/mutation"]; exists {
|
||||
t.Fatal("mutating returned path reached retained inspection")
|
||||
}
|
||||
if _, exists := againByPath["/backends/local/extra_params/token"]; !exists {
|
||||
t.Fatalf("original sensitive path was lost after mutation: %+v", again)
|
||||
}
|
||||
if _, exists := againByPath["/backends/local/extra_params/visible"]; !exists {
|
||||
t.Fatalf("original public path was lost after mutation: %+v", again)
|
||||
}
|
||||
for _, difference := range again {
|
||||
if difference.Left.YAML == "caller mutation" || difference.Right.YAML == "caller mutation" {
|
||||
t.Fatalf("mutating returned snapshots reached retained inspection: %+v", difference)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigurationInspectionDiffMasksSensitiveShapeReplacement(t *testing.T) {
|
||||
left := comparisonInspectionFrom(t, "before", "same-model", map[string]any{
|
||||
"details": map[string]any{
|
||||
"token": "nested-secret-sentinel",
|
||||
"label": "public detail",
|
||||
},
|
||||
})
|
||||
right := comparisonInspectionFrom(t, "after", "same-model", map[string]any{
|
||||
"details": "replacement",
|
||||
})
|
||||
|
||||
differences, err := left.Diff(right)
|
||||
if err != nil {
|
||||
t.Fatalf("compare sensitive shape replacement: %v", err)
|
||||
}
|
||||
if len(differences) != 1 || differences[0].Path.String() != "/backends/local/extra_params/details" {
|
||||
t.Fatalf("shape replacement differences = %+v", differences)
|
||||
}
|
||||
difference := differences[0]
|
||||
if !difference.Left.Redacted || !difference.Right.Redacted || !difference.HiddenChanged {
|
||||
t.Fatalf("shape replacement masking = %+v", difference)
|
||||
}
|
||||
if strings.Contains(difference.Left.YAML, "nested-secret-sentinel") || strings.Contains(difference.Right.YAML, "nested-secret-sentinel") {
|
||||
t.Fatalf("shape replacement disclosed a sensitive value: %+v", difference)
|
||||
}
|
||||
}
|
||||
|
||||
func comparisonInspection(t *testing.T, model string) promptkit.ConfigurationInspection {
|
||||
t.Helper()
|
||||
return comparisonInspectionFrom(t, "comparison fixture", model, nil)
|
||||
}
|
||||
|
||||
func comparisonInspectionFrom(t *testing.T, source, model string, extraParams map[string]any) promptkit.ConfigurationInspection {
|
||||
t.Helper()
|
||||
backend := map[string]any{"endpoint": "https://example.invalid/v1"}
|
||||
if extraParams != nil {
|
||||
backend["extra_params"] = extraParams
|
||||
}
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: configkit.Mapping(source, map[string]any{
|
||||
"backends": map[string]any{"local": backend},
|
||||
"profiles": map[string]any{"offline": map[string]any{
|
||||
"backend": "local",
|
||||
"model": model,
|
||||
}},
|
||||
})}}}},
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatalf("construct comparison engine: %v", err)
|
||||
}
|
||||
inspection, err := engine.InspectConfiguration()
|
||||
if err != nil {
|
||||
t.Fatalf("inspect comparison engine: %v", err)
|
||||
}
|
||||
return inspection
|
||||
}
|
||||
|
||||
func differencesByPath(differences []configkit.Difference) map[string]configkit.Difference {
|
||||
byPath := make(map[string]configkit.Difference, len(differences))
|
||||
for _, difference := range differences {
|
||||
byPath[difference.Path.String()] = difference
|
||||
}
|
||||
return byPath
|
||||
}
|
||||
|
||||
func hasSourceLabel(view promptkit.ConfigurationInspection, label string) bool {
|
||||
for _, source := range view.Sources() {
|
||||
if source.Label == label {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -27,6 +27,23 @@ func (e *Engine) InspectConfiguration() (ConfigurationInspection, error) {
|
||||
return ConfigurationInspection{source: e.internal.InspectConfiguration()}, nil
|
||||
}
|
||||
|
||||
// Diff compares this view's captured effective configuration with other. This
|
||||
// view is the earlier (left) value and other is the later (right) value. The
|
||||
// comparison delegates to ConfigKit's frozen effective-value semantics, so
|
||||
// source or resolution history alone does not create differences. It does not
|
||||
// reread external sources or compare referenced file contents, resolved
|
||||
// credential contents, later execution preparation, or request-specific
|
||||
// overrides. Either zero view returns ErrInvalidConfig and no records; other
|
||||
// ConfigKit errors are returned unchanged. An empty diff is not evidence of
|
||||
// equivalent engine behavior or execution. Records can contain ordinary
|
||||
// application data, so callers choose where and to whom to disclose them.
|
||||
func (v ConfigurationInspection) Diff(other ConfigurationInspection) ([]configkit.Difference, error) {
|
||||
if v.source == nil || other.source == nil {
|
||||
return nil, ErrInvalidConfig
|
||||
}
|
||||
return configuration.Diff(v.source, other.source)
|
||||
}
|
||||
|
||||
// Metadata returns detached effective presence and history records.
|
||||
func (v ConfigurationInspection) Metadata() configkit.Metadata {
|
||||
if v.source == nil {
|
||||
|
||||
@@ -144,9 +144,17 @@ func TestConfigurationInspectionUsesCapturedSourcesWithoutProviderCalls(t *testi
|
||||
if err != nil {
|
||||
t.Fatalf("select inputs: %v", err)
|
||||
}
|
||||
otherSource := &unreadableAfterConstructionFS{FS: fstest.MapFS{
|
||||
"config/prompt.yaml": &fstest.MapFile{Data: []byte(capturedPromptDocument)},
|
||||
"config/body.tmpl": &fstest.MapFile{Data: []byte("hello")},
|
||||
}}
|
||||
otherInputs, err := promptkit.FilesystemInputs("comparison source", otherSource, "config")
|
||||
if err != nil {
|
||||
t.Fatalf("select comparison inputs: %v", err)
|
||||
}
|
||||
var modelCalls atomic.Int64
|
||||
client := countingLLMClient{calls: &modelCalls}
|
||||
endpoint, model := "https://example.invalid/v1", "offline-model"
|
||||
endpoint, model, otherModel := "https://example.invalid/v1", "offline-model", "comparison-model"
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||
Underlays: []configkit.Underlay{{Inputs: inputs}, {Inputs: []configkit.Input{{Source: promptkit.ProfileSource("offline profile", map[string]promptkit.ProfileConfig{
|
||||
"offline": {Endpoint: &endpoint, Model: &model},
|
||||
@@ -155,11 +163,42 @@ func TestConfigurationInspectionUsesCapturedSourcesWithoutProviderCalls(t *testi
|
||||
if err != nil {
|
||||
t.Fatalf("construct engine: %v", err)
|
||||
}
|
||||
otherEngine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||
Underlays: []configkit.Underlay{{Inputs: otherInputs}, {Inputs: []configkit.Input{{Source: promptkit.ProfileSource("comparison profile", map[string]promptkit.ProfileConfig{
|
||||
"offline": {Endpoint: &endpoint, Model: &otherModel},
|
||||
})}}}},
|
||||
}}, promptkit.WithLLMClient(client))
|
||||
if err != nil {
|
||||
t.Fatalf("construct comparison engine: %v", err)
|
||||
}
|
||||
source.closed.Store(true)
|
||||
otherSource.closed.Store(true)
|
||||
view, err := engine.InspectConfiguration()
|
||||
if err != nil {
|
||||
t.Fatalf("inspect configuration: %v", err)
|
||||
}
|
||||
otherView, err := otherEngine.InspectConfiguration()
|
||||
if err != nil {
|
||||
t.Fatalf("inspect comparison configuration: %v", err)
|
||||
}
|
||||
var expected configkit.Difference
|
||||
for index := 0; index < 3; index++ {
|
||||
differences, err := view.Diff(otherView)
|
||||
if err != nil || len(differences) != 1 {
|
||||
t.Fatalf("compare captured configurations = (%+v, %v), want one difference", differences, err)
|
||||
}
|
||||
if differences[0].Path.String() != "/profiles/offline/model" ||
|
||||
!strings.Contains(differences[0].Left.YAML, model) ||
|
||||
!strings.Contains(differences[0].Right.YAML, otherModel) {
|
||||
t.Fatalf("captured difference = %+v", differences[0])
|
||||
}
|
||||
if index == 0 {
|
||||
expected = differences[0]
|
||||
} else if differences[0].Path.String() != expected.Path.String() ||
|
||||
differences[0].Left.YAML != expected.Left.YAML || differences[0].Right.YAML != expected.Right.YAML {
|
||||
t.Fatalf("captured difference changed between reads: first=%+v next=%+v", expected, differences[0])
|
||||
}
|
||||
}
|
||||
if _, found := view.Explain(configkit.Keys("prompts", "captured", "1", "messages").Index(0).Key("content_file")); !found {
|
||||
t.Fatal("captured prompt content path was not inspectable")
|
||||
}
|
||||
@@ -167,24 +206,32 @@ func TestConfigurationInspectionUsesCapturedSourcesWithoutProviderCalls(t *testi
|
||||
if err := view.WriteEffective(&report); err != nil || !strings.Contains(report.String(), "captured") {
|
||||
t.Fatalf("write captured report = (%q, %v)", report.String(), err)
|
||||
}
|
||||
if source.reads.Load() != 0 || modelCalls.Load() != 0 {
|
||||
t.Fatalf("inspection touched source or provider: reads=%d requests=%d", source.reads.Load(), modelCalls.Load())
|
||||
if source.reads.Load() != 0 || otherSource.reads.Load() != 0 || modelCalls.Load() != 0 {
|
||||
t.Fatalf("inspection or comparison touched source or provider: reads=(%d,%d) requests=%d", source.reads.Load(), otherSource.reads.Load(), modelCalls.Load())
|
||||
}
|
||||
|
||||
const calls = 16
|
||||
const calls = 24
|
||||
var wait sync.WaitGroup
|
||||
errs := make(chan error, calls)
|
||||
for index := 0; index < calls; index++ {
|
||||
wait.Add(1)
|
||||
go func(index int) {
|
||||
defer wait.Done()
|
||||
if index%2 == 0 {
|
||||
switch index % 3 {
|
||||
case 0:
|
||||
_, err := engine.InspectConfiguration()
|
||||
errs <- err
|
||||
return
|
||||
case 1:
|
||||
_, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "captured", SessionID: "direct"})
|
||||
errs <- err
|
||||
case 2:
|
||||
differences, err := view.Diff(otherView)
|
||||
if err == nil && (len(differences) != 1 || differences[0].Path.String() != expected.Path.String() ||
|
||||
differences[0].Left.YAML != expected.Left.YAML || differences[0].Right.YAML != expected.Right.YAML) {
|
||||
err = errors.New("concurrent comparison returned an unexpected difference")
|
||||
}
|
||||
errs <- err
|
||||
}
|
||||
_, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "captured", SessionID: "direct"})
|
||||
errs <- err
|
||||
}(index)
|
||||
}
|
||||
wait.Wait()
|
||||
@@ -194,7 +241,7 @@ func TestConfigurationInspectionUsesCapturedSourcesWithoutProviderCalls(t *testi
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if source.reads.Load() != 0 || modelCalls.Load() != calls/2 {
|
||||
t.Fatalf("concurrent inspection/execution = source reads %d, model calls %d", source.reads.Load(), modelCalls.Load())
|
||||
if source.reads.Load() != 0 || otherSource.reads.Load() != 0 || modelCalls.Load() != calls/3 {
|
||||
t.Fatalf("concurrent inspection/comparison/execution = source reads (%d,%d), model calls %d", source.reads.Load(), otherSource.reads.Load(), modelCalls.Load())
|
||||
}
|
||||
}
|
||||
|
||||
25
doc.go
25
doc.go
@@ -4,15 +4,16 @@
|
||||
// Applications construct an [Engine] with [NewEngine], select ConfigKit
|
||||
// sources for prompts, profiles, and backends, and
|
||||
// call [Engine.InspectConfiguration], [Engine.InspectPrompt],
|
||||
// [Engine.InspectProfile], [Engine.Prepare],
|
||||
// [Engine.PrepareExecution], [Engine.Run], or [Engine.RunPrepared]. Concrete
|
||||
// registries, repositories, validators, and the built-in OpenAI-compatible
|
||||
// client remain internal implementation details.
|
||||
// [Engine.InspectRequirements], [Engine.InspectProfile], [Engine.ListProfiles],
|
||||
// [Engine.ListPrompts], [Engine.Prepare], [Engine.PrepareExecution],
|
||||
// [Engine.Run], or [Engine.RunPrepared]. Concrete registries, repositories,
|
||||
// validators, and the built-in OpenAI-compatible client remain internal
|
||||
// implementation details.
|
||||
//
|
||||
// # Concurrency and ownership
|
||||
//
|
||||
// An Engine supports concurrent InspectConfiguration, InspectPrompt,
|
||||
// InspectProfile, Prepare,
|
||||
// InspectRequirements, InspectProfile, ListProfiles, ListPrompts, Prepare,
|
||||
// PrepareExecution, Run, and RunPrepared calls. Engine-local backend policies
|
||||
// bound admitted Run and RunPrepared calls and model generations where
|
||||
// configured, while different backend pools and unlimited backends continue
|
||||
@@ -20,11 +21,12 @@
|
||||
// still receive concurrent calls and must be safe for that use.
|
||||
//
|
||||
// ConfigKit captures the selected configuration when NewEngine constructs an
|
||||
// Engine. Prepare,
|
||||
// PrepareExecution, and Run copy request maps, slices, pointer values, and
|
||||
// JSON-compatible extra parameters before using them. InspectPrompt and
|
||||
// InspectProfile return copied inspection values. Returned values and values
|
||||
// passed to extension interfaces are likewise isolated from engine state.
|
||||
// Engine. Prepare, PrepareExecution, and Run copy request maps, slices, pointer
|
||||
// values, and JSON-compatible extra parameters before using them. InspectPrompt,
|
||||
// InspectRequirements, and InspectProfile return copied inspection values.
|
||||
// ListProfiles and ListPrompts return caller-owned identity slices.
|
||||
// Returned values and values passed to extension interfaces are likewise
|
||||
// isolated from engine state.
|
||||
// Callers own those copies and may mutate them after the call that supplied or
|
||||
// returned them. [CapacityError] values are caller-owned and may be mutated
|
||||
// without affecting engine state or another error. Immutable [GenerationError]
|
||||
@@ -58,7 +60,8 @@
|
||||
// [ConfigurationSources], [ConfigurationInspection], [BackendConfig],
|
||||
// [ProfileConfig], [RunRequest],
|
||||
// [ArtifactRef], [ExecutionTargetOverride], [ProfileInspection],
|
||||
// [PromptInputDefinition], [PromptInspection], [PreparedExecution],
|
||||
// [PromptIdentity], [PromptInputDefinition], [PromptInspection],
|
||||
// [RequirementInspectionRequest], [RequirementInspection], [PreparedExecution],
|
||||
// [CapacityError], [GenerationError], [RetryExhaustedError], and
|
||||
// [ExecutionEvent], do not have stable JSON
|
||||
// representations. Direct API keys are nevertheless excluded from JSON for
|
||||
|
||||
@@ -19,6 +19,20 @@ The following Go fragments are illustrative and omit surrounding package,
|
||||
import, and error-handling code. Use the maintained examples for complete
|
||||
programs.
|
||||
|
||||
## Go Compatibility
|
||||
|
||||
The `go` directive in the selected release's [module metadata](../../go.mod)
|
||||
declares its minimum supported consumer Go version. Maintainers build and
|
||||
validate releases with the locally selected Go patch release; the minimum
|
||||
does not simply track that patch version.
|
||||
|
||||
Successful release validation on the selected patch is sufficient to presume
|
||||
support across its Go series, subject to the declared minimum and concrete
|
||||
evidence of incompatibility. Specific compatibility concerns are investigated
|
||||
as they arise. Minor releases may raise the minimum; patch releases preserve
|
||||
it. Release notes explain increases and any required consumer action. See the
|
||||
[release policy](../policy/release.md#consumer-minimum-and-release-toolchain).
|
||||
|
||||
## Construct An Engine
|
||||
|
||||
Create an engine with [`NewEngine`](../../engine.go). Select each directory
|
||||
@@ -104,6 +118,43 @@ engines and prepared handles retain their captured definitions; schema and
|
||||
request artifact resources still follow their runtime lifecycle. Inspection
|
||||
does not watch or reload sources automatically.
|
||||
|
||||
## Discover Available Profiles And Prompts
|
||||
|
||||
Use [`Engine.ListProfiles`](../../listing.go) and
|
||||
[`Engine.ListPrompts`](../../listing.go) to discover exact identities from the
|
||||
engine's final captured configuration, then inspect a selected prompt:
|
||||
|
||||
```go
|
||||
profiles, err := engine.ListProfiles(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(profiles) > 0 {
|
||||
_, err := engine.InspectProfile(ctx, profiles[0])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
prompts, err := engine.ListPrompts(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(prompts) > 0 {
|
||||
selected := prompts[0]
|
||||
inspection, err := engine.InspectPrompt(ctx, selected.ID, selected.Version)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_ = inspection
|
||||
}
|
||||
```
|
||||
|
||||
These lists describe effective membership, not ConfigKit contribution history,
|
||||
and do not promise that a profile is ready to execute. Use inspection for
|
||||
details and preparation to resolve request resources. The linked method GoDoc
|
||||
owns ordering, ownership, concurrency, and error behavior; identifiers may be
|
||||
sensitive application data.
|
||||
|
||||
## Inspect The Captured Configuration
|
||||
|
||||
Use [`Engine.InspectConfiguration`](../../configuration_inspection.go) for an
|
||||
@@ -130,6 +181,41 @@ shows backend-completed execution settings for one profile, and `Prepare` or
|
||||
`PreparedExecution.Details` shows request-specific rendered messages, inputs,
|
||||
and validation resources. None of those views is a substitute for the others.
|
||||
|
||||
To review changes between two independently captured configurations, compare
|
||||
their inspection views:
|
||||
|
||||
```go
|
||||
earlier, err := earlierEngine.InspectConfiguration()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
later, err := candidateEngine.InspectConfiguration()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
differences, err := earlier.Diff(later)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, difference := range differences {
|
||||
if difference.HiddenChanged {
|
||||
// Report a protected change without inferring its value.
|
||||
}
|
||||
if difference.Left.Redacted || difference.Right.Redacted {
|
||||
// Apply the application's disclosure policy before presenting snapshots.
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
See the [`ConfigurationInspection.Diff` GoDoc](../../configuration_inspection.go)
|
||||
and ConfigKit's [v0.7.0 effective-configuration diff guidance](https://gitea.maximumdirect.net/eric/ConfigKit/src/tag/v0.7.0/docs/config.md#effective-configuration-diffs)
|
||||
for the exact comparison contract. The operation compares only frozen effective
|
||||
configuration values: it does not reread external files or compare their
|
||||
contents, resolved credential contents, later execution preparation, or
|
||||
request-specific overrides. An empty diff is not proof of equivalent execution.
|
||||
Records can contain ordinary application data, so use the application's
|
||||
disclosure policy when presenting them.
|
||||
|
||||
## Inspect A Prompt Before Preparation
|
||||
|
||||
Use [`Engine.InspectPrompt`](../../engine.go) to check one configured prompt's
|
||||
@@ -154,6 +240,50 @@ rendered messages, and use prepared execution when that work must remain tied
|
||||
to later execution. The method's [GoDoc](../../engine.go) owns exact fields,
|
||||
hash, ownership, and error semantics.
|
||||
|
||||
## Inspect Requirements Before Preparation
|
||||
|
||||
Use [`Engine.InspectRequirements`](../../requirements.go) when the application
|
||||
needs the prompt's recognizable input and caller-variable dependencies for a
|
||||
particular context before it has input artifacts. Unlike `InspectPrompt`, it
|
||||
also walks captured template syntax and selects a profile identity; unlike
|
||||
`Prepare`, it does not load artifacts or render messages.
|
||||
|
||||
```go
|
||||
requirements, err := engine.InspectRequirements(ctx, promptkit.RequirementInspectionRequest{
|
||||
PromptID: "meeting.summary",
|
||||
Vars: map[string]string{"audience": "project leads"},
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, input := range requirements.Inputs {
|
||||
if input.Kind == promptkit.RequirementRequired {
|
||||
// Plan to bind this input before preparation.
|
||||
}
|
||||
if input.Kind == promptkit.RequirementOptional && input.Explanation != "" {
|
||||
// A recognizable reference may occur only on a known skipped path.
|
||||
}
|
||||
}
|
||||
for _, variable := range requirements.Variables {
|
||||
if variable.Kind == promptkit.RequirementRequired && !variable.Supplied {
|
||||
// The key is required even though this inspection did not receive it.
|
||||
}
|
||||
}
|
||||
if !requirements.Complete {
|
||||
// Preserve the partial inventory and account for its discovery issues.
|
||||
}
|
||||
```
|
||||
|
||||
Optional entries can include recognizable inputs and variables found only in
|
||||
inactive branches. A missing required variable is reported separately from
|
||||
discovery completeness: missing a value used only for direct text substitution
|
||||
does not by itself make the inventory incomplete. Conversely, an unresolved
|
||||
dependency-bearing branch can make discovery incomplete while retaining known
|
||||
conditional entries. Do not treat an empty partial inventory as proof that no
|
||||
input bindings are needed. See the method's [GoDoc](../../requirements.go) for
|
||||
the exact classifications, supported syntax, ownership, and error contract.
|
||||
|
||||
## Prepare Without Model Execution
|
||||
|
||||
[`Engine.Prepare`](../../engine.go) resolves the selected prompt and profile,
|
||||
|
||||
@@ -59,37 +59,43 @@ exist.
|
||||
## Maintainer Validation
|
||||
|
||||
For ordinary work, select validation proportionate to the change. Do not repeat
|
||||
successful checks for unchanged content merely to create a commit. The full
|
||||
release-candidate checks have one executable entry point from the repository root:
|
||||
successful checks for unchanged content merely to create a commit. During
|
||||
development, run relevant Go tests and vet, format changed Go files, and execute
|
||||
the maintained examples when they are affected:
|
||||
|
||||
```sh
|
||||
./scripts/release-check.sh
|
||||
go test ./...
|
||||
go vet ./...
|
||||
gofmt -w path/to/changed.go
|
||||
python3 scripts/check-examples.py
|
||||
git diff --check
|
||||
```
|
||||
|
||||
The [script](../scripts/release-check.sh) owns the executable checks:
|
||||
race-enabled tests, vet, the three offline consumer examples with semantic output
|
||||
assertions, tracked Go formatting, local Markdown links, module/workspace
|
||||
hygiene, whitespace, and the credential-pattern scan. It does not publish,
|
||||
change files, validate release-note structure, or retire roadmaps. It requires Go,
|
||||
Git, a POSIX shell, and Python 3. Tests and examples require no provider
|
||||
credentials or live inference calls.
|
||||
Release candidates use `./scripts/release.sh VERSION`, which runs the selected
|
||||
profile checks and publishes the tag. The [release procedure](release.md)
|
||||
describes its requirements and recovery. For an optional diagnostic of a
|
||||
committed candidate without publication, run `./scripts/releasekit.sh check
|
||||
VERSION`; this is not a required pre-publication step.
|
||||
|
||||
The script can run on intended uncommitted changes during development.
|
||||
Formatting and documentation checks use tracked files; stage intended new
|
||||
files before final validation so those checks include them. The release
|
||||
procedure separately enforces a clean, committed candidate.
|
||||
The standalone `python3 scripts/check-examples.py` command checks semantic
|
||||
expectations for the three offline consumer examples. The focused
|
||||
`python3 scripts/test-release.py` command exercises the release tooling against
|
||||
disposable local repositories; use it when changing that tooling, not as a
|
||||
recurring release gate. Tests and examples require no provider credentials or
|
||||
live inference calls.
|
||||
|
||||
Ordinary validation can run on intended uncommitted work. Release validation
|
||||
uses committed candidate contents and the selected local Go installation.
|
||||
|
||||
### Change Review
|
||||
|
||||
Inspect `git status --short --untracked-files=all` and the complete diff before
|
||||
accepting a change. Reject unintended files, credentials, private data,
|
||||
generated binaries, test output, downloaded assets, and template residue.
|
||||
The automated credential scan does not replace review of the actual change.
|
||||
|
||||
For releases, follow the [release procedure](release.md). Its publication
|
||||
command runs the full local checks once on the final committed candidate and
|
||||
automates release-note structure and Git checks. Do not repeat ordinary change
|
||||
review as a release sign-off. Roadmap retirement remains preparation housekeeping.
|
||||
For releases, follow the [release procedure](release.md). Do not repeat its
|
||||
candidate checks as a separate sign-off. Roadmap retirement remains preparation
|
||||
housekeeping.
|
||||
|
||||
Follow the [agent commit policy](policy/agents.md#git-commit-policy) for local
|
||||
commits. Ordinary commits may leave unrelated work in the workspace; preserve
|
||||
|
||||
@@ -51,7 +51,8 @@ message files on ordinary runs; construct a new engine to observe edits.
|
||||
Schema and request artifact reads still happen during preparation. Use
|
||||
`InspectConfiguration` for an explicit effective-configuration or provenance
|
||||
report, `InspectProfile` for backend-completed profile settings, and preparation
|
||||
for request-specific rendered details.
|
||||
for request-specific rendered details. For identity discovery before inspection,
|
||||
see the consumer guide's [profile and prompt listing](consumers/pkg-promptkit.md#discover-available-profiles-and-prompts).
|
||||
|
||||
## Prompt Definitions
|
||||
|
||||
@@ -124,6 +125,12 @@ Optional `session_id` uses the same template variables and input helper. Its
|
||||
rendered value is trimmed, omitted if empty, and limited to 256 Unicode code
|
||||
points. A nonblank direct request session ID bypasses the template.
|
||||
|
||||
`InspectRequirements` can enumerate recognizable dependencies from captured
|
||||
templates before artifact content is available. Its conservative syntax
|
||||
coverage does not change template validity or rendering; see the [consumer
|
||||
guide](consumers/pkg-promptkit.md#inspect-requirements-before-preparation) and
|
||||
the method's [GoDoc](../requirements.go) for that inspection boundary.
|
||||
|
||||
### Cache Control
|
||||
|
||||
Optional `cache_control` has required `type: ephemeral` and optional `ttl`,
|
||||
|
||||
@@ -78,9 +78,8 @@ The client conditionally includes:
|
||||
|
||||
The engine resolves backend, profile, and request extra-parameter maps by
|
||||
whole-map replacement rather than key merging. The resulting effective map is
|
||||
then merged directly into the top-level body after JSON serialization is
|
||||
verified. Empty keys and collisions with these reserved fields are rejected
|
||||
before any provider call:
|
||||
merged into the top-level body. Empty keys, values that cannot be encoded, and
|
||||
collisions with these reserved fields are rejected before any provider call:
|
||||
|
||||
- `model`
|
||||
- `session_id`
|
||||
|
||||
@@ -34,9 +34,13 @@ resolved request target may supply the endpoint. Generation preparation then:
|
||||
1. validates shared execution-setting invariants and the final selected base
|
||||
endpoint;
|
||||
2. maps the internal request into the OpenAI-compatible chat payload;
|
||||
3. validates and merges extra parameters;
|
||||
3. checks extra-parameter keys against empty and reserved names, then merges
|
||||
their values into the payload;
|
||||
4. composes `/chat/completions` through parsed URL path operations;
|
||||
5. resolves authentication and freezes the URL, body, and optional credential.
|
||||
5. encodes the complete body once on success. If encoding fails, extra values
|
||||
are checked individually to retain invalid-request classification; otherwise
|
||||
the complete-request failure remains a request failure;
|
||||
6. resolves authentication and freezes the URL, body, and optional credential.
|
||||
|
||||
Each execution of that prepared request creates a fresh HTTP request, body
|
||||
reader, and attempt timeout context. It closes the response body and returns
|
||||
|
||||
@@ -11,28 +11,28 @@ contributor workflow and validation.
|
||||
|
||||
| Component | Implemented responsibility | References |
|
||||
| --- | --- | --- |
|
||||
| Root `promptkit` package | Provides the supported engine facade, ConfigKit source helpers, profile and backend mapping conveniences, injection options, public requests and results, explicit inspection, observation values, opaque prepared-execution handles, value conversion, redacted formatting, and typed capacity and generation error mapping. | [Package GoDoc](../../doc.go), [observations](../../observations.go), [prepared execution](../../prepared_execution.go), [backend API](../../backends.go), [engine facade](../../engine.go) |
|
||||
| Root `promptkit` package | Provides the supported engine facade, ConfigKit source helpers, profile and backend mapping conveniences, injection options, public requests and results, prompt/profile/requirement inspection and identity listing, observation values, opaque prepared-execution handles, value conversion, redacted formatting, and typed capacity and generation error mapping. | [Package GoDoc](../../doc.go), [observations](../../observations.go), [prepared execution](../../prepared_execution.go), [backend API](../../backends.go), [engine facade](../../engine.go), [identity listing](../../listing.go), [requirement facade](../../requirements.go) |
|
||||
| `examples/go-library/prepare` | Demonstrates offline preparation and explicit configuration inspection with a prompt file, programmatic profile, and inline input. It is not a public library package. | [Example program](../../examples/go-library/prepare/main.go) |
|
||||
| `examples/go-library/run` | Demonstrates an offline downstream consumer using a prompt file, programmatic profile, inline input, an injected deterministic model client, and `Run`. It is not a public library package. | [Example program](../../examples/go-library/run/main.go) |
|
||||
| `examples/go-library/observe` | Demonstrates offline built-in HTTP retry and synchronous metadata observation across structured proposal and assessment runs using embedded sources, two profiles on one finite backend, and an application decision. It is not a public library package. | [Example program](../../examples/go-library/observe/main.go) |
|
||||
| `internal/backend` | Holds a defensive, immutable lookup and capacity-policy index for final backend definitions validated by the configuration adapter. | [Backend registry](../../internal/backend/registry.go) |
|
||||
| `internal/catalog` | Selects released keyed catalog assets and PromptKit-owned profile definitions as ConfigKit underlays; effective validation belongs to the configuration loader. | [Catalog adapter](../../internal/catalog/catalog.go), [internal sources](sources.md#profiles-and-built-ins) |
|
||||
| `internal/configuration` | Loads keyed backend, profile, and prompt configuration through ConfigKit; records effective field presence, validates final values, captures indexed prompt definitions and referenced message content, constructs backend and presence-aware profile snapshots, and retains an explicit read-only inspection view. | [Typed loader](../../internal/configuration/load.go), [internal sources](sources.md#typed-configuration-loading) |
|
||||
| `internal/configuration` | Loads keyed backend, profile, and prompt configuration through ConfigKit; records effective field presence, validates final values, captures indexed prompt definitions, referenced message content, compiled templates, input metadata snapshots, sorted identity-only profile and prompt inventories, backend and presence-aware profile snapshots, and an explicit read-only inspection view. | [Typed loader](../../internal/configuration/load.go), [identity listing](../../internal/configuration/listing.go), [internal sources](sources.md#typed-configuration-loading) |
|
||||
| `internal/capacity` | Owns engine-local bounded execution admission and FIFO model-generation permits for limited backend IDs, including cancellation-safe waiter removal and client wrapping. | [Internal capacity management](capacity.md) |
|
||||
| `internal/domain` | Defines internal framework values for requests, artifacts, prompt definitions, profiles, execution targets, rendering, generation, and validation, and owns source-neutral invariants for shared execution settings, OpenAI-compatible base endpoints, session identifiers, and output contracts. Source parsing, required fields, other source-specific normalization, defaulting, and boundary-specific error classification remain with their callers. | [Domain declarations](../../internal/domain/domain.go), [endpoint invariant](../../internal/domain/endpoint.go) |
|
||||
| `internal/engine` | Assembles selected catalog and caller configuration into one snapshot, then constructs capacity, rendering, validation, generation, repair, and the use-case runner for each engine. | [Engine assembly](../../internal/engine/engine.go), [internal sources](sources.md#profiles-and-built-ins), [internal runner](runner.md) |
|
||||
| `internal/domain` | Defines internal framework values for requests, artifacts, prompt definitions, requirement inspection, profiles, execution targets, rendering, generation, and validation, and owns source-neutral invariants for shared execution settings, OpenAI-compatible base endpoints, session identifiers, and output contracts. Source parsing, required fields, other source-specific normalization, defaulting, and boundary-specific error classification remain with their callers. | [Domain declarations](../../internal/domain/domain.go), [requirement values](../../internal/domain/requirements.go), [endpoint invariant](../../internal/domain/endpoint.go) |
|
||||
| `internal/engine` | Assembles selected catalog and caller configuration into one snapshot, then constructs capacity, rendering, validation, generation, repair, and the use-case runner for each engine. It delegates identity listing directly to the captured configuration inventory rather than the runner. | [Engine assembly](../../internal/engine/engine.go), [internal sources](sources.md#profiles-and-built-ins), [internal runner](runner.md) |
|
||||
| `internal/generation` | Coordinates frozen built-in HTTP attempts and one-shot injected calls through active backend permits, including bounded status retries, cancellable backoff, and attempt observations. | [Internal generation coordination](generation.md) |
|
||||
| `internal/defaults` | Defines application-neutral framework constants and constructs the default execution target. It contains no CLI, server, or inbound HTTP limits. | [Framework defaults](../../internal/defaults/defaults.go) |
|
||||
| `internal/filecatalog` | Provides deterministic YAML discovery and path helpers for operating-system filesystems and `fs.FS` sources. | [File catalog](../../internal/filecatalog/catalog.go) |
|
||||
| `internal/jsonvalue` | Validates and deeply copies bounded JSON-compatible extra-parameter and prepared-schema trees while preserving supported concrete value types and rejecting cycles or excessive depth and work. | [JSON values](../../internal/jsonvalue/jsonvalue.go) |
|
||||
| `internal/promptdef` | Provides a narrow captured-prompt lookup contract, not-found identity, and execution-ready definition hash. | [Prompt lookup](../../internal/promptdef/repository.go), [framework formats](../formats.md) |
|
||||
| `internal/profile` | Provides a narrow captured-profile lookup contract and not-found identity. | [Profile lookup](../../internal/profile/repository.go), [framework formats](../formats.md) |
|
||||
| `internal/prompt` | Renders prompt messages from Go templates with artifact, variable, session, and cache-control data. | [Go-template renderer](../../internal/prompt/go_renderer.go) |
|
||||
| `internal/prompt` | Renders prompt messages from Go templates with artifact, variable, session, and cache-control data, and conservatively enumerates dependencies from captured template syntax. | [Go-template renderer](../../internal/prompt/go_renderer.go), [requirement discovery](../../internal/prompt/requirements.go) |
|
||||
| `internal/artifact` | Resolves ordinary inline and unrestricted caller-selected file references into copied artifacts with metadata and hashes. | [Internal sources and validation](sources.md) |
|
||||
| `internal/validate` | Validates basic, JSON, and JSON Schema output using operating-system filesystem or `fs.FS` schema sources and creates operation-local validation plans with canonical contained schema resources. | [Framework formats](../formats.md#schemas), [internal sources and validation](sources.md) |
|
||||
| `internal/llm` | Defines the internal generation boundary and implements outbound OpenAI-compatible chat requests from resolved execution targets, including bounded structured non-success response decoding, successful-response decoding, authentication, deadline handling, and ownership of the OpenAI-compatible reserved request-field policy. | [Internal model client](llm.md) |
|
||||
| `internal/observation` | Owns invocation-local run IDs, presence-aware usage and phase accounting, and synchronous metadata event delivery for ordinary and claimed prepared runs and generation attempts. | [Internal execution observation](observation.md) |
|
||||
| `internal/usecase` | Selects captured prompts and profiles, applies request overrides and completion checks, and coordinates ordinary and single-claim prepared execution across rendering, artifact loading, operation-local validation plans, generation, capacity, and bounded repair. | [Internal runner](runner.md), [prepared-execution implementation](../../internal/usecase/prepared_execution.go) |
|
||||
| `internal/usecase` | Selects captured prompts and profiles, enumerates requirements from captured prompt syntax, applies request overrides and completion checks, and coordinates ordinary and single-claim prepared execution across rendering, artifact loading, operation-local validation plans, generation, capacity, and bounded repair. | [Internal runner](runner.md), [requirement inspection](../../internal/usecase/requirements_inspection.go), [prepared-execution implementation](../../internal/usecase/prepared_execution.go) |
|
||||
|
||||
The internal engine assembles these internal components without exposing their
|
||||
representations. Consumers depend only on the root facade.
|
||||
|
||||
@@ -32,16 +32,29 @@ The runner has no durable run or session store.
|
||||
|
||||
## Shared Prompt Selection
|
||||
|
||||
The runner uses one prompt-selection and hashing boundary for ordinary
|
||||
preparation and exact prompt inspection. Preparation retains its early
|
||||
request-ID check before direct-session normalization; both operations select
|
||||
one captured definition and its construction-time hash. Referenced message
|
||||
content and static template syntax were captured when the engine was built.
|
||||
Ordinary preparation selects the full prompt definition and construction-time
|
||||
hash. It retains its early request-ID check before direct-session
|
||||
normalization. Referenced message content and static template syntax were
|
||||
captured when the engine was built.
|
||||
|
||||
Inspection stops after that structural lookup. It does not render templates or
|
||||
touch profile, artifact, schema, renderer, validator, admission, or model
|
||||
collaborators. The root [`Engine.InspectPrompt`](../../engine.go) GoDoc owns
|
||||
the public operation's exact contract.
|
||||
`InspectPrompt` selects captured metadata through an optional repository
|
||||
capability, falling back to the full definition and existing hash resolution
|
||||
for internal repositories without it. It returns owned metadata without
|
||||
rendering templates or touching profile, artifact, schema, renderer, validator,
|
||||
admission, or model collaborators. The root
|
||||
[`Engine.InspectPrompt`](../../engine.go) GoDoc owns the public operation's
|
||||
exact contract.
|
||||
|
||||
`InspectRequirements` uses the same metadata selection to resolve the
|
||||
request-profile/default identity, then obtains the exact captured requirement
|
||||
snapshot by the selected prompt ID and version. The analyzer receives only
|
||||
those identity fields rather than an execution definition. It analyzes the
|
||||
snapshot with the supplied caller variables and normalized direct-session
|
||||
bypass; it does not resolve target readiness or enter the preparation
|
||||
pipeline. Missing snapshot capability is a prompt-load failure rather than a
|
||||
source fallback. The root
|
||||
[`Engine.InspectRequirements`](../../requirements.go) GoDoc owns the public
|
||||
operation's exact contract.
|
||||
|
||||
## Shared Profile Selection
|
||||
|
||||
@@ -92,8 +105,11 @@ one execution-precedence and error-ordering implementation while allowing a
|
||||
full backend pool to reject work before expensive schema, artifact, and
|
||||
rendering operations. `Prepare` discards the plan after returning its public
|
||||
metadata. `Run` retains the plan through initial and repaired-output validation
|
||||
and discards it when the operation ends. Prepared execution stores the same
|
||||
kind of plan only in its private payload.
|
||||
and discards it when the operation ends. Prepared execution transfers the
|
||||
completed preparation's owned message, input-hash, and copied-schema values
|
||||
into its private payload, while deep-copying mutable provider extras and
|
||||
creating an independent details snapshot. Claim completion or discard clears
|
||||
the execution payload without removing the details snapshot.
|
||||
|
||||
Pointer-based numeric overrides preserve an explicit zero. Invalid negative or
|
||||
out-of-range values fail as invalid requests. Endpoint overrides do not change
|
||||
|
||||
@@ -21,6 +21,21 @@ profile inheritance. It retains ConfigKit's result privately for explicit
|
||||
inspection, not for per-run resolution. The runtime/domain packages do not
|
||||
depend on ConfigKit.
|
||||
|
||||
The `ConfigurationInspection.Diff` facade method compares two retained results
|
||||
through a private wrapper and a narrow `internal/configuration.Diff` bridge.
|
||||
The wrapper keeps `Result[document]` inside the adapter; the bridge delegates
|
||||
once to ConfigKit v0.7.0 and returns its public difference records. Comparison
|
||||
uses only frozen effective values and does not reopen sources, resolve runtime
|
||||
credentials, or enter the execution path. ConfigKit applies sensitivity from
|
||||
both results; reports may still contain ordinary application data. See the
|
||||
[consumer comparison guide](../consumers/pkg-promptkit.md#inspect-the-captured-configuration)
|
||||
and ConfigKit's [v0.7.0 comparison contract](https://gitea.maximumdirect.net/eric/ConfigKit/src/tag/v0.7.0/docs/config.md#effective-configuration-diffs).
|
||||
|
||||
After prompt materialization and effective-profile snapshot construction, the
|
||||
adapter captures lexically ordered, identity-only inventories from the final
|
||||
prompt and indexed-profile maps. Its listing methods return caller-owned copies
|
||||
of those inventories without consulting ConfigKit or copying definitions.
|
||||
|
||||
The root `DirectoryInputs` and `FilesystemInputs` helpers discover sorted YAML
|
||||
files under an explicit root, provide resource context, and leave imports
|
||||
disabled. Each returned slice belongs to one strict-composition group; file
|
||||
@@ -35,21 +50,68 @@ The configuration adapter indexes each prompt by exact ID and version,
|
||||
materializes file-backed message content using the exact effective field's
|
||||
origin and resource context, and checks static message/session template syntax
|
||||
at construction. It stores captured content, compiled templates, declared
|
||||
input metadata, and a definition hash. It closes content-file handles before
|
||||
input metadata, and a definition hash. Alongside the unchanged hashed
|
||||
definition, it retains content-type and description presence for each declared
|
||||
input and provides an internal requirement snapshot with those declarations
|
||||
and the captured parses. Metadata values come from the final normalized
|
||||
declaration; absent and null fields remain distinguishable from explicit
|
||||
strings, including empty strings. It closes content-file handles before
|
||||
returning and retains no source capability for public inspection.
|
||||
|
||||
Role and cache-control normalization is applied to execution-owned message
|
||||
copies. The retained ConfigKit configuration keeps effective input message
|
||||
metadata and its resource origins: recording a sequence-wide normalization in
|
||||
ConfigKit v0.5.0 would replace origins of unchanged content-file fields.
|
||||
copies. ConfigKit v0.7.0 retains effective input message metadata and its
|
||||
resource origins; recording a sequence-wide normalization in the retained
|
||||
configuration would replace origins of unchanged content-file fields.
|
||||
|
||||
The runner selects from this index for preparation and `InspectPrompt`.
|
||||
Neither path rescans configuration or reopens message files. `InspectPrompt`
|
||||
The runner selects full definitions from this index for preparation and uses
|
||||
captured prompt metadata for `InspectPrompt` and `InspectRequirements`; the
|
||||
internal requirement snapshot lookup selects by exact prompt ID and version.
|
||||
These paths do not rescan configuration or reopen message files. `InspectPrompt`
|
||||
returns declared metadata without rendering or resolving profile, input, or
|
||||
schema resources. Runtime rendering still checks request variables and input
|
||||
helpers. The [prompt adapter tests](../../internal/configuration/prompts_test.go)
|
||||
schema resources, and its public shape and prompt hashes do not include the
|
||||
captured presence state. Runtime rendering still checks request variables and
|
||||
input helpers. The [prompt adapter tests](../../internal/configuration/prompts_test.go)
|
||||
cover resource origins, context containment, capture, and selection.
|
||||
|
||||
The configuration adapter shares exact ID/version selection across its
|
||||
internal prompt lookups. The execution snapshot lookup returns an owned prompt
|
||||
definition and cached hash; metadata inspection returns owned input metadata
|
||||
without copying execution templates. Profile selection likewise has an
|
||||
identity-only lookup for consumers that do not need the execution target,
|
||||
while the full profile lookup continues to return an owned target and deep copy
|
||||
of provider extras. These lookups use captured indexes and do not query
|
||||
ConfigKit again.
|
||||
|
||||
The `internal/prompt` requirement walker uses the captured parse trees and
|
||||
declaration metadata without executing templates or reading artifacts. It
|
||||
collects literal `input` names, direct root caller keys, and literal-key
|
||||
`index` lookups. For `if` and `with`, it classifies branches when their
|
||||
conditions are simple supported values: direct values, literals, negation, or
|
||||
two-string equality/inequality. It uses Go template truthiness, so a supplied
|
||||
nonempty string such as `false` is true. Missing decision values and
|
||||
content-dependent conditions leave dependency-bearing branches conditional
|
||||
and report incomplete discovery; unresolved conditions with no branch
|
||||
dependencies do not by themselves make discovery incomplete. Recognizable
|
||||
references on known skipped paths remain optional inventory entries with an
|
||||
inactive-use explanation, unless another applicable use establishes a
|
||||
stronger classification. Skipped expressions are scanned only for
|
||||
syntactically recognizable names and are not evaluated. A direct caller value
|
||||
or literal-key lookup can also supply an input name, including through simple
|
||||
parentheses or a direct pipeline to `input`; names are used exactly as
|
||||
supplied. Unresolved names and unsupported dependency-bearing expressions
|
||||
produce structured limitations while retaining recognizable references.
|
||||
Where an unresolved name or scope could affect existing optional entries, those
|
||||
entries are reported as conditional rather than inactive-only or definitively
|
||||
optional. Direct root-map `with` forms retain caller-map identity. Known scalar
|
||||
values remain separate from unknown scopes; accesses through untracked local
|
||||
maps report incomplete discovery rather than silently losing caller keys.
|
||||
Unsupported name expressions reuse value analysis's conservative syntax scan
|
||||
instead of recursively rescanning the same subtree.
|
||||
Invoked named templates are scanned conservatively, and
|
||||
dependency-bearing range scopes make discovery incomplete. Uncalled definitions
|
||||
and bypassed session templates are not traversed. This internal walker does not
|
||||
change rendering behavior.
|
||||
|
||||
## Profiles And Built-Ins
|
||||
|
||||
`internal/catalog` selects the released keyed OpenRouter and Rakestrawhome
|
||||
@@ -66,10 +128,16 @@ snapshots with backend defaults and presence/suppression state. A profile can
|
||||
remain partial until runtime overrides complete it. `InspectProfile` selects
|
||||
that captured target without prompt, artifact, schema, credential lookup, or
|
||||
provider work. Preparation applies runtime overrides and then checks target
|
||||
completeness. The [target tests](../../internal/configuration/targets_test.go)
|
||||
cover effective target interpretation, while the [catalog tests](../../internal/catalog/catalog_test.go)
|
||||
completeness. Identity-only profile selection can identify a partial captured
|
||||
profile without copying its target or requiring readiness. The
|
||||
[target tests](../../internal/configuration/targets_test.go) cover effective
|
||||
target interpretation, while the [catalog tests](../../internal/catalog/catalog_test.go)
|
||||
cover selected released assets.
|
||||
|
||||
The internal engine delegates listing through a narrow identity-inventory
|
||||
interface directly to the configuration snapshot. The execution runner has no
|
||||
listing responsibility.
|
||||
|
||||
## Ordinary Artifacts
|
||||
|
||||
`internal/artifact` accepts explicitly typed inline references even when their
|
||||
@@ -79,10 +147,15 @@ and after opening the file. It copies content into an artifact, records
|
||||
metadata and an opaque content-equality value, and applies a content-type
|
||||
fallback.
|
||||
|
||||
Regular files are read synchronously in bounded chunks. Cancellation is
|
||||
checked before opening, before and after every read, and before publishing the
|
||||
artifact, so a canceled read never publishes partial content. The ordinary
|
||||
reader does not detach file reads into background goroutines.
|
||||
Regular files are read synchronously in bounded chunks. The opened file's size
|
||||
is only an initial-capacity hint, capped at 1 MiB; reads continue to EOF, and
|
||||
the returned body determines artifact size and hash. If an overreported hint
|
||||
leaves unused capacity greater than the actual body length, the completed body
|
||||
is copied to release the excess backing storage, including when the body is
|
||||
empty. Cancellation is checked before opening, before and after every read,
|
||||
and before publishing the artifact, so a
|
||||
canceled read never publishes partial content. The ordinary reader does not
|
||||
detach file reads into background goroutines.
|
||||
|
||||
This ordinary reader does not implement an inbound HTTP security boundary. In
|
||||
particular, it does not constrain files to an application root or impose an
|
||||
@@ -106,7 +179,9 @@ between artifact conversion chunks, around each message, and before publishing
|
||||
the complete prompt. It cannot interrupt template work already in progress and
|
||||
never publishes a partial prompt after observing cancellation. It validates and
|
||||
canonicalizes message roles before carrying roles, session IDs, and cache
|
||||
control into the rendered prompt. The
|
||||
control into the rendered prompt. Message and session templates execute into
|
||||
per-call string builders, so their rendered strings do not require a final
|
||||
buffer-to-string copy. The
|
||||
[renderer tests](../../internal/prompt/renderer_test.go) own rendering behavior.
|
||||
|
||||
## Schemas And Output Validation
|
||||
@@ -114,7 +189,8 @@ control into the rendered prompt. The
|
||||
`internal/validate` provides validators backed by an operating-system
|
||||
filesystem or an `fs.FS`. Invalid generated content is returned as a validation
|
||||
result; inability to load, register, or compile a schema is an operational
|
||||
error.
|
||||
error. Basic validation checks body emptiness on the byte slice directly
|
||||
instead of first making a response-sized string copy.
|
||||
|
||||
Every preparation operation creates one operation-local validation plan. None,
|
||||
basic, and JSON modes retain the effective output contract without source
|
||||
|
||||
@@ -6,29 +6,37 @@ Material: policy documents under `docs/policy/` and the Policy Reading Guide in
|
||||
`docs/development.md`, adapted for this repository; root `AGENTS.md`, copied
|
||||
without modification from upstream `agents/AGENTS.md`.
|
||||
|
||||
The release policy uses upstream commit
|
||||
`e833faddf120c789a9b45348edaf75720de16348`; the remaining material uses
|
||||
`d84a38ebab26f048cafbe59a2246700c35d29843`.
|
||||
|
||||
Copyright (C) 2026 Eric Rakestraw
|
||||
|
||||
SPDX-License-Identifier: CC-BY-SA-4.0
|
||||
|
||||
License and notices: [CC BY-SA 4.0](licenses/CC-BY-SA-4.0.txt) |
|
||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/COPYRIGHT) |
|
||||
[LICENSE](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/LICENSE).
|
||||
[LICENSE](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/LICENSE) |
|
||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/policy/src/commit/e833faddf120c789a9b45348edaf75720de16348/COPYRIGHT) |
|
||||
[LICENSE](https://gitea.maximumdirect.net/eric/policy/src/commit/e833faddf120c789a9b45348edaf75720de16348/LICENSE).
|
||||
|
||||
[Source](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843).
|
||||
|
||||
[Source](https://gitea.maximumdirect.net/eric/policy/src/commit/e833faddf120c789a9b45348edaf75720de16348).
|
||||
|
||||
## ConfigKit
|
||||
|
||||
Selected module: `gitea.maximumdirect.net/eric/ConfigKit` at `v0.5.0`.
|
||||
Selected module: `gitea.maximumdirect.net/eric/ConfigKit` at `v0.7.1`.
|
||||
|
||||
Copyright (C) 2026 Eric Rakestraw
|
||||
|
||||
SPDX-License-Identifier: LGPL-3.0-or-later
|
||||
|
||||
License and notices: [LGPL 3.0](../../LICENSE) |
|
||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/ConfigKit/src/commit/dc09959cca4e105f49ec316c6d4ba585bd2340bd/COPYRIGHT) |
|
||||
[LICENSE](https://gitea.maximumdirect.net/eric/ConfigKit/src/commit/dc09959cca4e105f49ec316c6d4ba585bd2340bd/LICENSE).
|
||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/ConfigKit/src/commit/6fffde4af8f31f99d3ee295c426eb3234cb0fb77/COPYRIGHT) |
|
||||
[LICENSE](https://gitea.maximumdirect.net/eric/ConfigKit/src/commit/6fffde4af8f31f99d3ee295c426eb3234cb0fb77/LICENSE).
|
||||
|
||||
[Source](https://gitea.maximumdirect.net/eric/ConfigKit/src/commit/dc09959cca4e105f49ec316c6d4ba585bd2340bd).
|
||||
[Source](https://gitea.maximumdirect.net/eric/ConfigKit/src/commit/6fffde4af8f31f99d3ee295c426eb3234cb0fb77).
|
||||
|
||||
## Promptkit OpenRouter Catalog v3
|
||||
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
# Release Policy
|
||||
|
||||
**Base Policy Version:** 1.4
|
||||
**Base Policy Version:** 1.6
|
||||
|
||||
**Last Updated**: 2026-09-27
|
||||
**Last Updated**: 2026-10-08
|
||||
|
||||
**Upstream Source:** [release.md](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/docs/policy/release.md)
|
||||
**Upstream Source:** [release.md](https://gitea.maximumdirect.net/eric/policy/src/commit/e833faddf120c789a9b45348edaf75720de16348/docs/policy/release.md)
|
||||
|
||||
## Purpose
|
||||
|
||||
@@ -12,6 +12,9 @@ This policy defines a proportionate workflow for validating and publishing an
|
||||
intended source revision under a fresh version. Releases should require little
|
||||
routine intervention, avoid repeated work, and preserve published contents.
|
||||
|
||||
Adopt stronger verification or reproducibility requirements through an explicit
|
||||
project-specific decision justified by the intended audience and support needs.
|
||||
|
||||
The repository-specific commands belong in `docs/release.md`. Historical release
|
||||
summaries belong in `docs/releases/`. The procedure must agree with the adopted
|
||||
policy and link to it rather than introduce additional release requirements.
|
||||
@@ -59,7 +62,8 @@ requires a deliberate update to the policy and procedure where needed.
|
||||
For releases at or after `v1.0.0`:
|
||||
|
||||
- a major release may make intentional incompatible changes;
|
||||
- a minor release may add backward-compatible behavior; and
|
||||
- a minor release may add backward-compatible behavior or raise the Go consumer
|
||||
minimum as described below; and
|
||||
- a patch release may make backward-compatible fixes.
|
||||
|
||||
Before `v1.0.0`, a minor release may change a documented public, operational,
|
||||
@@ -71,14 +75,44 @@ Additional compatibility promises and deliberate exceptions belong in the
|
||||
adopted policy or canonical contract documentation. Release notes explain
|
||||
changes; they do not redefine current compatibility contracts.
|
||||
|
||||
## Release Documentation
|
||||
## Consumer Minimum And Release Toolchain
|
||||
|
||||
Distinguish the minimum toolchain supported for consumers from the toolchain
|
||||
maintainers use to build and validate a release. Record the consumer minimum in
|
||||
the language's standard metadata and explain its support meaning in canonical
|
||||
documentation. Define the selected release toolchain and its setup commands in
|
||||
`docs/release.md`. Toolchain selection and upgrade cadence are repository-specific
|
||||
choices.
|
||||
|
||||
For Go repositories, normally declare the initial stable release of the selected
|
||||
series as the consumer minimum, such as `go 1.26.0`. Use a higher patch minimum
|
||||
when a dependency or an identified correctness fix requires it, and document
|
||||
the reason.
|
||||
|
||||
For release-policy purposes, successful build and release validation on the
|
||||
selected Go patch release are sufficient to presume support across that Go
|
||||
series, subject to the declared consumer minimum and concrete evidence of
|
||||
incompatibility. Investigate specific compatibility concerns as they arise.
|
||||
|
||||
Use the selected release toolchain for candidate validation and release builds,
|
||||
including CI jobs performing those tasks. A newer validation patch version alone
|
||||
leaves the consumer minimum unchanged.
|
||||
|
||||
A minor release may raise the Go consumer minimum, including from one Go series
|
||||
to the next, both before and after `v1.0.0`. This is an explicit exception to the
|
||||
backward-compatibility requirement for minor releases. Patch releases preserve
|
||||
the consumer minimum. Explain an increase's impact and required consumer action
|
||||
in the release note. Other incompatible changes remain subject to the versioning
|
||||
rules above.
|
||||
|
||||
## Release Documentation And Preparation
|
||||
|
||||
Every new release must include `docs/releases/<version>.md` in the tagged
|
||||
source tree. Use the complete version tag as the filename and begin with a
|
||||
title containing the product name and that exact version.
|
||||
|
||||
Include a short summary and any relevant compatibility impact or upgrade
|
||||
instructions. Use the Promptkit section structure below and keep the note
|
||||
instructions. Choose sections suited to the release and keep the note
|
||||
proportionate to the change.
|
||||
|
||||
Keep exact current contracts in their canonical documents and update affected
|
||||
@@ -186,7 +220,8 @@ release without replacing its contents or reusing its version.
|
||||
Maintain `docs/release.md` as a short, executable maintainer runbook. It must
|
||||
identify:
|
||||
|
||||
1. prerequisites, primary branch, and authoritative remote;
|
||||
1. prerequisites, selected release toolchain, primary branch, and authoritative
|
||||
remote;
|
||||
2. version and release-note preparation;
|
||||
3. one publication command with its selected local checks and automated guards;
|
||||
4. declared deliverables, required build targets, and automation responsibilities;
|
||||
@@ -210,20 +245,39 @@ annotated stable SemVer tags. It has no binaries, hosted download pages, or
|
||||
asynchronous deliverables. The source commit is pushed through main before the
|
||||
tag. The [release procedure](../release.md) owns commands and recovery.
|
||||
|
||||
The existing publication script requires Summary, Compatibility, Upgrade, and
|
||||
Changes headings beneath a title of `# Promptkit vMAJOR.MINOR.PATCH`. State
|
||||
“No action required” or “None” where appropriate. This retained local structure
|
||||
matches the automated release-note guard; changing it is separate work.
|
||||
Release preparation uses the latest Go installation available in the development
|
||||
environment to build the maintainer release executable and validate the
|
||||
candidate, including the consumer examples. Keep that same local toolchain
|
||||
active throughout those operations.
|
||||
|
||||
The [publication script](../../scripts/release.sh) runs the
|
||||
[local checks](../../scripts/release-check.sh) once for the final committed
|
||||
candidate: race-enabled tests, vet, three offline consumer examples, tracked
|
||||
Go formatting, Markdown links, module/workspace hygiene, whitespace, and a
|
||||
credential-pattern scan. These protect concurrency, assembled library use,
|
||||
source distribution, and documentation integrity. A separate ordinary test run
|
||||
or standalone build is not an additional release gate. Candidate, tag, and
|
||||
synchronized-main checks remain automated. No CI or module-availability wait
|
||||
is required. Licensing footer URLs are preparation work, not a network gate.
|
||||
For a minor release, advance Promptkit's consumer minimum to the initial stable
|
||||
release of that Go series, retaining a higher patch minimum when required by a
|
||||
dependency or an identified correctness fix. Patch releases preserve the
|
||||
existing consumer minimum. Commit any module update before validating the final
|
||||
candidate. The [release procedure](../release.md#prepare-the-candidate) owns
|
||||
setup commands, and the consumer guide explains
|
||||
[Go compatibility](../consumers/pkg-promptkit.md#go-compatibility).
|
||||
|
||||
Use a title containing `Promptkit` and the exact version, followed by a short
|
||||
summary and any relevant compatibility impact or upgrade instructions. The
|
||||
note is committed with the candidate and supplies the annotated tag message.
|
||||
No fixed set of section headings is required.
|
||||
|
||||
The [publication command](../../scripts/release.sh) delegates to the released
|
||||
ReleaseKit tool selected in `go.mod`. The committed
|
||||
[profile](../../releasekit.json) selects the race-enabled Go suite, full vet,
|
||||
committed Go formatting, local Markdown links, workspace and filesystem
|
||||
replacement inspection, and the offline semantic checks for the three
|
||||
maintained consumer examples. These are the selected checks for the final
|
||||
candidate; a separate ordinary test run or standalone build is not an
|
||||
additional release gate. The upstream tool owns candidate, remote, and tag
|
||||
publication checks. It returns native status 0 for confirmed publication, 1
|
||||
for a known validation or publication failure, 2 for invalid input or profile,
|
||||
and 3 when the push outcome is unknown. For status 3, resolve the exact
|
||||
captured publication endpoint before retrying; the repository-specific
|
||||
[procedure](../release.md#failure-recovery) links to the upstream recovery
|
||||
instructions. Do not wait for CI or Go module availability. Licensing footer
|
||||
URLs are preparation work, not a network gate.
|
||||
|
||||
## Catalog Profile Compatibility
|
||||
|
||||
|
||||
@@ -58,17 +58,24 @@ and repository hygiene.
|
||||
Introducing hosted CI later would supplement, not silently redefine, this
|
||||
documented validation model.
|
||||
|
||||
Full release-candidate validation includes race-enabled package tests, explicit
|
||||
static analysis, and execution with automated semantic checks of the three maintained
|
||||
offline consumer examples. The test run builds the packages; the examples
|
||||
also exercise ordinary compilation. Separate ordinary tests and a standalone
|
||||
build are not additional release gates. Ordinary tests remain useful during
|
||||
development. The preparation example protects assembled preparation and
|
||||
The selected ReleaseKit profile runs the race-enabled package suite and full
|
||||
vet, checks committed Go formatting, local Markdown links, and module
|
||||
distribution hygiene, and executes semantic assertions for the three
|
||||
maintained offline consumer examples. ReleaseKit owns this shared validation
|
||||
machinery and publication workflow. The test run builds the packages; the
|
||||
examples also exercise ordinary compilation. Separate ordinary tests and a
|
||||
standalone build are not additional release gates. Ordinary tests remain useful
|
||||
during development. The preparation example protects assembled preparation and
|
||||
inspection behavior. The execution example separately protects assembled
|
||||
`Run`, injected-client, validation, usage, and result behavior. The observation
|
||||
example protects assembled built-in retries, structured proposal and assessment
|
||||
runs, and application-owned interpretation of metadata.
|
||||
|
||||
The standalone `python3 scripts/test-release.py` command exercises Promptkit's
|
||||
ReleaseKit wiring against disposable local repositories. It is focused
|
||||
development validation when release tooling changes, not part of the default
|
||||
Go test suite or a recurring release check.
|
||||
|
||||
Tests in the default suite must be deterministic, offline, and independent of
|
||||
real credentials. They must not invoke paid APIs, use live network
|
||||
dependencies, or depend on mutable external services. Tests that require live
|
||||
|
||||
157
docs/release.md
157
docs/release.md
@@ -1,64 +1,101 @@
|
||||
# Release Procedure
|
||||
|
||||
This runbook implements the [release policy](policy/release.md). A successful
|
||||
push of the selected annotated tag to origin completes the release. Never wait
|
||||
for CI to finish or succeed.
|
||||
This runbook implements the [release policy](policy/release.md). Promptkit
|
||||
publishes source-only Go module tags from `main` to `origin`. A successful push
|
||||
of the selected annotated tag, after its source commit is available on `main`,
|
||||
completes the release. Do not wait for CI or Go module availability.
|
||||
|
||||
## Prepare The Candidate
|
||||
|
||||
Required tools are Git, Go with race-detector support, POSIX shell, and Python
|
||||
3. Publication requires push access to origin. Work from the repository root.
|
||||
3. Work from the repository root with push access to `origin`.
|
||||
|
||||
Select an unused version and create `docs/releases/<version>.md` using the
|
||||
[required structure](policy/release.md#release-documentation). Describe
|
||||
compatibility impact and consumer actions and update affected canonical docs.
|
||||
For catalog updates, record selected catalog versions and any consumer action,
|
||||
linking to the catalog policy rather than repeating its contracts.
|
||||
Use the latest Go installation available locally for the release tool, the
|
||||
candidate validation, and the maintained examples. Make it the active `go`
|
||||
command and record its version:
|
||||
|
||||
Follow the [OSS policy](policy/oss.md) when preparing licensing and attribution
|
||||
changes. Each new note must end with the following navigation, replacing
|
||||
`vMAJOR.MINOR.PATCH` with the selected release tag:
|
||||
|
||||
```markdown
|
||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit/src/tag/vMAJOR.MINOR.PATCH/COPYRIGHT) | [LICENSE](https://gitea.maximumdirect.net/eric/promptkit/src/tag/vMAJOR.MINOR.PATCH/LICENSE) | [ATTRIBUTION](https://gitea.maximumdirect.net/eric/promptkit/src/tag/vMAJOR.MINOR.PATCH/docs/oss/attribution.md)
|
||||
```sh
|
||||
export GOTOOLCHAIN=local
|
||||
go version
|
||||
```
|
||||
|
||||
Check these file paths against the candidate; a future tag need not resolve
|
||||
remotely during preparation. The annotated tag derives from this same note,
|
||||
including its footer. Promptkit publishes no separate hosted release body or
|
||||
download page. If such a channel is deliberately introduced, reuse the note
|
||||
and preserve its footer. No extra network checks, artifacts, or approval gates
|
||||
are introduced, and published notes are not retrofitted.
|
||||
For a minor release, advance Promptkit's consumer minimum to the initial stable
|
||||
release of that Go series before committing the candidate:
|
||||
|
||||
Retire completed roadmaps, preserving planned or deferred work and repairing
|
||||
links. This is housekeeping, not a separate approval gate. Review changes
|
||||
through the ordinary development workflow, commit them, and push main to
|
||||
origin. No repeated release-specific diff review is required.
|
||||
```sh
|
||||
release_go_version=$(go env GOVERSION)
|
||||
release_go_series=${release_go_version#go}
|
||||
release_go_series=${release_go_series%.*}
|
||||
go mod edit "-go=${release_go_series}.0"
|
||||
```
|
||||
|
||||
For example, Go 1.26.8 normally results in `go 1.26.0`. Keep a higher patch
|
||||
minimum if a dependency or identified correctness fix requires it. For a patch
|
||||
release, leave the consumer minimum unchanged; using a newer validation
|
||||
toolchain does not itself raise that minimum. If the candidate needs a higher
|
||||
minimum, prepare a minor release instead. See the
|
||||
[consumer-minimum policy](policy/release.md#consumer-minimum-and-release-toolchain).
|
||||
|
||||
Prepare module caches before validation; this may update `go.sum` and belongs
|
||||
before the candidate commit:
|
||||
|
||||
```sh
|
||||
GOTOOLCHAIN=local go mod download
|
||||
```
|
||||
|
||||
Choose an unused stable version and create `docs/releases/<version>.md` with a
|
||||
title containing `Promptkit` and that exact version, a short summary, and any
|
||||
relevant compatibility impact or upgrade instructions. No fixed section
|
||||
headings are required. Describe a raised Go minimum and its compatibility
|
||||
impact under the [versioning policy](policy/release.md#versioning-and-compatibility).
|
||||
End each new note with the licensing footer below, using the selected version:
|
||||
|
||||
```markdown
|
||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit/src/tag/vX.Y.Z/COPYRIGHT) | [LICENSE](https://gitea.maximumdirect.net/eric/promptkit/src/tag/vX.Y.Z/LICENSE) | [ATTRIBUTION](https://gitea.maximumdirect.net/eric/promptkit/src/tag/vX.Y.Z/docs/oss/attribution.md)
|
||||
```
|
||||
|
||||
Check the footer paths against the candidate; the future tag need not resolve
|
||||
remotely during preparation. Follow the [OSS policy](policy/oss.md) for these
|
||||
links. Promptkit publishes no binaries, hosted download page, or other
|
||||
asynchronous deliverables.
|
||||
|
||||
Retire completed roadmaps while preserving deferred work and repairing incoming
|
||||
links. Review the intended changes through the ordinary development workflow,
|
||||
commit the candidate on `main`, and push `main` to `origin` before publication.
|
||||
Preparation may update module metadata; the release bootstrap and validation
|
||||
preserve the committed candidate.
|
||||
|
||||
## Validate And Publish
|
||||
|
||||
Run the following with the chosen unused version (replace the placeholder):
|
||||
For an optional local diagnostic of a committed candidate, run:
|
||||
|
||||
```sh
|
||||
./scripts/releasekit.sh check vMAJOR.MINOR.PATCH
|
||||
```
|
||||
|
||||
This is not a required step before publication. The upstream
|
||||
[candidate-check contract](https://gitea.maximumdirect.net/eric/ReleaseKit/src/tag/v0.4.1/docs/consumers/check.md)
|
||||
describes its requirements.
|
||||
|
||||
Run the single publication command with the chosen version:
|
||||
|
||||
```sh
|
||||
./scripts/release.sh vMAJOR.MINOR.PATCH
|
||||
```
|
||||
|
||||
The [publication script](../scripts/release.sh) checks a clean, synchronized
|
||||
main candidate, the version and unused tag, and the release note's title and
|
||||
required headings. It runs the [local checks](../scripts/release-check.sh)
|
||||
once, rechecks candidate identity and publication state, generates the
|
||||
annotation from the committed note plus the validated commit, and pushes only
|
||||
the selected annotated tag.
|
||||
The familiar wrapper delegates to the native ReleaseKit executable selected in
|
||||
`go.mod`. The committed [profile](../releasekit.json) selects the race-enabled
|
||||
Go suite, full vet, committed Go formatting, local Markdown links, workspace
|
||||
and filesystem replacement inspection, and semantic checks for the three
|
||||
maintained offline consumer examples. ReleaseKit also verifies the committed
|
||||
candidate and publication state, then publishes only the selected tag. Its
|
||||
checks run once for this candidate; do not repeat them as a separate sign-off.
|
||||
|
||||
There is no separate tag-message authoring, example-output inspection,
|
||||
metadata inspection, or human sign-off. Compatibility and release-note content
|
||||
remain ordinary editorial responsibilities. Development validation is useful
|
||||
but does not require a second full run before invoking the publication script.
|
||||
|
||||
The script stops successfully after the tag push. CI completion, CI success,
|
||||
remote read-back, and Go module availability are not release completion gates.
|
||||
Do not wait or poll for them. Provider probes and inference calls are never
|
||||
part of release validation.
|
||||
The native command returns status 0 for confirmed publication, 1 for a known
|
||||
validation or publication failure, 2 for invalid input or profile, and 3 when
|
||||
the push outcome remains unknown. Successful publication needs no remote
|
||||
read-back, CI wait, or installation check. Provider probes and inference calls
|
||||
are not part of validation.
|
||||
|
||||
## Catalog Profile Compatibility
|
||||
|
||||
@@ -67,28 +104,32 @@ catalog compatibility boundary. This heading preserves older incoming links.
|
||||
|
||||
## Failure Recovery
|
||||
|
||||
If local checks fail, fix the candidate, commit and push changes, then rerun
|
||||
the publication command. It never automatically removes a tag.
|
||||
For status 1, use the diagnostic to distinguish a validation failure from a
|
||||
known publication conflict. Correct and commit candidate defects, push `main`,
|
||||
and rerun publication with an unused version when appropriate. Status 2 means
|
||||
the invocation or profile must be corrected.
|
||||
|
||||
If a push fails or its result is ambiguous, set `RELEASE_VERSION` to the
|
||||
attempted version and inspect origin before retrying:
|
||||
For status 3, do not retry or remove the local tag until the remote outcome is
|
||||
resolved. ReleaseKit reports the attempted version, tag object, and recovery
|
||||
state, including the captured publication endpoint when available. Use that
|
||||
endpoint—not merely the current `origin` URL—to query the exact tag; follow the
|
||||
upstream
|
||||
[failure-recovery procedure](https://gitea.maximumdirect.net/eric/ReleaseKit/src/tag/v0.4.1/docs/release.md#resolve-failures):
|
||||
|
||||
```sh
|
||||
git ls-remote --tags origin "refs/tags/$RELEASE_VERSION" "refs/tags/$RELEASE_VERSION^{}"
|
||||
git rev-parse "refs/tags/$RELEASE_VERSION" "refs/tags/$RELEASE_VERSION^{}"
|
||||
releasekit_endpoint='COPY_CAPTURED_ENDPOINT'
|
||||
releasekit_version='vX.Y.Z'
|
||||
test "$(git ls-remote --get-url -- "$releasekit_endpoint")" = "$releasekit_endpoint" &&
|
||||
git ls-remote --refs -- "$releasekit_endpoint" "refs/tags/$releasekit_version"
|
||||
```
|
||||
|
||||
Matching remote tag and peeled commit values confirm publication, even if the
|
||||
push command reported an error. If the remote query fails, publication status
|
||||
is still unknown. Do not delete or recreate the local tag until absence on
|
||||
origin is confirmed. Once confirmed unpublished, an unsuccessful attempt's
|
||||
local tag can be removed and the publication command rerun. Never overwrite
|
||||
a published tag.
|
||||
|
||||
CI or module-resolution problems may be investigated separately; temporary
|
||||
service, network, or authentication failures do not require another release.
|
||||
Only an actual defect requiring changes to published contents warrants a new
|
||||
appropriate version. See the [recovery policy](policy/release.md#immutability-and-failure-recovery).
|
||||
Compare the returned object ID with the reported tag object. An exact match
|
||||
confirms publication. A missing ref confirms absence only when the endpoint
|
||||
identity check and query both succeed. If either check fails, the outcome
|
||||
remains unknown; correct the transport configuration and resolve it before
|
||||
retrying. Only after authoritative absence is confirmed may an unpublished
|
||||
local tag be removed before correcting the candidate or retrying. Never move,
|
||||
overwrite, or delete a published tag.
|
||||
|
||||
## Policy Changes
|
||||
|
||||
|
||||
74
docs/releases/v0.12.0.md
Normal file
74
docs/releases/v0.12.0.md
Normal file
@@ -0,0 +1,74 @@
|
||||
# Promptkit v0.12.0
|
||||
|
||||
This release replaces Promptkit's configuration loader with ConfigKit and
|
||||
migrates maintained catalogs to keyed `/v3` assets. It intentionally changes
|
||||
the public construction API and configuration format.
|
||||
|
||||
## Summary
|
||||
|
||||
Backends, profiles, and versioned prompts now share one layered configuration
|
||||
snapshot built when `NewEngine` is called. ConfigKit supplies composition,
|
||||
inheritance, presence, and source history; Promptkit validates the final
|
||||
effective definitions and captures prompt content and templates. Consumers can
|
||||
inspect the captured configuration explicitly without triggering a reload.
|
||||
|
||||
## Compatibility
|
||||
|
||||
This is an incompatible minor release under Promptkit's pre-1.0
|
||||
[versioning policy](../policy/release.md#versioning-and-compatibility). The
|
||||
former source options, `Config.PromptDir`, `Config.ProfileDir`, and the old
|
||||
profile/backend convenience values have been removed. Configuration documents
|
||||
now use literal keys beneath `backends`, `profiles`, and `prompts`; embedded
|
||||
`id`/`version` fields and `base_profile` are no longer accepted. Source groups
|
||||
compose strictly within a group and overlay in the supplied order. Explicit
|
||||
zero, false, empty, and null values can now differ from omission; maps overlay
|
||||
recursively, and an empty map does not clear an inherited map.
|
||||
|
||||
All final definitions, including unused ones, are validated during engine
|
||||
construction. Prompt text and static templates are captured then; an existing
|
||||
engine does not automatically reload changed configuration or message files.
|
||||
Request artifacts, schemas, credentials, and provider operations retain their
|
||||
runtime boundaries. See the [format reference](../formats.md) for the current
|
||||
configuration contract and the [consumer migration guide](../consumers/pkg-promptkit.md#migrate-from-the-previous-configuration-api)
|
||||
for API and source-selection changes.
|
||||
|
||||
Maintained OpenRouter and Rakestrawhome catalogs are selected as `/v3` modules,
|
||||
both at `v3.20260930.0`. Their asset format changed; consumers that read catalog
|
||||
assets directly must adopt the keyed format. The maintained Promptkit profile
|
||||
IDs remain available under the [catalog compatibility policy](../policy/release.md#catalog-profile-compatibility),
|
||||
which does not promise fixed profile data.
|
||||
|
||||
## Upgrade
|
||||
|
||||
```sh
|
||||
go get gitea.maximumdirect.net/eric/promptkit@v0.12.0
|
||||
```
|
||||
|
||||
Replace removed source and backend registration options with
|
||||
`Config.Sources`, explicitly ordered ConfigKit underlays, and optionally a
|
||||
highest-priority main input. Use `DirectoryInputs` or `FilesystemInputs` for
|
||||
selected directories, `ProfileSource` and `BackendSource` for typed
|
||||
programmatic values, or `configkit.Mapping` for explicit null and combined
|
||||
documents. Move profile `base_profile` to `extends` and embedded identities to
|
||||
mapping keys. Review zero/false/null values and any assumptions that replacing
|
||||
a model or backend discards unrelated explicit settings. Rebuild the engine
|
||||
when definitions or referenced message files change. The
|
||||
[consumer guide](../consumers/pkg-promptkit.md#migrate-from-the-previous-configuration-api)
|
||||
provides the migration details and maintained examples.
|
||||
|
||||
## Changes
|
||||
|
||||
- Add ConfigKit v0.5.0 as the configuration boundary and consume released
|
||||
OpenRouter and Rakestrawhome `/v3` catalog modules.
|
||||
- Add keyed backend, profile, and prompt definitions with final-value domain
|
||||
validation, explicit presence/null behavior, and source-relative content
|
||||
capture.
|
||||
- Add `ConfigurationSources`, directory discovery helpers, programmatic
|
||||
profile/backend source builders, and read-only configuration inspection.
|
||||
- Preserve execution workflows while rendering captured templates, applying
|
||||
backend defaults after inheritance, and honoring explicit suppression of
|
||||
provider settings.
|
||||
- Remove superseded loader, catalog preflight, and inheritance machinery;
|
||||
update examples, format guidance, and internal documentation.
|
||||
|
||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.12.0/COPYRIGHT) | [LICENSE](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.12.0/LICENSE) | [ATTRIBUTION](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.12.0/docs/oss/attribution.md)
|
||||
69
docs/releases/v0.13.0.md
Normal file
69
docs/releases/v0.13.0.md
Normal file
@@ -0,0 +1,69 @@
|
||||
# Promptkit v0.13.0
|
||||
|
||||
This release adds prompt requirement discovery and profile/prompt listing,
|
||||
reduces avoidable configuration and execution allocations, and standardizes
|
||||
maintainer publication through ReleaseKit. The minimum supported Go version
|
||||
increases to 1.26.0.
|
||||
|
||||
## Discover Requirements And Available Definitions
|
||||
|
||||
- `Engine.InspectRequirements` reports recognizable input and caller-variable
|
||||
dependencies for a selected prompt and supplied caller context before input
|
||||
artifacts exist. It classifies dependencies as required, conditional, or
|
||||
optional and reports structured limitations when discovery is incomplete.
|
||||
Discovery coverage is not proof that requirements are satisfied or execution
|
||||
will succeed. See the [requirement inspection guide](../consumers/pkg-promptkit.md#inspect-requirements-before-preparation)
|
||||
and [GoDoc](../../requirements.go).
|
||||
- `Engine.ListProfiles` returns effective profile IDs, and `Engine.ListPrompts`
|
||||
returns exact ID/version pairs as `PromptIdentity` values. Lists come from the
|
||||
engine's captured configuration, are lexically ordered and caller-owned, and
|
||||
include consumer definitions and maintained profiles that remain effective.
|
||||
Listing performs no provider or execution work. See the
|
||||
[discovery guide](../consumers/pkg-promptkit.md#discover-available-profiles-and-prompts)
|
||||
and [GoDoc](../../listing.go).
|
||||
|
||||
## Efficiency And Tooling
|
||||
|
||||
- Reduce response-sized copies in basic validation and rendered-message output,
|
||||
and encode valid provider extra parameters only as part of the complete
|
||||
request rather than serializing them twice.
|
||||
- Use bounded initial file-size reservations without limiting accepted content,
|
||||
releasing excess backing storage for small or empty completed artifacts.
|
||||
- Reuse owned preparation values when creating prepared-execution snapshots,
|
||||
while preserving independent inspection details and caller-owned data.
|
||||
- Use lightweight captured profile identities and prompt metadata for inspection
|
||||
instead of copying execution targets and definitions unnecessarily.
|
||||
- Upgrade ConfigKit to v0.6.0 for more efficient loading, resolution, metadata,
|
||||
and inspection processing. Configuration semantics remain unchanged; these
|
||||
improvements primarily benefit construction and explicit operator inspection,
|
||||
not upstream inference latency.
|
||||
- Adopt module-selected ReleaseKit v0.3.0 for maintainer validation and tag
|
||||
publication. The [release procedure](../release.md) owns the current commands
|
||||
and recovery workflow; successful tag publication remains the completion
|
||||
point, without waiting for CI.
|
||||
|
||||
These changes preserve existing execution and ownership contracts. Benefits
|
||||
depend on workload; this release introduces no fixed performance guarantee.
|
||||
|
||||
## Compatibility And Upgrade
|
||||
|
||||
Consumers upgrading from v0.12.0 need **Go 1.26.0 or newer**, rather than the
|
||||
previous 1.25.5 minimum. Update application build environments and any CI images
|
||||
still using Go 1.25 before upgrading:
|
||||
|
||||
```sh
|
||||
go get gitea.maximumdirect.net/eric/promptkit@v0.13.0
|
||||
```
|
||||
|
||||
No application-code or configuration migration is required from v0.12.0 beyond
|
||||
the Go toolchain update. The new discovery methods are optional additions;
|
||||
existing construction, inspection, preparation, and execution workflows remain
|
||||
available. See [Go compatibility](../consumers/pkg-promptkit.md#go-compatibility)
|
||||
for the distinction between the consumer minimum and maintainer toolchain.
|
||||
|
||||
Maintainers should follow the current release runbook rather than the retired
|
||||
`scripts/release-check.sh`. The publication entry point remains
|
||||
`scripts/release.sh`; ReleaseKit now owns its shared validation and publication
|
||||
mechanics.
|
||||
|
||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.13.0/COPYRIGHT) | [LICENSE](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.13.0/LICENSE) | [ATTRIBUTION](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.13.0/docs/oss/attribution.md)
|
||||
48
docs/releases/v0.14.0.md
Normal file
48
docs/releases/v0.14.0.md
Normal file
@@ -0,0 +1,48 @@
|
||||
# Promptkit v0.14.0
|
||||
|
||||
This release adds comparison of captured configuration inspections and updates
|
||||
ConfigKit and the maintainer release tool. The minimum supported Go version
|
||||
remains 1.26.0.
|
||||
|
||||
## Compare Captured Configurations
|
||||
|
||||
`ConfigurationInspection.Diff` compares the effective configuration captured by
|
||||
two engines. Consumers can use it to review a candidate configuration, inspect
|
||||
an overlay's effect, or present configuration changes to operators.
|
||||
|
||||
Comparison delegates to ConfigKit and returns its difference records, including
|
||||
presence information, paired redaction, and protected-change indicators. It
|
||||
does not reload sources or contact providers. Source-history differences alone
|
||||
do not produce value differences.
|
||||
|
||||
An empty diff is not proof of equivalent execution: referenced file contents,
|
||||
resolved credentials, later execution preparation, and request-specific
|
||||
overrides are outside the comparison. Records can contain ordinary application
|
||||
data; callers retain responsibility for their disclosure.
|
||||
|
||||
See the [configuration comparison guide](../consumers/pkg-promptkit.md#inspect-the-captured-configuration)
|
||||
and [GoDoc](../../configuration_inspection.go) for usage and the exact contract.
|
||||
|
||||
## Dependency And Tooling Updates
|
||||
|
||||
- Upgrade ConfigKit from v0.6.0 to v0.7.1, providing the upstream comparison API.
|
||||
v0.7.1 also updates ConfigKit's maintainer tooling without changing its runtime
|
||||
behavior from v0.7.0.
|
||||
- Upgrade the module-selected ReleaseKit tool from v0.3.0 to v0.4.1. Promptkit
|
||||
retains its existing root-module profile, publication entry point, and release
|
||||
gates. See the [release procedure](../release.md) for maintainer commands.
|
||||
|
||||
## Compatibility And Upgrade
|
||||
|
||||
No application-code, configuration, or Go-toolchain migration is required from
|
||||
v0.13.0. Configuration comparison is an optional API addition; existing
|
||||
construction, inspection, preparation, and execution workflows remain available.
|
||||
|
||||
```sh
|
||||
go get gitea.maximumdirect.net/eric/promptkit@v0.14.0
|
||||
```
|
||||
|
||||
See [Go compatibility](../consumers/pkg-promptkit.md#go-compatibility) for the
|
||||
distinction between the consumer minimum and maintainer toolchain.
|
||||
|
||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.14.0/COPYRIGHT) | [LICENSE](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.14.0/LICENSE) | [ATTRIBUTION](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.14.0/docs/oss/attribution.md)
|
||||
@@ -1,505 +0,0 @@
|
||||
# ConfigKit Adoption Assessment
|
||||
|
||||
Status: assessment aligned to released ConfigKit v0.5.0. Upstream capability
|
||||
prerequisites are available; PromptKit integration and migration decisions remain
|
||||
for the feature roadmap and implementation plan. No runtime changes are part of
|
||||
this assessment.
|
||||
|
||||
## Goal And Recommendation
|
||||
|
||||
Use ConfigKit to standardize PromptKit's configuration semantics and simplify
|
||||
its loading machinery, improving ConfigKit where a reusable capability is
|
||||
justified. Preserve substantive consumer capabilities, not historical loading
|
||||
sequences, file syntax, or Go signatures. Document intentional breaking changes
|
||||
and their migration paths.
|
||||
|
||||
Recommend adoption through an eager, immutable definition-loading boundary.
|
||||
ConfigKit should own YAML interpretation, source-tier combination, strict typed
|
||||
decoding, sibling inheritance, presence, and source diagnostics. PromptKit should
|
||||
own definition selection, LLM-specific rules, and execution. Do not retain the lazy loading
|
||||
architecture merely to minimize migration, or introduce a second generic
|
||||
configuration framework inside PromptKit.
|
||||
|
||||
The expected benefits are consistent configuration semantics for applications
|
||||
using both libraries and removal of repeated discovery, partial metadata parsing,
|
||||
lazy error arbitration, and repeated profile-chain resolution. Some of those
|
||||
savings come from the loading redesign rather than ConfigKit itself. ConfigKit
|
||||
adds ongoing value by owning the generic decoding behavior and its tests.
|
||||
|
||||
## Released ConfigKit Capabilities
|
||||
|
||||
ConfigKit v0.5.0 satisfies the two requested prerequisites and also implements
|
||||
sibling inheritance. Adopt the released public contracts rather than waiting for
|
||||
the earlier proposals to be implemented:
|
||||
|
||||
1. **Filesystem sources:** `NewFilesystem(label, fsys).File(name)` borrows an
|
||||
`fs.FS` with reusable namespace identity. Imports and resource contexts retain
|
||||
that namespace. `Result.SourceContext` and `Resolution.SourceContext` expose
|
||||
passive `ResourceContext` values with `FS` and `BaseDir`.
|
||||
2. **Source-backed underlays:** `LoadLayered` accepts a main `Input`
|
||||
and ordered `Underlay` groups. Each group composes strictly; groups overlay
|
||||
recursively from lowest to highest priority. `Input{Source: NoSource()}` omits
|
||||
the main input. `Mapping(label, map[string]any)` snapshots supported built-in
|
||||
scalar/list/map values with explicit presence; it does not ingest arbitrary
|
||||
typed structs. Generic programmatic input is not represented as CLI overrides.
|
||||
3. **Sibling inheritance:** the reserved `extends` key names one immediate sibling
|
||||
mapping. Relationships resolve after precedence assembly and before typed
|
||||
decoding, with inherited origins, history, and resource context preserved.
|
||||
This is built-in behavior, not the opt-in/registered-collection API originally
|
||||
suggested by PromptKit.
|
||||
|
||||
Adopt ConfigKit's recursive combination model rather than preserving PromptKit's
|
||||
whole-profile shadowing or atomic provider-map replacement across source tiers.
|
||||
No hidden replacement or inheritance pass should run after ConfigKit loading.
|
||||
Use released sibling inheritance for profiles, leaving domain validation in
|
||||
PromptKit. This assessment does not authorize code changes in either repository.
|
||||
|
||||
## Evidence And Scope
|
||||
|
||||
PromptKit's runtime baseline remains v0.11.2; this assessment has since received
|
||||
documentation-only revisions. Updated upstream review on 2026-09-30:
|
||||
|
||||
- PromptKit: `12d6d80024e80c552d122a076c22429bf6f2f6c2` (v0.11.2).
|
||||
- ConfigKit: `dc09959cca4e105f49ec316c6d4ba585bd2340bd` (v0.5.0), verified
|
||||
against the published tag. Its clean checkout passed `go test ./...` during
|
||||
the release review. This replaces the earlier v0.4.2 and uncommitted-roadmap
|
||||
comparison as the upstream basis for adoption.
|
||||
|
||||
PromptKit evidence: [consumer workflows](../consumers/pkg-promptkit.md),
|
||||
[file formats](../formats.md), [source implementation](../internal/sources.md),
|
||||
[runner](../internal/runner.md), and the implementation owners identified below.
|
||||
ConfigKit evidence at that release: `docs/config.md`, `docs/releases/v0.5.0.md`,
|
||||
`layered.go`, `source.go`, `resource_context.go`, and ADR 0009 in the sibling
|
||||
checkout. That checkout is review context, not a build dependency or required
|
||||
directory layout. The released configuration contract owns exact upstream behavior;
|
||||
the summaries here identify its consequences for the proposed PromptKit design.
|
||||
|
||||
This is a source-and-contract assessment, not a performance benchmark or full
|
||||
correctness audit. It does not claim measured allocation, latency, or line-count
|
||||
savings. Current code and canonical contracts remain authoritative until an
|
||||
approved implementation changes them.
|
||||
|
||||
## 1. Capability Inventory
|
||||
|
||||
| Preserve the ability to… | Replaceable mechanics and representation |
|
||||
| --- | --- |
|
||||
| Define named, versioned prompts, inspect their declared inputs, and select a version explicitly | Directory scans on each operation, YAML metadata prepasses, filename layout, constructor signatures |
|
||||
| Use inline and file-backed message templates, variables, input helpers, session templates, roles, and cache control | Timing of definition/content reads and private repository interfaces |
|
||||
| Configure named profiles, alias/inherit another profile, and override selected settings | Zero-as-absence fields, per-call chain traversal, raw YAML decoding types |
|
||||
| Supply embedded application defaults, operator definitions, maintained catalogs, and programmatic profiles | Repository-wrapper stacks and exact source-option API |
|
||||
| Register custom/local backends, use built-ins or endpoint-only profiles, and control backend capacity | Configuration DTO shapes and registry assembly order |
|
||||
| Prepare, inspect, execute, or freeze an execution for later single-claim execution | Source reloading during ordinary preparation; exact preparation error ordering |
|
||||
| Override per-run settings, credentials, reasoning, session, and output contract; append rendered messages | Public value signatures if presence or clearer configuration separation requires revision |
|
||||
| Supply inline/file artifacts and inject artifact readers, model clients, and HTTP clients | Assembly APIs; runtime collaborators need not be YAML-decodable |
|
||||
| Validate output, use provider-native structured output, bounded repair, transport retries, cancellation, and observation | Internal wiring, not the substantive execution behaviors |
|
||||
| Receive results, usage, identities, hashes, validation outcomes, and inspectable failures | Configuration error types and when definition failures appear |
|
||||
|
||||
Preserve independent profile and prompt inspection as useful operations. They
|
||||
need not retain the old guarantee that engine construction has never examined
|
||||
other definitions. Preserve custom backend identity and capacity behavior,
|
||||
optional upstream credentials, and the distinction between failed generated
|
||||
content and operational errors. This project does not authorize silently
|
||||
dropping execution features to simplify the configuration layer.
|
||||
|
||||
Add explicit configuration inspection for consumers implementing operator-facing
|
||||
CLI commands or debug output. This is a new capability motivated by this adoption,
|
||||
not an assertion that the current prompt/profile inspection methods already
|
||||
provide configuration provenance.
|
||||
|
||||
## 2. Responsibility Mapping
|
||||
|
||||
| Concern | Recommended owner and disposition |
|
||||
| --- | --- |
|
||||
| YAML grammar, scalar interpretation, one-document enforcement, unknown fields, typed ranges, null legality | Delegate to ConfigKit; remove corresponding PromptKit parser logic and redundant parser tests |
|
||||
| Field presence, source coordinates, and configuration provenance | Delegate to ConfigKit; retain its inspection state with the loaded snapshot for explicit operator queries, separate from execution-ready values |
|
||||
| Operator-facing configuration inspection | Provide a PromptKit consumer API reusing ConfigKit inspection facilities; applications own CLI commands, presentation, and disclosure policy |
|
||||
| File/directory/embedded source selection and definition ID indexing | Retain a small PromptKit collection loader; ConfigKit loads documents, not a prompt catalog |
|
||||
| Configuration filesystem sources, relative imports, and generic layer origins | Delegate to the prerequisite ConfigKit APIs; do not maintain parallel reader-based loading machinery |
|
||||
| Definition discovery and relative message-content references | Retain PromptKit's domain resource adapters, using ConfigKit source context and standard `io/fs` facilities where sufficient |
|
||||
| Profile-source priority and combination | PromptKit selects tiers; ConfigKit owns strict within-tier composition and recursive cross-tier overlay; eliminate repository fallback and whole-definition shadowing |
|
||||
| Named profile inheritance | Delegate sibling graph resolution, combination, and derivation to ConfigKit's `extends`; remove the custom profile resolver |
|
||||
| Backend selection and domain normalization | Retain PromptKit domain rules; use recorded application resolution for configuration transformations not already handled by structural inheritance |
|
||||
| LLM setting bounds, role/session normalization, output rules, reserved provider fields | Retain shared PromptKit domain owners; ConfigKit validation callbacks may invoke them |
|
||||
| Definition defaults | Use ConfigKit defaults where they do not erase inheritance presence; apply effective execution defaults after inheritance |
|
||||
| Programmatic configuration | Prefer labeled mapping input with explicit presence; evaluate whether typed PromptKit convenience constructors justify conversion code, without YAML round trips or zero-value inference |
|
||||
| Runtime clients, observers, artifact readers, request data | Retain normal Go injection and request APIs; do not load them through ConfigKit |
|
||||
| Credentials | Retain optional environment-name and direct request-key semantics; do not replace these with ConfigKit's required typed secret resolution |
|
||||
| Catalog manifest, ownership, reserved names, secret-safety checks | Retain the catalog adapter; reuse the common ConfigKit profile decoder |
|
||||
| JSON Schema resources and generated-output validation | Retain JSON Schema implementation; these are not application configuration documents |
|
||||
| Application imports, CLI overrides, deployment overlays, secrets-directory bootstrap | Leave with consuming applications unless a concrete PromptKit need is accepted |
|
||||
| Generic source/decoding diagnostics | Reuse ConfigKit; add only PromptKit identity/context and domain relationship diagnostics |
|
||||
|
||||
## 3. Recommended Target Loading Model
|
||||
|
||||
### Definition Snapshot
|
||||
|
||||
Create one immutable definition snapshot during engine construction. Discover
|
||||
the explicitly supplied sources, assemble their contributions through ConfigKit,
|
||||
decode the effective configuration, normalize domain values, and resolve relationships
|
||||
before publishing the engine. No provider call or credential availability check
|
||||
belongs in this phase. Publish no partially initialized engine on failure.
|
||||
|
||||
Prefer keyed collections, such as `profiles.<name>`, assembled before one typed
|
||||
loading lifecycle instead of independently decoding each profile and recombining
|
||||
the results. Multiple files can contribute to those collections; this does not
|
||||
require a monolithic file. Reconsider the previous one-typed-document-per-definition
|
||||
recommendation. Settle prompt ID/version representation and catalog format migration
|
||||
in the feature roadmap. Support OS directories, individual files, embedded/custom
|
||||
`fs.FS` sources, and programmatic input through the new source model; source selection
|
||||
and definition discovery remain PromptKit concerns, not a second composition engine.
|
||||
|
||||
ConfigKit expands structural inheritance before typed defaults/decoding. Perform
|
||||
any remaining domain-specific resolution through its recorded lifecycle before
|
||||
semantic validation and result publication. Reuse its presence and provenance;
|
||||
do not add a generic merge/defaulting pass or reconstruct origins after loading.
|
||||
Runtime client assembly and resource interpretation remain outside typed decoding.
|
||||
|
||||
Load and freeze referenced message text with definitions. Validate static prompt
|
||||
structure and template syntax at loading, using the same template function setup
|
||||
as execution; render only when request variables and artifacts exist. Preserve
|
||||
source-relative content resolution and containment without expanding the threat
|
||||
model. A snapshot is a set of captured reads, not an atomic filesystem transaction.
|
||||
|
||||
Resolve each effective content reference using its contributing source's context,
|
||||
not the highest tier's directory. Use ConfigKit's released direct context
|
||||
query rather than a consumer-maintained filesystem registry. Composite origins
|
||||
must not be assigned an arbitrary directory; ambiguous or absent context needs
|
||||
an actionable resource error or explicit consumer-supplied context. ConfigKit
|
||||
does not snapshot external text: PromptKit opens it explicitly and freezes it
|
||||
before engine publication. Caller-owned filesystem lifetimes remain caller-owned;
|
||||
close resource handles acquired by PromptKit without retaining ConfigKit load handles.
|
||||
|
||||
Keep request artifacts and schema compilation at preparation. A request can
|
||||
replace the output contract, so an unused default schema should not need eager
|
||||
compilation. Prepared execution continues to freeze its effective resources.
|
||||
The snapshot covers definitions/message text, not environment values, request
|
||||
files, every schema file, or execution state.
|
||||
|
||||
### Composition And Validation
|
||||
|
||||
Use explicit source categories in the existing useful priority order:
|
||||
programmatic profiles, operator source, application fallback, PromptKit-owned
|
||||
profiles, then external catalogs. Express these as explicit ordered tiers, not
|
||||
source discovery order. Within a tier, mappings compose recursively and overlapping
|
||||
non-mapping contributions conflict, even when equal. Disjoint contributions to
|
||||
the same profile may therefore combine; do not preserve a blanket duplicate-ID
|
||||
rejection. Across tiers, mappings overlay recursively, retaining omitted lower
|
||||
children. Effective prompt identities/versions must remain addressable, with an
|
||||
omitted version ambiguous when multiple versions exist; finalize their keyed
|
||||
representation without reintroducing whole-definition shadowing.
|
||||
|
||||
All sources undergo syntax, acquisition, and composition checks; higher tiers
|
||||
cannot hide those failures. Decode and validate effective values after assembly,
|
||||
not every intermediate tier against the final typed schema. A replaced lower
|
||||
value may consequently have a type that would not fit the final schema. An unused
|
||||
invalid effective definition still fails construction. Catalog ownership,
|
||||
reserved-name, and secret-safety checks remain distinct source-policy requirements,
|
||||
not a reason to impose intermediate typed validation on ordinary inputs.
|
||||
|
||||
Preserve profile templates that become usable through descendants or runtime
|
||||
overrides: eager loading must not require every consumer profile to be an
|
||||
independently runnable endpoint/model pair. Validate supplied fields and graph
|
||||
integrity eagerly (ConfigKit handles missing bases and cycles), and check final execution
|
||||
requirements after runtime selection/overrides. Prompt default-profile references
|
||||
can be checked against the assembled collection; a prompt with no default remains
|
||||
valid for callers that supply a profile explicitly.
|
||||
|
||||
ConfigKit's ordering is typed defaults, source-underlay tiers, strictly composed
|
||||
main input, selected overlay, then CLI overrides. PromptKit's API must map its
|
||||
source priorities explicitly onto supported roles; do not mislabel programmatic
|
||||
input as CLI flags. Underlay-only loading avoids inventing a main document.
|
||||
Imports remain opt-in, and the named-overlay convenience need not be exposed by
|
||||
PromptKit merely because ConfigKit supports it.
|
||||
|
||||
A prompt's `version` is a domain identifier, not automatically a ConfigKit
|
||||
application-schema version. Do not configure `VersionRule` for prompt versions.
|
||||
If a separate configuration schema version is later needed, respect ConfigKit's
|
||||
designated main-input ownership and pre-underlay check timing; do not assume
|
||||
last-wins version selection across catalogs.
|
||||
|
||||
### Source Presence And Released Sibling Inheritance
|
||||
|
||||
Adopt ConfigKit source-layer semantics: absence retains lower input; explicit
|
||||
zero, false, and empty strings replace it; mappings combine recursively; empty
|
||||
mappings retain lower children; sequences replace as units; null explicitly
|
||||
replaces, subject to final schema legality. No reset/delete operator is assumed.
|
||||
Preserve presence through domain resolution, backend/framework
|
||||
default application, request overrides, and outbound request construction.
|
||||
Otherwise accepting `temperature: 0` during decoding would accomplish nothing.
|
||||
|
||||
Use ConfigKit metadata for both file and programmatic input. A final Go value
|
||||
alone does not distinguish absent from explicit zero. Avoid eagerly defaulting
|
||||
profile fields in ways that erase that distinction before relationship resolution.
|
||||
|
||||
Provider `extra_params` maps combine recursively across source tiers too: `{}`
|
||||
does not clear lower keys. Retire the proposed atomic source-map replacement and
|
||||
null-as-unspecified conventions; do not emulate them after loading. An operator
|
||||
overriding only `model` also retains a lower `extends` declaration and other
|
||||
omitted settings. Diagnose and document these consequences explicitly.
|
||||
|
||||
Replace `base_profile` with `extends` on sibling profile mappings. A string names
|
||||
one exact immediate sibling; null cancels the relationship while preserving the
|
||||
child's own settings. Omission retains a lower-tier directive. There is no ancestor
|
||||
scope search or cross-collection reference. Superseded references are history,
|
||||
not active relationships to validate; every effective relationship resolves,
|
||||
including profiles not selected for execution.
|
||||
|
||||
Children recursively overlay their resolved parents with the same value rules:
|
||||
zero/false/empty strings and sequences/null replace, mappings combine, and `{}`
|
||||
does not clear inherited keys. Inherited values count as explicit input for
|
||||
presence queries and absence-based defaults, even when false, zero, or null.
|
||||
Do not repeat inheritance in PromptKit's application-resolution callback. Use
|
||||
`Origin.Inheritance` and retained history for explicit operator inspection.
|
||||
Inherited resource references retain their declaring source's context.
|
||||
|
||||
Enclosing inheritance resolves before nested relationships; nested inherited
|
||||
directives resolve at each destination against its siblings. Defaults and
|
||||
application-derived values do not trigger another inheritance pass. Model this
|
||||
deliberately in PromptKit configuration rather than promising arbitrary graph
|
||||
resolution over runtime values. Partial bases must fit the typed schema and
|
||||
PromptKit's validation; ConfigKit grants them no special validation exemption.
|
||||
|
||||
`extends` is reserved in every input mapping, including provider-parameter maps,
|
||||
across all source kinds. There is no opt-in switch or ordinary-data escape in the
|
||||
released contract. Audit catalog and consumer payload needs before finalizing
|
||||
their format; a literal provider field with that name cannot silently pass through
|
||||
the configuration loader. Do not invent an encoding workaround in this assessment.
|
||||
|
||||
The existing request-level override API remains a separate runtime boundary.
|
||||
The feature roadmap must review its interaction with configured values without
|
||||
silently subjecting runtime request data to configuration inheritance directives.
|
||||
|
||||
### Reloading, Errors, And Application Integration
|
||||
|
||||
Later definition edits take effect through a newly constructed engine. Do not add
|
||||
watchers, mutable registry reloads, or a second lazy compatibility mode. Applications
|
||||
choose how to swap engines and let in-flight work finish. Old prepared handles
|
||||
remain associated with their original engine. Simultaneously active engines have
|
||||
separate backend capacity pools; migration guidance must not imply a global limit
|
||||
across a reload overlap.
|
||||
|
||||
Return configuration failures from construction with file/path/location and
|
||||
useful definition identity where available. Recommend preserving ConfigKit's
|
||||
public `LoadError` through wrapping for `errors.As`, together with a PromptKit
|
||||
configuration category; do not copy its entire diagnostic API into the facade.
|
||||
Domain-only relationship failures remain PromptKit errors with relevant source
|
||||
locations. Start with deterministic fail-fast loading across files and ConfigKit's
|
||||
existing multi-diagnostic behavior within a load, rather than inventing a batch
|
||||
diagnostic subsystem. Do not expose raw credentials or add implicit logging.
|
||||
|
||||
Runtime failures continue to distinguish invalid requests, missing selections,
|
||||
rendering/artifact/schema errors, provider/transport errors, capacity rejection,
|
||||
and content-validation results. Cancellation remains effective at
|
||||
PromptKit-controlled boundaries; ConfigKit's current synchronous `Load` has no
|
||||
context argument, so do not promise interruption inside it or create goroutines
|
||||
to simulate that guarantee.
|
||||
|
||||
Applications using ConfigKit can present the same underlying loading diagnostics
|
||||
for their application settings and PromptKit files. They can also translate
|
||||
application-owned typed settings into PromptKit profiles/backends. Do not require
|
||||
applications to encode Go clients into YAML or adopt ConfigKit merely to call
|
||||
PromptKit's runtime API. The operator-inspection capability below supplies the
|
||||
concrete consumer purpose for exposing effective configuration and its provenance.
|
||||
|
||||
### Explicit Configuration Inspection
|
||||
|
||||
Provide a consumer-facing API for inspecting effective configuration and its
|
||||
provenance so applications can implement targeted CLI commands or debug output.
|
||||
Keep three activities distinct:
|
||||
|
||||
- **Construction:** load and resolve configuration, retain ConfigKit's provenance
|
||||
and inspection state, and publish execution-ready definitions. Resolve any
|
||||
presence and origin information needed for definition construction here.
|
||||
- **Ordinary execution:** use the ready definitions directly. Do not analyze
|
||||
configuration provenance, traverse origin/history records, reconstruct
|
||||
inheritance explanations, or render effective-configuration reports as part of
|
||||
a prompt run. Carry execution-relevant presence explicitly in resolved values
|
||||
rather than querying configuration metadata during each run.
|
||||
- **Explicit inspection:** query effective settings and their contributing
|
||||
sources or explanations only when a consumer invokes the inspection API.
|
||||
Inspection uses the same loaded configuration snapshot as the engine; it does
|
||||
not reread configuration files, open referenced resources, or contact providers.
|
||||
|
||||
Reuse ConfigKit's inspection and redaction facilities rather than maintaining a
|
||||
second provenance system. The API should make targeted queries practical; exact
|
||||
signatures, query scope, and safe result ownership belong in the feature roadmap.
|
||||
Do not imply that configuration inspection predicts all runtime request overrides
|
||||
or reveals the values of execution-time credentials. Existing prompt/profile
|
||||
inspection, prepared-run details, and execution observations have distinct purposes;
|
||||
configuration inspection does not replace them or add provenance to run events.
|
||||
|
||||
Applications decide whether and how to expose inspection through operator tools.
|
||||
PromptKit adds no CLI, implicit debug logging, or automatic explanation output.
|
||||
Redacted configuration can still contain sensitive application content, so callers
|
||||
retain responsibility for access control and disclosure. Retention has a memory
|
||||
cost, but explanation generation and provenance analysis stay off the run path.
|
||||
|
||||
## 4. Simplification Accounting
|
||||
|
||||
| Current implementation | Expected disposition |
|
||||
| --- | --- |
|
||||
| `internal/profile/filesystem_repository.go`: metadata scans, selected strict decode, stream-end checks, malformed-file selection arbitration | Replace with source acquisition, ConfigKit assembly/decoding, and indexed effective values; remove the metadata prepass and per-lookup filesystem path |
|
||||
| `internal/promptdef/filesystem_repository.go`: metadata selection and repeated decoding/content opening | Replace with eager prompt loads and an ID/version index; keep domain normalization and contained content reading |
|
||||
| Profile filesystem, `fs.FS`, eager, memory, and overlay repository variants | Collapse loading into source adapters plus immutable collections; preserve only interfaces that still serve a real boundary |
|
||||
| `internal/profile/resolving_repository.go` | Remove custom graph traversal, cycle detection, and zero-sentinel merging in favor of ConfigKit sibling inheritance; retain only PromptKit domain validation |
|
||||
| `internal/engine` raw repository-stack assembly | Replace with ConfigKit source tiers and one effective typed collection; engine capacity/client assembly remains |
|
||||
| `internal/catalog` profile metadata extraction | Obtain explicit field information from ConfigKit; retain manifests, ownership, namespace, connection-metadata and secret checks |
|
||||
| `internal/filecatalog` | Retain/reduce discovery and containment helpers actually needed by the eager loader; do not claim the entire package disappears |
|
||||
| `internal/domain` source YAML tags and runtime/source coupling | Separate configuration DTOs from execution values where presence requires it; reuse domain validation rather than duplicating it |
|
||||
| Direct `gopkg.in/yaml.v3` use for definition loading | Remove once all such consumers migrate; ConfigKit brings its own private YAML dependency |
|
||||
|
||||
New integration work is real: collection DTOs, ConfigKit calls and error wrapping,
|
||||
source-context access, collection indexing, programmatic mapping ergonomics, and
|
||||
conversion to runtime definitions. Catalog documents may need new keyed shapes;
|
||||
do not hide that migration behind a permanent second loader.
|
||||
Some defensive copies remain necessary at public/injected-client boundaries;
|
||||
ConfigKit's detached load result is not permission to expose shared engine maps.
|
||||
`internal/jsonvalue` still serves programmatic values, request parameters, and
|
||||
prepared schema ownership, so adoption does not eliminate it.
|
||||
|
||||
Maintenance savings depend on actually deleting the old lazy paths and their
|
||||
special-case tests. Move generic YAML/scalar behavior coverage to ConfigKit's
|
||||
ownership; PromptKit retains representative adapter tests plus domain, precedence,
|
||||
snapshot, inheritance, embedded-source, error, and execution integration tests.
|
||||
Do not duplicate ConfigKit's entire parser suite or discard execution regression
|
||||
coverage because construction fixtures need migration.
|
||||
|
||||
Runtime benefit: ordinary runs no longer rediscover definition directories,
|
||||
redecode definitions, reread template text, or traverse inheritance chains.
|
||||
Cost: construction loads even unused definitions and retains their text and
|
||||
resolved values. Retain ConfigKit inspection state needed by the explicit operator
|
||||
API without duplicating its history in runtime definitions or per-run values.
|
||||
These are reasoned tradeoffs, not measured performance
|
||||
claims. Adoption is not justified by the decoder replacement alone.
|
||||
|
||||
### Algorithmic Efficiency
|
||||
|
||||
Efficiency is an explicit design criterion for both the upstream work and this
|
||||
shared-library refactor. Prefer optimal or near-optimal algorithms relative to
|
||||
input and produced-data size, not a blanket O(n) claim with an undefined n.
|
||||
|
||||
- Acquire/parse each accepted source occurrence once, assemble input, and run
|
||||
one typed loading lifecycle rather than decoding intermediate tiers. Reuse
|
||||
captured content instead of reparsing it for metadata, selection, and validation.
|
||||
- Build identity indexes in one pass, with expected constant-time map lookup.
|
||||
Do not scan whole collections for every lookup or precedence decision.
|
||||
- Delegate inheritance expansion to ConfigKit once during loading rather than
|
||||
rebuilding its graph in PromptKit. Assess scaling against input, materialized
|
||||
inherited values, nested destination resolution, and retained history; do not
|
||||
claim unconditional O(V + E) total work from the count of profiles and edges.
|
||||
ConfigKit bounds expansion and retained evidence through its existing budgets;
|
||||
representative catalog sizes must fit them without new per-run processing.
|
||||
- Avoid whole-collection copying per layer, repeated serialization, and repeated
|
||||
prefix/history traversal. Analyze layer processing against total input nodes
|
||||
and produced values/provenance, including intentionally retained history.
|
||||
- Keep configuration discovery, decoding, and provenance analysis out of ordinary
|
||||
execution. Perform explanation work only on explicit inspection calls. Copy only
|
||||
where ownership requires it; do not trade caller isolation for speed.
|
||||
|
||||
These are design targets, not promised complexity for every operation. Sorting
|
||||
for deterministic discovery may require O(F log F) for F files, and processing
|
||||
message text costs work proportional to its bytes. Construction time, retained
|
||||
memory, and frequent-call costs all matter. Prefer straightforward indexes and
|
||||
clear immutable ownership over speculative global caches or intricate optimizers.
|
||||
The implementation plan should include focused benchmarks for consequential
|
||||
repeated-work risks, such as shared-ancestor profiles and many layers, rather than
|
||||
machine-dependent timing gates or benchmarks for every helper.
|
||||
|
||||
## 5. Upstream Readiness And Remaining Opportunities
|
||||
|
||||
Filesystem sources, underlays, programmatic mappings, source contexts, and sibling
|
||||
inheritance are released in v0.5.0. No additional upstream feature is currently a
|
||||
demonstrated prerequisite. Context-aware loading remains a possible future request
|
||||
if concrete consumers need it, not a reason to delay adoption. Existing loading
|
||||
is synchronous; do not simulate cancellation with detached background work.
|
||||
|
||||
Do not propose a public raw AST, permissive identity-only decoder, or dynamic
|
||||
subtree conversion solely to preserve PromptKit's current scanning design.
|
||||
Do not upstream backend/model selection, credential timing for inference,
|
||||
profile catalog reservations, or prompt-template functions.
|
||||
|
||||
ConfigKit and PromptKit both declare LGPL-3.0-or-later. Adoption must still update
|
||||
dependency notices under the OSS policy and inspect the exact selected release
|
||||
and its transitive notices. Their current Go minimums do not require a PromptKit
|
||||
toolchain increase. Use a released module dependency, not a sibling-directory
|
||||
replacement. Select v0.5.0 as the assessed adoption baseline; any newer selection
|
||||
needs its own review. This assessment changes neither dependencies nor licenses.
|
||||
|
||||
## 6. Decisions And Migration
|
||||
|
||||
The agreed direction now incorporates ConfigKit's v0.5.0 contracts. This supersedes
|
||||
the earlier recommendations for independently typed documents, whole-definition
|
||||
shadowing, intermediate typed validation, and atomic maps/null-as-unspecified at
|
||||
the source boundary.
|
||||
|
||||
1. **Loading lifetime and failure scope.** Keep eager construction and frozen
|
||||
definitions/content. Syntax/acquisition/composition failures remain unmaskable;
|
||||
typed and domain validation apply to effective values. A separate reusable
|
||||
public snapshot lifecycle remains possible but is not currently needed.
|
||||
2. **Source organization.** Prefer keyed collections composed across files and
|
||||
tiers before typed loading. Settle exact profile and prompt/version shapes,
|
||||
source-role mapping, and catalog migration in the feature roadmap. Independent
|
||||
typed loads followed by custom combination are not the selected direction.
|
||||
3. **Presence and relationships.** Accept recursive source-tier overlay, retained
|
||||
children for empty mappings, explicit null replacement, and effective metadata.
|
||||
Adopt built-in sibling inheritance, including reserved `extends`, recursive
|
||||
mapping combination, null cancellation, and inherited explicit presence. Do
|
||||
not preserve a parallel PromptKit structural resolver.
|
||||
4. **Diagnostic boundary.** Keep wrapping ConfigKit's public loading errors rather
|
||||
than mirroring them. Retain actionable domain relationship and resource errors
|
||||
without duplicating ConfigKit's generic diagnostic/provenance system.
|
||||
5. **Operator inspection.** Expose effective configuration and provenance through
|
||||
an explicit consumer API backed by the engine's loaded snapshot. Retain the
|
||||
required ConfigKit inspection state at construction; never analyze it during
|
||||
ordinary prompt execution. Finalize targeted query shapes and ownership in the
|
||||
feature roadmap, with presentation remaining application-owned.
|
||||
|
||||
Use released mapping-based programmatic input and direct filesystem resource-context
|
||||
queries. If real consumer examples show
|
||||
mapping construction to be repetitive, consider small typed PromptKit conveniences
|
||||
with explicit presence rather than requiring an upstream typed-ingestion API.
|
||||
|
||||
Remaining PromptKit decisions are the exact collection/version format, catalog
|
||||
migration, programmatic ergonomics, operator-inspection surface, and validation
|
||||
of partial base profiles. Also settle how operators discard unwanted provider
|
||||
settings or old relationships using the released null semantics without a hidden
|
||||
replacement pass. Check the reserved-key impact on arbitrary provider maps. These
|
||||
are integration decisions, not unresolved ConfigKit API proposals.
|
||||
|
||||
Expected consumer migration:
|
||||
|
||||
- Update definitions for ConfigKit's strict scalar rules, rejected YAML
|
||||
anchors/aliases/tags, input limits, and null/type checks. Quote prompt version
|
||||
strings rather than introducing a special numeric-version conversion merely
|
||||
for historical YAML coercion.
|
||||
- Move standalone definitions into the agreed keyed contribution format. Review
|
||||
same-name contributions: disjoint fields within a tier can combine, and higher
|
||||
tiers retain omitted lower fields, including `extends` references. Convert
|
||||
`base_profile` to sibling `extends`; use `extends: null` to cancel a lower-tier
|
||||
relationship, not to erase all settings already supplied to the child.
|
||||
- Review zero/false/empty/null values explicitly. Provider maps merge across
|
||||
tiers, `{}` does not clear them, and null replaces rather than means omission.
|
||||
Use labeled programmatic mappings or approved presence-aware conveniences.
|
||||
Apply those same combination rules to inherited settings. Rename ordinary
|
||||
configuration keys named `extends`; examine provider maps for cases where
|
||||
renaming would change their substantive meaning and resolve those explicitly.
|
||||
- Handle effective-definition errors at construction, fix syntax/composition
|
||||
failures even in overridden inputs, and deliberately rebuild/swap engines
|
||||
after definition changes.
|
||||
- Keep runtime credential lookup optional, avoid baking resolved secrets into
|
||||
definition snapshots, and preserve request-scoped key overrides.
|
||||
- Update maintained catalog data for collection shapes and any incompatible syntax;
|
||||
coordinate catalog releases before the PromptKit dependency update. Preserve
|
||||
maintained profile names and catalog ownership rules, not every old scalar
|
||||
spelling. Do not validate live provider/model availability.
|
||||
- Update error inspection, constructor/source options, and programmatic profile
|
||||
literals as required. Keep representative preparation, execution, inspection,
|
||||
custom backend, fallback, and embedded workflows in the migration guidance.
|
||||
|
||||
The subsequent feature roadmap should carry forward these choices, resolve the
|
||||
remaining PromptKit integration decisions, and specify the target end state.
|
||||
The staged implementation plan can now adopt released ConfigKit support without
|
||||
waiting for upstream feature work, coordinating any catalog format releases before
|
||||
their consumption. Cover deletions/replacements, contract tests, canonical documentation,
|
||||
examples, attribution, and minor-release migration notes. Neither document should
|
||||
become a reason to preserve obsolete compatibility machinery.
|
||||
@@ -1,339 +0,0 @@
|
||||
# ConfigKit Adoption
|
||||
|
||||
Status: agreed feature roadmap. ConfigKit v0.5.0 supplies the required upstream
|
||||
capabilities. The [implementation plan](implementation.md) owns execution steps.
|
||||
The [configuration-boundary ADR](../adr/0001-configuration-boundary.md) records
|
||||
the handoff decision. This document owns the intended end state, not
|
||||
implementation authorization.
|
||||
|
||||
## Purpose
|
||||
|
||||
Use ConfigKit to standardize PromptKit's configuration semantics and simplify
|
||||
its loading machinery, replacing custom logic wherever ConfigKit is the
|
||||
appropriate owner. Preserve substantive consumer capabilities, not historical
|
||||
file syntax, loading order, or Go signatures. Pre-1.0 breaking changes are
|
||||
acceptable and must have migration guidance.
|
||||
|
||||
**This is an intentional breaking change. We are intentionally replacing PromptKit's configuration handling code in favor of adopting ConfigKit's simpler, standardized API. Our goal is to arrive at a new version of PromptKit that is leaner, simpler, and uses a ConfigKit-standard configuration syntax. We MUST NOT give any weight to preserving arbitrary implementation details from PromptKit's current codebase--those should be discarded in favor of the simplest viable implementation that supports PromptKit's substantive LLM-focused consumer capabilities.**
|
||||
|
||||
Applications using both libraries should share configuration interpretation,
|
||||
layering, inheritance, presence, and inspection semantics. PromptKit should
|
||||
concentrate on LLM-specific validation, resource interpretation, and execution.
|
||||
The [adoption assessment](configkit-assessment.md) records the supporting analysis;
|
||||
this roadmap owns the agreed feature end state. Where the earlier assessment
|
||||
differs, this roadmap and the maintainer's direction above take precedence.
|
||||
|
||||
Do not preserve source-option wrappers, repository interfaces, incidental error
|
||||
ordering, or old field shapes solely because they exist today. Reuse existing
|
||||
code only when it remains the simplest appropriate owner of a substantive
|
||||
capability. Migration documentation replaces compatibility machinery, not vice versa.
|
||||
|
||||
## Baseline And Dependency
|
||||
|
||||
The current PromptKit runtime baseline is v0.11.2. Its
|
||||
[source implementation](../internal/sources.md), [format contract](../formats.md),
|
||||
and [consumer guide](../consumers/pkg-promptkit.md) describe existing behavior,
|
||||
not the target described here.
|
||||
|
||||
The reviewed ConfigKit release is v0.5.0 at
|
||||
`dc09959cca4e105f49ec316c6d4ba585bd2340bd`. It provides native `fs.FS` sources,
|
||||
`LoadLayered`, source-backed underlays, labeled programmatic mappings, passive
|
||||
resource contexts, and built-in sibling inheritance. Its released configuration
|
||||
contract and GoDoc own exact upstream semantics. No further upstream feature is
|
||||
currently a demonstrated prerequisite. Review any newer version before selection.
|
||||
|
||||
Adopt a released Go module, not a local `replace` or workspace requirement.
|
||||
Update attribution and transitive license notices under the OSS policy. This
|
||||
feature does not change either project's license or require a Go minimum increase
|
||||
relative to the reviewed versions.
|
||||
|
||||
## Substantive Capabilities To Preserve
|
||||
|
||||
- Named/versioned prompts, declared-input inspection, templates, inline and
|
||||
referenced message content, variables, input helpers, roles, and cache control.
|
||||
- Maintained catalogs, application defaults, operator configuration, programmatic
|
||||
profiles, aliases/inheritance, built-in/custom/local backends, and endpoint-only
|
||||
profiles.
|
||||
- Preparation without generation, ordinary execution, and frozen single-claim
|
||||
prepared execution; independent prompt and profile inspection.
|
||||
- Runtime execution-setting, reasoning, session, credential, and output-contract
|
||||
overrides; appended already-rendered messages.
|
||||
- Inline/file input artifacts and injected artifact, model, and HTTP collaborators.
|
||||
- Backend capacity management, cancellation, transient retries, structural output
|
||||
validation, provider-native structured output, bounded repair, and observations.
|
||||
- Results, effective identities/settings, usage, hashes, and meaningful errors.
|
||||
|
||||
Retain optional environment credentials and direct request credentials. ConfigKit
|
||||
secret resolution must not introduce a new requirement that an upstream API key
|
||||
exist. Domain validation must not probe provider/model availability.
|
||||
|
||||
## Target Configuration Model
|
||||
|
||||
### One Shared Loading Lifecycle
|
||||
|
||||
Construction supplies selected configuration inputs to ConfigKit, which composes,
|
||||
overlays, expands inheritance, and decodes them. PromptKit interprets and validates
|
||||
the final effective configuration at the ConfigKit handoff and produces
|
||||
execution-ready definitions. ConfigKit's effective-value resolution and validation
|
||||
hooks may implement that handoff. Publish no partially initialized engine on failure.
|
||||
|
||||
ConfigKit owns generic parsing, combination, presence, origins, and inheritance.
|
||||
PromptKit owns input selection/discovery, definition indexing, LLM-specific rules,
|
||||
and translation of final values into runtime settings. Input adapters select or
|
||||
encode contributions; they do not inspect or validate raw source contents.
|
||||
Effective presence/null information and resource context supplied by ConfigKit
|
||||
support interpretation at handoff. Historical inspection remains an explicitly
|
||||
invoked operator capability, separate from domain enforcement.
|
||||
Runtime clients and callbacks remain normal Go dependencies, not YAML types.
|
||||
|
||||
Use `backends.<id>`, `profiles.<id>`, and `prompts.<id>.<version>` keyed mappings
|
||||
assembled before typed decoding rather than independently decoding definitions and combining them
|
||||
afterward. Keys own identity; do not duplicate `id`/`version` fields. These paths
|
||||
describe mapping levels, not dot-splitting of literal IDs or version keys.
|
||||
Multiple files can contribute to a collection; a monolithic file is not required.
|
||||
When a prompt ID has multiple versions, selection without a version remains
|
||||
ambiguous. This shape fits sibling inheritance without extra identity reconciliation.
|
||||
|
||||
Supply explicit source groups through a small construction surface compatible
|
||||
with ConfigKit's `Input`/underlay concepts. Offer convenient file/directory/embedded
|
||||
entry points where useful, not a parallel set of historical loading semantics.
|
||||
|
||||
Accept ConfigKit's labeled mapping sources for programmatic configuration. Add
|
||||
only small typed conveniences justified by the minimal local-backend and
|
||||
application-profile workflows. Helpers encode explicit presence and feed the same
|
||||
loading lifecycle; they neither merge configurations nor infer omission from zeros.
|
||||
Backend helpers also produce inputs to this lifecycle. Backend definitions,
|
||||
including catalog connection and capacity settings, cross the same ConfigKit
|
||||
handoff as prompts and profiles. The immutable runtime registry is built from
|
||||
effective backend values. HTTP/model clients and other runtime collaborators
|
||||
remain outside typed configuration.
|
||||
|
||||
Keep the public package a deliberate facade and implementation private. Do not
|
||||
expose internal engine/domain types or introduce a parallel generic loader merely
|
||||
to preserve old source options. ConfigKit's supported public types may be used
|
||||
where intentional integration is simpler than maintaining equivalent wrappers.
|
||||
|
||||
### Sources, Tiers, And Presence
|
||||
|
||||
Support ordinary files/directories, embedded or caller-supplied `fs.FS`, and
|
||||
programmatic definitions. Explicitly selected directories may be discovered by
|
||||
PromptKit; ConfigKit handles their configuration contributions. Do not infer
|
||||
priority from traversal order.
|
||||
|
||||
Retain the useful priority categories, lowest to highest: external
|
||||
catalogs, PromptKit-owned profiles, application fallback, operator configuration,
|
||||
and programmatic configuration. Express intentional priority as ordered tiers.
|
||||
Within each tier, independent contributions compose strictly; overlapping
|
||||
non-mapping values conflict, even when equal. Disjoint fields under the same
|
||||
identity can combine. Across tiers, mappings recursively overlay.
|
||||
These rules also govern backend configuration; consumers can add backend IDs or
|
||||
overlay existing definitions through their selected tiers. Registry construction
|
||||
validates the resulting values, not the identity of their contributors.
|
||||
|
||||
Use ConfigKit's source roles honestly: underlay groups are ordered defaults,
|
||||
main input overlays underlays, and optional named overlay/CLI facilities retain
|
||||
their upstream meaning. Underlay-only loading is supported; do not manufacture
|
||||
a main document or disguise programmatic values as CLI flags. Exposing every
|
||||
ConfigKit convenience through PromptKit is not a requirement.
|
||||
|
||||
Adopt the shared value rules: omission retains lower input; explicit zero, false,
|
||||
empty string, sequence, and null replace; mappings combine recursively; empty
|
||||
mappings do not clear lower children. Null remains explicit and subject to the
|
||||
final schema. Provider-parameter maps obey these rules too. No hidden whole-profile
|
||||
replacement, arbitrary merge callback, or new reset/delete operator belongs here.
|
||||
|
||||
Preserve presence into execution-ready settings so an explicit zero does not
|
||||
disappear during backend/default resolution or outbound request construction.
|
||||
Use ConfigKit metadata at construction, not as a per-run lookup mechanism.
|
||||
|
||||
### Sibling Inheritance
|
||||
|
||||
Use ConfigKit's reserved `extends` rather than PromptKit's `base_profile` and
|
||||
custom resolver. A string identifies an immediate sibling mapping; null cancels
|
||||
the relationship without deleting the child's own settings. Layering completes
|
||||
before inheritance. Only effective relationships resolve, but all of them do,
|
||||
including entries never selected for execution.
|
||||
|
||||
Children recursively overlay resolved parents. Inherited values count as explicit
|
||||
input. Origins and resource context remain attached to their declaring source.
|
||||
Nested relationships follow ConfigKit's destination-local rules; defaults and
|
||||
application-derived values do not trigger another inheritance pass.
|
||||
|
||||
The keyword is reserved throughout configuration, including dynamic provider
|
||||
maps; it is not opt-in. Keep runtime request provider-parameter maps ordinary
|
||||
JSON-compatible data outside configuration parsing. Consumers needing a literal
|
||||
provider field named `extends` supply it at that runtime boundary. Do not rename
|
||||
the outbound key or introduce a second opaque configuration format to escape it.
|
||||
Document and test this distinction, including any affected maintained payloads.
|
||||
|
||||
### Validation And Errors
|
||||
|
||||
ConfigKit owns acquisition, syntax, composition, inheritance, and typed-decoding
|
||||
failures. PromptKit preserves those errors and validates final effective domain
|
||||
values. An unused invalid effective definition fails construction; a lower-layer
|
||||
value legitimately replaced by ConfigKit is not a separate validation target.
|
||||
This boundary applies equally to application and maintained catalog configuration.
|
||||
PromptKit does not perform catalog preflight, raw-source or intermediate-state
|
||||
validation, or history-based enforcement, including equivalent test-only checks.
|
||||
Catalog maintainers own publishing checks in their respective repositories.
|
||||
|
||||
Domain validators remain shared with runtime request validation where appropriate.
|
||||
Allow schema-valid partial profiles without an `abstract` flag. Validate supplied
|
||||
domain values at loading; final runnable-target requirements are checked after
|
||||
profile selection and runtime overrides at execution preparation.
|
||||
Do not require credentials, artifacts, or provider connectivity during construction.
|
||||
|
||||
Preserve absence versus explicit null in resolved settings. Null on an optional
|
||||
provider setting deliberately suppresses it; it must not restore the same setting
|
||||
from backend defaults. A null provider map suppresses the configured lower map,
|
||||
while null entries inside a provider map remain literal JSON null values. This
|
||||
supports reusable bases and deliberate removal of stale settings without changing
|
||||
ConfigKit's merge rules. Framework-required settings such as timeouts have explicit
|
||||
field-specific null semantics rather than blanket nil defaulting.
|
||||
|
||||
Preserve ConfigKit public loading errors through wrapping for `errors.As`, with a
|
||||
PromptKit configuration category. Use source-aware domain/resource failures where
|
||||
ConfigKit is not the owner. Do not mirror its diagnostic framework or promise
|
||||
aggregation beyond what the underlying operation provides. Runtime operational
|
||||
errors remain distinct from generated-content validation failures.
|
||||
|
||||
Prompt versions are domain identifiers, not ConfigKit `VersionRule` declarations.
|
||||
Do not introduce an application schema-version policy merely to adopt ConfigKit.
|
||||
|
||||
## Snapshot, Resource, And Execution Boundaries
|
||||
|
||||
The engine captures effective definitions and referenced message text during
|
||||
construction. Validate static template syntax then; render with request data
|
||||
later. Resolve content references from the effective field's contributing source
|
||||
context, not the last layer's directory. Ambiguous/contextless resource references
|
||||
need an explicit source context or an actionable error, not an invented directory.
|
||||
|
||||
ConfigKit returns passive resource contexts, not snapshots of referenced text.
|
||||
PromptKit explicitly opens and closes its resource handles. Caller filesystem
|
||||
ownership remains unchanged. Preserve appropriate source containment without
|
||||
claiming that every `fs.FS` is a security sandbox. Captured reads are not an atomic
|
||||
transaction over a concurrently changing filesystem.
|
||||
|
||||
Definition changes require new engine construction. No watcher, background
|
||||
refresh, or lazy compatibility mode is added. Applications own engine replacement;
|
||||
prepared handles remain bound to their original engine and overlapping engines
|
||||
retain separate capacity pools.
|
||||
|
||||
Request artifacts and schema preparation remain invocation work. A request can
|
||||
replace the output contract, so construction need not compile unused schemas.
|
||||
Prepared execution continues to freeze its effective resources. Environment
|
||||
credential lookup remains at the established execution/preparation boundary,
|
||||
not frozen into the configuration snapshot.
|
||||
|
||||
Ordinary execution uses indexed ready definitions and resolved presence values.
|
||||
It performs no configuration discovery, parsing, inheritance traversal, or
|
||||
provenance analysis. Context cancellation remains cooperative at controlled
|
||||
boundaries; do not add detached work to pretend synchronous ConfigKit loading is
|
||||
instantly interruptible.
|
||||
|
||||
## Explicit Operator Inspection
|
||||
|
||||
Add an API consumers can use to implement targeted CLI commands or debug output
|
||||
for effective configuration, contributing sources, and explanations. Retain
|
||||
ConfigKit inspection state with the engine's configuration snapshot and reuse
|
||||
its redaction and provenance facilities.
|
||||
|
||||
Inspection is explicitly invoked and uses that snapshot without reloading files,
|
||||
opening resources, or contacting providers. It does not predict every runtime
|
||||
override, reveal execution-time credential values, or replace prepared-run details
|
||||
and execution observations. No provenance traversal, explanation generation, or
|
||||
configuration-report rendering occurs during ordinary prompt runs.
|
||||
|
||||
Expose a read-only inspection view backed by retained ConfigKit state, with
|
||||
targeted path queries, source/history information, and explicit redacted effective
|
||||
and explanation output. Reuse suitable public ConfigKit inspection values rather
|
||||
than mirroring them. Do not expose mutable runtime definitions, resource capabilities
|
||||
merely for diagnostics, or request credentials. The scope is generic targeted
|
||||
inspection, not just narrow profile/prompt origin summaries.
|
||||
|
||||
PromptKit ships no CLI and emits no automatic debug reports. Applications own
|
||||
presentation, access control, and disclosure; redaction does not make all prompt
|
||||
or configuration content safe to publish.
|
||||
|
||||
## Required Refactoring And Coordination
|
||||
|
||||
- Replace profile/prompt metadata prepasses, selected-file decoding, and per-call
|
||||
scans with ConfigKit collection loading and immutable indexed definitions.
|
||||
- Remove the custom profile inheritance resolver and error-driven repository
|
||||
overlay stack. Do not retain an old-format compatibility engine alongside the
|
||||
new semantics.
|
||||
- Route catalog and programmatic backend definitions through ConfigKit; retire
|
||||
the separate backend-manifest decoder and pre-handoff registration validation.
|
||||
Retain backend endpoint, capacity, and provider-parameter domain validation on
|
||||
the final effective values.
|
||||
- Retain only needed discovery and message-resource adapters; remove direct YAML
|
||||
dependency usage once no remaining owner needs it. Preserve JSON Schema,
|
||||
runtime JSON-value ownership, domain validation, and execution collaborators.
|
||||
- Adapt maintained catalogs and PromptKit-owned definitions to the agreed keyed
|
||||
format and released inheritance semantics, including keyed backend assets.
|
||||
Preserve maintained name commitments and the intended shipped backend settings.
|
||||
Catalog source-policy enforcement belongs to catalog maintainers. Coordinate
|
||||
compatible catalog releases before consuming the changed asset format.
|
||||
Publish new keyed assets directly in the existing data-only catalog projects,
|
||||
recording the format transition rather than maintaining duplicate legacy assets
|
||||
or a permanent conversion adapter. Catalog maintainers select release/version
|
||||
handling under their own policies; old consumers must not be assumed capable
|
||||
of decoding the new shape. Pin the compatible released versions in PromptKit.
|
||||
This document requests no immediate changes in those repositories.
|
||||
- Keep configuration conversion, normalization, and inspection connected to
|
||||
ConfigKit's lifecycle instead of reconstructing effective values/history after
|
||||
loading. Translate once into execution-owned values with necessary copying.
|
||||
- Update canonical formats, consumer guidance, GoDoc, internal inventory, examples,
|
||||
architecture boundaries, dependencies, and OSS notices when implementation lands.
|
||||
|
||||
## Efficiency And Validation Expectations
|
||||
|
||||
Target linear or near-linear work relative to actual input, output, and retained
|
||||
history, accounting separately for sorting, resource reads, and domain work.
|
||||
Acquire/parse each accepted source occurrence once, avoid decoding intermediate
|
||||
tiers, and index definition selection. Do not duplicate inheritance expansion or
|
||||
copy complete accumulated collections for every added layer.
|
||||
|
||||
Materialized inherited data and provenance can exceed source size. Exercise real
|
||||
catalog shapes within ConfigKit's budgets; do not claim unconditional O(n) runtime
|
||||
or memory. Prefer clear ownership and straightforward indexes over global caches
|
||||
or elaborate optimization. Operator inspection may perform work proportional to
|
||||
the requested report; that work remains off the execution path.
|
||||
|
||||
Tests should use real ConfigKit with offline sources and synthetic provider
|
||||
collaborators. PromptKit owns its adapter, domain, lifecycle, and execution tests,
|
||||
not a duplicate ConfigKit parser/graph suite. Retire tests whose only purpose was
|
||||
the removed loading machinery or source-policy enforcement. Exercise catalog
|
||||
integration through the final assembled configuration and resulting behavior.
|
||||
Preserve meaningful execution regressions.
|
||||
|
||||
Representative acceptance evidence must cover:
|
||||
|
||||
- Minimal local-backend construction and execution without elaborate configuration.
|
||||
- Embedded defaults plus operator and programmatic contributions, including
|
||||
accurate inherited relative-resource origins.
|
||||
- Profile inheritance, changing model/backend, canceling old relationships, and
|
||||
discarding unwanted settings with documented semantics.
|
||||
- Explicit zero/false/null behavior reaching execution, construction failures,
|
||||
independent snapshots, and unchanged substantive preparation/execution features.
|
||||
- Targeted inspection with useful redacted origins and no provenance work during
|
||||
ordinary runs.
|
||||
|
||||
Use focused scaling/allocation benchmarks for large collections, shared bases,
|
||||
and many tiers where they expose repeated-work risks. Do not add timing-based
|
||||
release gates or provider-network tests.
|
||||
|
||||
## Migration And Completion
|
||||
|
||||
Ship as an intentionally incompatible minor release with concrete migration
|
||||
guidance. Cover keyed files, `base_profile` to `extends`, reserved-key behavior,
|
||||
recursive maps, explicit null versus omission, changed error timing, programmatic
|
||||
construction, backend layering in place of registration-only restrictions, and
|
||||
explicit engine replacement. Explain that overriding a model
|
||||
does not discard lower endpoint, reasoning, provider, or inheritance settings.
|
||||
|
||||
The end state is one shared configuration model with the old loading/resolution
|
||||
machinery removed, useful operator inspection, and all substantive execution
|
||||
workflows still available. The six previously open design choices have been
|
||||
approved and incorporated into the owning sections above. This is not a
|
||||
compatibility-preserving reinterpretation of the existing configuration engine.
|
||||
@@ -42,20 +42,21 @@ checkpoint equality or when a broader semantic identity design is selected.
|
||||
### Eager Source Validation
|
||||
|
||||
**Reason for deferral:** Exact prompt and profile inspection may already
|
||||
provide a sufficiently small validation surface. Experience from downstream
|
||||
adoption should establish whether an engine-wide operation would add enough
|
||||
value to justify its broader contract.
|
||||
provide a sufficiently small validation surface alongside construction-time
|
||||
definition validation. Experience from downstream adoption should establish
|
||||
whether a separate operation would add enough value to justify its broader
|
||||
contract.
|
||||
|
||||
Promptkit could provide an explicit offline operation that discovers and
|
||||
structurally validates configured prompt, profile, and schema sources without
|
||||
model generation. The normal `NewEngine` path would remain lazy.
|
||||
Promptkit could provide an explicit offline operation that also checks schema
|
||||
resources without model generation. `NewEngine` already validates effective
|
||||
definitions and captures prompt content; schema resources remain
|
||||
request-dependent.
|
||||
|
||||
An eager operation would need coherent handling for duplicate prompt IDs and
|
||||
versions, strict YAML decoding, referenced content files, profile/backend
|
||||
membership, schema syntax and transitive references, context cancellation,
|
||||
and source-specific public errors. Credential declarations must remain
|
||||
separate from credential values; checking current environment availability,
|
||||
if supported at all, should be an explicit option and must not expose secrets.
|
||||
Such an operation would need coherent handling for schema syntax and transitive
|
||||
references, context cancellation, and source-specific public errors. Credential
|
||||
declarations must remain separate from credential values; checking current
|
||||
environment availability, if supported at all, should be an explicit option
|
||||
and must not expose secrets.
|
||||
|
||||
Reconsider this idea after downstream use of `InspectPrompt`,
|
||||
`InspectProfile`, and fixture-based preparation demonstrates a concrete gap.
|
||||
|
||||
@@ -1,642 +0,0 @@
|
||||
# ConfigKit Adoption Implementation Plan
|
||||
|
||||
Status: ready for staged implementation under the agreed configuration boundary.
|
||||
This is a planning document; writing it does not implement the feature or
|
||||
authorize publishing dependencies.
|
||||
|
||||
## Authority And Working Method
|
||||
|
||||
The [ConfigKit feature roadmap](configkit.md) owns the intended end state and
|
||||
approved policy. Its intentional-breaking-change direction is authoritative:
|
||||
preserve substantive prompting capabilities, not the old loader, signatures,
|
||||
source wrappers, incidental error ordering, or configuration syntax.
|
||||
The [configuration-boundary ADR](../adr/0001-configuration-boundary.md) records
|
||||
the ownership decision: PromptKit interprets and validates the final effective
|
||||
configuration supplied by ConfigKit. Presence/null information and effective
|
||||
resource context are part of that handoff. Source history serves explicit
|
||||
operator inspection, not validation or catalog enforcement. This applies to
|
||||
tests as well as production code.
|
||||
|
||||
Implement the stages below in order, one stage per coding-agent prompt. Each
|
||||
stage includes its tests and affected current-state documentation. Read
|
||||
[development](../development.md) and the applicable policies before starting;
|
||||
in particular, follow [agents](../policy/agents.md),
|
||||
[architecture](../policy/architecture.md), [testing](../policy/testing.md),
|
||||
[documentation](../policy/documentation.md), and [OSS](../policy/oss.md).
|
||||
Catalog work also follows each catalog repository's own instructions and release
|
||||
procedure. Do not turn this plan's internal organization into new public promises.
|
||||
|
||||
Begin each stage by inspecting the worktree and earlier stages. Commit completed
|
||||
units under the agent policy. Keep stages buildable; temporary private wiring
|
||||
needed to connect successive stages must be removed by Stage 11. Do not ship a
|
||||
legacy mode, dual loader, or old-format conversion layer. Do not add prototype
|
||||
applications or a generic configuration framework of our own.
|
||||
|
||||
Planning baselines:
|
||||
|
||||
- PromptKit runtime v0.11.2, commit
|
||||
`12d6d80024e80c552d122a076c22429bf6f2f6c2`; approved roadmap at `521291d`.
|
||||
- ConfigKit v0.5.0, commit
|
||||
`dc09959cca4e105f49ec316c6d4ba585bd2340bd`, module
|
||||
`gitea.maximumdirect.net/eric/ConfigKit`.
|
||||
- Both catalog dependencies currently select `/v2` at `v2.20260926.0`.
|
||||
|
||||
Use ConfigKit v0.5.0 for this work. Do not silently upgrade to a different
|
||||
contract. The catalog asset migration needs coordinated releases, not a committed
|
||||
workspace or local replacement. Changing the backend asset contract is broader
|
||||
than the catalog profile-data compatibility exception: migrate the catalogs to
|
||||
`/v3` with `v3.YYYYMMDD.N` tags, preserving their date/revision convention and
|
||||
updating their own release documentation accordingly. Select exact dates and
|
||||
unused revisions at publication; record versions and full commits consumed.
|
||||
Publishing any repository requires separate authority.
|
||||
|
||||
## Implementation Decisions
|
||||
|
||||
### Construction Surface
|
||||
|
||||
Keep `NewEngine(Config, ...Option)` and the execution/injection surface where it
|
||||
still serves its substantive purpose. Replace profile/prompt source options with
|
||||
these fields and helpers; this is an intentional API break:
|
||||
|
||||
```go
|
||||
type ConfigurationSources struct {
|
||||
Underlays []configkit.Underlay
|
||||
Main *configkit.Input
|
||||
}
|
||||
|
||||
// Add Sources; remove PromptDir and ProfileDir.
|
||||
// Retain SchemaDir, Timeout, HTTPClient, and TransientRetries.
|
||||
type Config struct { /* existing non-definition settings; Sources ConfigurationSources */ }
|
||||
|
||||
func DirectoryInputs(directory string) ([]configkit.Input, error)
|
||||
func FilesystemInputs(label string, filesystem fs.FS, root string) ([]configkit.Input, error)
|
||||
```
|
||||
|
||||
The pointer distinguishes omitted main input without inspecting ConfigKit's
|
||||
opaque Source. Nil main becomes `configkit.NoSource()`; a supplied invalid Input
|
||||
is not silently ignored. Prepend two private underlays: external catalogs, then
|
||||
PromptKit-owned definitions. Append caller underlays in their supplied order;
|
||||
main is last and keeps its ConfigKit meaning. Imports follow each Input's flag.
|
||||
Do not expose named overlays, CLI overrides, or VersionRule through another
|
||||
PromptKit abstraction in this feature.
|
||||
|
||||
Show the usual application categories as caller underlays ordered fallback,
|
||||
operator, programmatic, with main omitted. For a single highest-priority document,
|
||||
main is also useful. These are explicit selections, not directory-name heuristics
|
||||
or a second automatic precedence algorithm. Each directory helper returns inputs
|
||||
for one strict-composition group, recursively finding `.yaml` and `.yml` files;
|
||||
sorted discovery makes diagnostics reproducible, not one file higher priority.
|
||||
An explicitly selected missing/unreadable root fails; an empty directory supplies
|
||||
no contributions. Helpers set imports false; callers may enable them explicitly.
|
||||
Use one ConfigKit filesystem namespace per helper invocation and root resource
|
||||
contexts at the supplied directory. Single files use ConfigKit directly.
|
||||
|
||||
Remove `WithPromptFS`, `WithPromptFile`, `WithProfileFS`, `WithProfileFile`,
|
||||
`WithFallbackProfileFS`, `WithProfiles`, and `WithBackend` at public cutover.
|
||||
Retain schema source options and runtime collaborator injection. Backend
|
||||
configuration moves into ConfigKit inputs as specified below. Do not require any
|
||||
prompts to exist merely to construct an engine for profile inspection.
|
||||
|
||||
Programmatic configuration uses labeled `configkit.Mapping` inputs. Replace the
|
||||
old zero-sentinel `Profile`/`OpenAICompatibleProfileConfig` pair with one small
|
||||
`ProfileConfig` convenience and a source builder:
|
||||
|
||||
```go
|
||||
type ProfileConfig struct {
|
||||
Extends, Backend, Endpoint, Model *string
|
||||
Temperature, TopP *float64
|
||||
MaxTokens, TimeoutSeconds *int
|
||||
ServiceTier, ReasoningEffort, APIKeyEnv *string
|
||||
APIKeyRequired *bool
|
||||
ExtraParams map[string]any
|
||||
}
|
||||
func ProfileSource(label string, profiles map[string]ProfileConfig) configkit.Source
|
||||
```
|
||||
|
||||
Nil convenience fields mean omitted; nonnil pointers preserve zero/false/empty.
|
||||
Nil ExtraParams omits the map; a nonnil empty map supplies an empty mapping, not a
|
||||
clear operation. Use raw Mapping for explicit null, prompt definitions, or richer
|
||||
configuration. Do not introduce a public generic optional-value type just for
|
||||
this helper. Convert fields explicitly to ConfigKit-supported built-in values,
|
||||
then call Mapping for snapshot ownership and load-time errors. Do not marshal
|
||||
through YAML or infer absence from a value. Helpers encode values; domain
|
||||
validation occurs on the final configuration at handoff.
|
||||
|
||||
Replace the registration-only Backend value with a presence-aware convenience:
|
||||
|
||||
```go
|
||||
type BackendConfig struct {
|
||||
Endpoint, APIKeyEnv *string
|
||||
ConcurrencyLimit, QueueCapacity *int
|
||||
ExtraParams map[string]any
|
||||
}
|
||||
func BackendSource(label string, backends map[string]BackendConfig) configkit.Source
|
||||
func LocalBackend(endpoint string, concurrencyLimit int) BackendConfig
|
||||
```
|
||||
|
||||
Use the same omission/pointer/map conventions as ProfileSource. LocalBackend
|
||||
supplies endpoint and concurrency pointers and leaves the remaining fields
|
||||
omitted; callers choose the ID as the map key. Neither helper validates settings
|
||||
before ConfigKit processes them. For example, place BackendSource with
|
||||
`map[string]BackendConfig{"local": LocalBackend(endpoint, 4)}` in an application
|
||||
underlay. Raw Mapping supports explicit null and combined backend/profile/prompt
|
||||
documents. Remove the old Backend registration value and its reservation rules;
|
||||
retain useful built-in ID constants. All backend contributions follow the same
|
||||
strict-composition and overlay rules as other configuration.
|
||||
|
||||
### Typed Configuration And Execution Values
|
||||
|
||||
Add `internal/configuration` as the cohesive ConfigKit adapter. Its private
|
||||
document DTO has `backends: map[string]backendConfig`,
|
||||
`profiles: map[string]profileConfig`, and
|
||||
`prompts: map[string]map[string]promptConfig`. Collection entries are mappings,
|
||||
not nullable definition pointers. No duplicate identity fields; ConfigKit consumes
|
||||
`extends`, so no DTO field or second resolver handles it. Use `configkit.Keys`,
|
||||
`Key`, and `Index` for semantic paths, never split IDs on dots.
|
||||
|
||||
Profile DTO fields match ProfileConfig's YAML spellings (`backend`,
|
||||
`reasoning_effort`, etc.). Nullable pointers plus ConfigKit presence metadata
|
||||
distinguish absent, value, and explicit null. Store that distinction in small
|
||||
private execution-owned fields or presence bits, not retained metadata queries.
|
||||
Runtime/domain packages must not import ConfigKit. Do not duplicate its tree,
|
||||
merge engine, inheritance graph, diagnostics, or presence framework.
|
||||
|
||||
Backend DTO fields match BackendConfig using `endpoint`, `api_key_env`,
|
||||
`concurrency_limit`, `queue_capacity`, and `extra_params`. Validate the final
|
||||
endpoint as required, absolute HTTP(S) with the existing URL rules. An absent or
|
||||
null API-key environment name means no lookup; a supplied name follows existing
|
||||
normalization and naming rules. Absent/null concurrency means zero (unlimited);
|
||||
supplied values must be nonnegative. Absent/null queue capacity selects the
|
||||
existing default of 1024 for positive concurrency and no queue for unlimited
|
||||
concurrency. Explicit queue capacity, including zero, requires positive
|
||||
concurrency and must satisfy existing nonnegative/overflow checks. Null provider
|
||||
maps mean no configured extra parameters; entries may contain JSON null.
|
||||
Required endpoint failures, unknown fields, and domain errors apply to final
|
||||
definitions, including unused ones. Registry construction consumes these final
|
||||
normalized values; it does not replay registration or source-reservation checks.
|
||||
|
||||
Use ConfigKit's Resolve/Validate callbacks only on its effective post-layering,
|
||||
post-inheritance typed values. These public lifecycle hooks implement the domain
|
||||
handoff and record normalization; they are not hooks for inspecting earlier
|
||||
contributions. Final presence lookup and resource-context lookup are sufficient
|
||||
for interpretation. History/Explain/HistoricalPaths are operator inspection
|
||||
facilities, not inputs to construction-time policy checks.
|
||||
|
||||
Use the existing prompt fields under each keyed version, without `id`/`version`.
|
||||
Use pointers for `content`, `content_file`, and optional clearable strings;
|
||||
preserve existing input, cache-control, output-contract, and role rules.
|
||||
`messages` must be nonempty; each message has exactly one nonnull content source.
|
||||
An explicitly supplied empty inline string is a valid template. Null content
|
||||
removes that selection; null content_file removes its reference. Output remains a
|
||||
required mapping with valid format and validation mode. Optional description,
|
||||
session template, default profile, cache control, inputs, and schema path can be
|
||||
cleared where their domain permits absence. Null required role/format/mode or a
|
||||
null whole definition is invalid. Scalar `required` and `repair_attempts` retain
|
||||
their false/zero defaults and are not nullable. Null inputs means no declarations.
|
||||
Unknown fields are ConfigKit errors, including old `id`, `version`, and
|
||||
`base_profile`. Prompt versions must be string mapping keys.
|
||||
|
||||
Literal definition keys remain exact; require nonblank identity but do not
|
||||
normalize/rekey maps after ConfigKit has resolved relationships. Update selection
|
||||
to use exact nonblank IDs/versions rather than silently trimming them into a
|
||||
different identity. Scalar normalization such as endpoint trimming is recorded
|
||||
through ConfigKit's resolution callbacks when
|
||||
mutating its typed configuration. Message roles and cache control are normalized
|
||||
on execution-owned copies instead: ConfigKit v0.5.0 treats sequence mutations as
|
||||
whole-value replacements, losing unchanged content_file origins. Inspection
|
||||
retains the effective input message metadata and its source context. Field
|
||||
references to definition IDs remain exact as well.
|
||||
|
||||
Profile field rules after configuration overlay/inheritance:
|
||||
|
||||
| Fields | Absent | Explicit value | Explicit null |
|
||||
| --- | --- | --- | --- |
|
||||
| `backend` | No selected backend | Exact registered ID; blank means none | No backend |
|
||||
| `endpoint` | Selected backend endpoint, if any | Normalize/validate nonblank URL; empty clears | Clear, do not recover backend endpoint |
|
||||
| `model` | Unspecified | Nonblank supplied model; empty clears | Unspecified |
|
||||
| `temperature`, `top_p`, `max_tokens` | Unspecified | Validate existing bounds; zero is explicit and sent | Suppress outbound control |
|
||||
| `reasoning_effort`, `service_tier` | Unspecified | Existing normalization; blank suppresses | Suppress outbound control |
|
||||
| `timeout_seconds` | Framework 600-second baseline | Nonnegative; zero disables generation deadline | Reset to framework baseline, not lower profile timeout |
|
||||
| `api_key_env` | Selected backend's optional lookup name | Trimmed value; blank disables lookup | Disable lookup, do not recover backend name |
|
||||
| `api_key_required` | False | Explicit boolean, including false overriding true | False |
|
||||
| `extra_params` | Backend defaults | ConfigKit-composed profile map, recursively overlaid on backend map | Suppress entire configured provider map |
|
||||
|
||||
Profile API-key environment names follow the existing lookup contract, including
|
||||
host-supported non-shell-identifier names. Do not extend the backend's naming
|
||||
regular expression to profiles. Trimming, explicit clearing, optional lookup,
|
||||
and direct request-key precedence remain unchanged.
|
||||
|
||||
Backend registry defaults are applied once during snapshot construction, after
|
||||
effective profile inheritance. Keep them out of the inheritance input: a parent's
|
||||
derived endpoint/key/defaults must not follow a child that changes backend.
|
||||
Provider map null entries are literal JSON null; empty mappings retain children.
|
||||
For backend-to-profile provider-map overlay, use a narrow JSON-value merge helper
|
||||
with the same recursive-map/value-replacement semantics; it is not a configuration
|
||||
loader. Both backend and profile configuration maps have already passed through
|
||||
ConfigKit's reserved-key semantics; only request-level maps bypass that lifecycle.
|
||||
|
||||
Record any defaults/normalizations written into the typed configuration via
|
||||
ConfigKit Resolve, with dependencies on relevant configuration paths. Do not
|
||||
project runtime collaborators or credentials into the configuration tree just to
|
||||
report them. Inspection includes effective backend configuration. It describes
|
||||
resolved configuration, while InspectProfile and preparation describe
|
||||
backend-completed execution settings. Make this boundary
|
||||
explicit rather than inventing synthetic provenance for registry defaults.
|
||||
|
||||
Retain APIKeyRequired's direct-key workflow: true suppresses a backend-derived
|
||||
environment source. An explicitly configured profile environment source, or a
|
||||
request environment override, remains usable; false restores ordinary optional
|
||||
credential semantics. No credential values belong in the DTO, inspection, or
|
||||
construction-time environment resolution. Raw profile `api_key` remains invalid.
|
||||
|
||||
Validate all effective supplied values and referenced backend IDs at construction.
|
||||
Do not require a complete model/endpoint there or reject unused partial profiles.
|
||||
Do not require a prompt default_profile to be runnable before a request can
|
||||
override it. Final target completeness is checked after runtime overrides.
|
||||
InspectProfile can report a partial target without credential or provider access.
|
||||
Keep request override contracts otherwise unchanged, including complete nonempty
|
||||
runtime ExtraParams replacement and reasoning clearing. A runtime literal
|
||||
`extra_params["extends"]` remains ordinary provider data. Do not expand this work
|
||||
into a redesign of every request override.
|
||||
|
||||
### Snapshot And Inspection Design
|
||||
|
||||
The loader returns indexed, execution-owned definitions plus a private retained
|
||||
ConfigKit result. An adapter may implement the runner's narrow lookup interfaces
|
||||
over maps during migration; no repository chain, fallback-on-error, or per-call
|
||||
configuration load survives. Preserve defensive ownership across injected clients
|
||||
and public results. Cache prompt definition hashes at construction; hash effective
|
||||
captured content, not a stale resource path. Hash bytes need not match the old
|
||||
configuration format. Preserve rendered-prompt hashing semantics.
|
||||
|
||||
Resolve each effective content_file using its exact field origin and ConfigKit
|
||||
SourceContext. Join its relative name to BaseDir inside the returned rooted FS;
|
||||
reject absolute/outside-root paths and close every opened handle. Preserve path
|
||||
text, including meaningful whitespace, after separately checking nonblankness.
|
||||
Do not choose the child's source for an inherited reference. If there is no unique
|
||||
contributing source with context, fail with an actionable path/source error; the
|
||||
caller can supply an explicit Input filesystem context or override the field.
|
||||
Retain no resource capability for the purpose of public diagnostics.
|
||||
|
||||
Capture message text and parse message/session template syntax at construction
|
||||
with the same function names/options used for runtime rendering. Do not invoke
|
||||
input helpers or require Vars/artifacts while parsing. Reuse compiled templates
|
||||
where practical, binding request-specific input functions on safe per-call clones;
|
||||
never mutate a shared template function closure with request state. Runtime
|
||||
missing-variable/input failures remain runtime errors.
|
||||
|
||||
Expose a concrete read-only facade value:
|
||||
|
||||
```go
|
||||
func (e *Engine) InspectConfiguration() (ConfigurationInspection, error)
|
||||
func (v ConfigurationInspection) Metadata() configkit.Metadata
|
||||
func (v ConfigurationInspection) Sources() []configkit.SourceInfo
|
||||
func (v ConfigurationInspection) Diagnostics() []configkit.Diagnostic
|
||||
func (v ConfigurationInspection) Explain(path configkit.Path) (configkit.Explanation, bool)
|
||||
func (v ConfigurationInspection) WriteEffective(w io.Writer) error
|
||||
func (v ConfigurationInspection) WriteExplanation(w io.Writer, path configkit.Path) error
|
||||
```
|
||||
|
||||
Store an unexported narrow implementation reference; do not expose Result.Config,
|
||||
ConfigKit SourceContext, mutable runtime values, or internal types. Delegate
|
||||
queries and redaction to the retained result. Nil Engine returns ErrInvalidConfig;
|
||||
the zero inspection view has empty query results and writer operations return
|
||||
ErrInvalidConfig. Writer errors remain inspectable. Mark credential-shaped dynamic
|
||||
keys sensitive using Definition.Secret; the existing secret-key vocabulary can
|
||||
serve as a redaction predicate, not a source-rejection rule. Do not redact
|
||||
`api_key_env` names as credential values. This is not a
|
||||
promise that arbitrary prompt/provider content is safe to disclose.
|
||||
|
||||
Keep the runner independent of this view. Inspection calls perform no discovery,
|
||||
resource reads, environment credential lookup, or provider operations. Schemas,
|
||||
artifacts, preparation ownership, generation admission, retry/repair, observations,
|
||||
and cancellation remain at their established runtime boundaries.
|
||||
|
||||
## Stage 1 — Add The Dependency And Typed Loading Core
|
||||
|
||||
**Completed:** 2026-09-30.
|
||||
|
||||
Scope: module metadata, `internal/configuration` DTOs, real ConfigKit loading,
|
||||
and its focused tests. Leave engine assembly on the old path for this stage.
|
||||
|
||||
1. Add ConfigKit v0.5.0; update go.sum and factual OSS notices for it and its
|
||||
selected transitive dependencies. Preserve project licenses and the Go floor.
|
||||
2. Add keyed backend/profile/prompt DTOs and a private load entry point accepting
|
||||
ConfigKit Inputs and underlays. Call LoadLayered once, with no VersionRule or
|
||||
secret resolution bindings for API credentials. Use callback-scoped
|
||||
Resolve/Validate correctly.
|
||||
3. Implement effective presence extraction and the field rules above. Validate
|
||||
final domain values through existing helpers, resolving backend references
|
||||
against the final backend map; profiles have no final target requirement yet.
|
||||
Record normalization/default mutations, not arbitrary DTO bookkeeping fields.
|
||||
4. Preserve `*configkit.LoadError` through wrapping. Domain callback messages
|
||||
identify paths/problems without interpolating raw values or credentials.
|
||||
5. Add representative real-loader tests: keyed literal IDs, strict group conflict,
|
||||
recursive cross-tier overlay, a shared base with explicit zero/null/false, an
|
||||
unused invalid effective definition, and preservation of a ConfigKit load
|
||||
error. Cover an invalid lower-layer domain value replaced by a valid final
|
||||
value: construction succeeds. Do not reproduce ConfigKit's parser/inheritance
|
||||
matrix or add preflight/source-policy tests.
|
||||
|
||||
Exit: focused tests and `go test ./...` pass; old engine behavior is not presented
|
||||
as migrated. Internal inventory documents only the loader now actually present.
|
||||
|
||||
## Stage 2 — Add Source Discovery And Prompt Resource Capture
|
||||
|
||||
**Completed:** 2026-09-30.
|
||||
|
||||
Scope: source selection helpers and construction-time prompt materialization.
|
||||
|
||||
1. Implement private directory/FS discovery behind the chosen helper signatures;
|
||||
wrap discovered paths as ConfigKit sources, retaining explicit root context.
|
||||
Reuse useful filecatalog path handling without carrying profile metadata scans.
|
||||
2. Materialize effective prompt message text using field origins and passive
|
||||
contexts. Move reusable prompt/input/cache/output validation into the loader
|
||||
or shared domain owner; do not copy entire legacy parsers.
|
||||
3. Build immutable prompt indexes by exact ID/version and an ambiguity indicator
|
||||
for omitted versions. Capture declared-input inspection data and prompt hashes.
|
||||
4. Validate static template syntax using the renderer's grammar/function setup.
|
||||
Refactor that setup into one private owner if needed; do not render at load.
|
||||
5. Test native and embedded sources, imported/inherited relative content from a
|
||||
different directory, explicit mapping context, contextless failure, handle
|
||||
closure, malformed static syntax, and resource mutation after capture. Use
|
||||
minimal fixtures, sharing scenarios where the same adapter is exercised.
|
||||
|
||||
Exit: loader creates usable frozen prompt definitions without request data.
|
||||
`go test ./internal/configuration ./internal/prompt ./internal/promptdef` passes;
|
||||
existing execution tests still work through the old assembly until cutover.
|
||||
|
||||
## Stage 3 — Make Execution Settings Presence-Aware
|
||||
|
||||
**Completed:** 2026-09-30.
|
||||
|
||||
Scope: domain settings, target resolution, wire-presence propagation, profile
|
||||
inspection. No new source discovery in execution packages.
|
||||
|
||||
1. Build the immutable backend registry from the final loaded backend map. Add
|
||||
execution-owned presence/suppression state and construct indexed profiles
|
||||
from loader output plus that registry. Apply registry-derived
|
||||
values after inheritance, using the field table above.
|
||||
2. Replace scalar-zero/blank merge assumptions in target resolution. Configuration
|
||||
explicit zero reaches injected clients and the built-in request body; null does
|
||||
not resurrect a provider setting. Extend existing TargetPresence plumbing
|
||||
through preparation/results as necessary rather than creating a parallel path.
|
||||
3. Preserve request-level override precedence, API-key availability behavior,
|
||||
endpoint-only profiles, backend identity/capacity, and partial-profile runtime
|
||||
completion. Move final completeness checks after overrides. InspectProfile
|
||||
returns partial settings without treating missing model/endpoint as load errors.
|
||||
4. Test explicit numeric zero versus omission, map overlay and null, backend
|
||||
change without derived defaults leaking from parent, timeout null/zero,
|
||||
credential-source suppression, and completion through request overrides.
|
||||
Include one built-in-client wire test, not just an injected-client assertion.
|
||||
|
||||
Exit: focused domain/usecase/client tests pass. Any temporary adaptation for the
|
||||
old assembly is private, clearly limited to reaching Stage 7, and does not become
|
||||
a compatibility promise. Update affected target/presence GoDoc with this change.
|
||||
|
||||
## Stage 4 — Migrate The OpenRouter Catalog Assets
|
||||
|
||||
**Completed:** 2026-09-30. Published OpenRouter catalog `v3.20260930.0`
|
||||
from `f17d208cb7ad1fb043cda2266cd3011d11b40aa0`.
|
||||
|
||||
Scope: `../promptkit-backend-openrouter` only. Catalog maintainers own source and
|
||||
publishing checks here; PromptKit consumes its configuration through ConfigKit.
|
||||
|
||||
1. Read that repository's instructions; preserve unrelated work. Convert each
|
||||
profile asset to `profiles: {literal-id: {settings...}}`, preserving inventory
|
||||
and values, changing `base_profile` to `extends`, and removing embedded ID.
|
||||
Keep per-profile files and the exported FS/Root package; no dependency on
|
||||
PromptKit and no second legacy asset tree.
|
||||
2. Audit zero/empty values that previously meant unspecified. Remove those when
|
||||
omission was intended; retain intentional explicit values. Review any dynamic
|
||||
provider `extends` keys rather than silently changing outbound payloads.
|
||||
3. Replace `catalog/backend.json` with `catalog/backend.yml` containing a
|
||||
`backends: {openrouter: {settings...}}` mapping. Preserve endpoint, API-key
|
||||
environment name, concurrency, queue, and extra-parameter values; remove the
|
||||
duplicate id and old manifest schema_version. The file is an ordinary ConfigKit
|
||||
input, not a separately decoded manifest. Update embedded assets, module path
|
||||
and imports to `/v3`, tests, and catalog format/release documentation, with an
|
||||
incompatible-format migration note. Keep FS/Root as the minimal export surface.
|
||||
Catalog-local publishing checks stay in that repository. No provider probes.
|
||||
4. Validate under its procedure and commit. If publication is authorized, publish
|
||||
the next unused `v3.YYYYMMDD.N`; otherwise report the exact commit awaiting
|
||||
publication. The date/revision is selected then, not guessed in this plan.
|
||||
|
||||
Exit: one reviewed migrated catalog commit; existing immutable tags still serve
|
||||
old consumers. The new major module path identifies the changed asset contract;
|
||||
old consumers retain the published `/v2` assets.
|
||||
|
||||
## Stage 5 — Migrate The Rakestrawhome Catalog Assets
|
||||
|
||||
**Completed:** 2026-09-30. Published Rakestrawhome catalog `v3.20260930.0`
|
||||
from `993443d88c451901f4c917fa00c204fd3b54d28d`.
|
||||
|
||||
Scope: `../promptkit-backend-rakestrawhome` only. Apply the same keyed-format,
|
||||
presence, notice, validation, and publication treatment as Stage 4 to its smaller
|
||||
inventory, including the `/v3` transition and keyed `backends.rakestrawhome`
|
||||
definition in `catalog/backend.yml`. Preserve backend endpoint, credentials
|
||||
metadata, and capacity policy.
|
||||
Do not couple its date/revision to OpenRouter's release.
|
||||
|
||||
Exit: both compatible catalog versions are published and ordinarily resolvable
|
||||
before Stage 6 pins them. If permission or publication is unavailable, stop at
|
||||
this external dependency checkpoint; do not commit a local replace, guessed
|
||||
version, or legacy-to-new runtime converter to bypass it.
|
||||
|
||||
## Stage 6 — Integrate Catalog Sources And Maintained Profiles
|
||||
|
||||
**Completed:** 2026-09-30.
|
||||
|
||||
Scope: `internal/catalog`, maintained definitions, catalog integration tests,
|
||||
dependency pins/notices. Keep this separate from public API migration.
|
||||
|
||||
1. Pin the published compatible catalog versions and exact attribution commits.
|
||||
Update imports to `/v3`. Replace manifest decoding with ConfigKit file Inputs.
|
||||
2. Supply backend and profile assets as ConfigKit Inputs in the first
|
||||
underlay. Supply PromptKit-owned keyed Mapping definitions as the next group:
|
||||
standard extends the selected catalog's current maintained base; auto retains
|
||||
its current direct model/backend definition. Preserve both literal names.
|
||||
3. Retire source-layout allowlists, catalog preflight, source ownership/reserved
|
||||
prefix checks, connection-field prohibitions, and secret-key source rejection.
|
||||
The adapter selects known asset entry points and discovers configuration files;
|
||||
ConfigKit acquires their contents. Domain validation sees only the final
|
||||
assembled configuration. Consumers may overlay catalog backends, profiles,
|
||||
and their bases through the same configuration model.
|
||||
4. Test real imported assets offline with the new loader, maintained-name
|
||||
resolution, consumer override of a maintained base/backend, and rejection of
|
||||
an invalid final backend. Test a replaced lower-layer domain error succeeds.
|
||||
Do not independently validate catalog-only intermediate configurations, parse
|
||||
raw assets in PromptKit tests, or audit origins to establish ownership.
|
||||
|
||||
Exit: catalog and owned definitions participate in the new keyed configuration
|
||||
model without a legacy profile parser on that path. Catalog integration tests
|
||||
pass against released modules with `GOWORK=off` and no replacements.
|
||||
|
||||
## Stage 7 — Cut Over Engine Assembly And Public Construction
|
||||
|
||||
**Completed:** 2026-09-30.
|
||||
|
||||
Scope: `engine.go`, configuration facade/helpers, `internal/engine`, facade
|
||||
adapters, and migration of their direct test setup.
|
||||
|
||||
1. Introduce the Construction Surface exactly as specified above. Remove the
|
||||
superseded source/profile options and convenience types rather than keeping
|
||||
deprecated pass-through wrappers. Keep intentional runtime injection APIs.
|
||||
2. Load the combined backend/profile/prompt configuration, construct its runtime
|
||||
registry and indexed snapshot, then assemble renderer, validator, capacity
|
||||
manager, and runner. Return no Engine on construction failure.
|
||||
Translate construction errors to ErrInvalidConfig while preserving LoadError
|
||||
via errors.As. Resource errors include semantic path and safe source context.
|
||||
3. Connect indexed prompt/profile lookups to preparation and inspection. Missing
|
||||
selections keep meaningful not-found errors; prompt version ambiguity stays a
|
||||
selection error. Old parse/load error categories need not be manufactured for
|
||||
errors now detected at construction.
|
||||
4. Move root test fixtures/setup and maintained example construction to ConfigKit
|
||||
sources, keyed definitions, ProfileSource, and BackendSource so all consumers
|
||||
still compile. Update local-backend examples to choose their ID as a map key.
|
||||
Update expected timing only for the deliberate eager-validation change. Keep
|
||||
behavioral assertions for prompting, overrides, and errors.
|
||||
5. Add public tests for a minimal custom local backend and programmatic profile,
|
||||
plus embedded defaults/operator/programmatic tiers. No elaborate consumer
|
||||
setup should be needed merely to send one prompt.
|
||||
|
||||
Exit: the public Engine uses only the new loading path. Root/package tests pass;
|
||||
all package/example sources compile. Update GoDoc and consumer construction
|
||||
guidance in this stage so removed options are not presented as usable.
|
||||
|
||||
## Stage 8 — Complete Frozen Rendering And Lifecycle Integration
|
||||
|
||||
**Completed:** 2026-09-30.
|
||||
|
||||
Scope: renderer/runtime consumption of the snapshot, ownership, and lifecycle
|
||||
regressions. Stage 2 already owns acquisition and static syntax validation.
|
||||
|
||||
1. Use captured text and parsed templates without per-run configuration reads,
|
||||
parsing, or inheritance. Bind input helpers per invocation safely; keep missing
|
||||
request data errors and direct-session template bypass behavior.
|
||||
2. Verify preparation and execution still handle artifacts and schema compilation
|
||||
at invocation time, honor output overrides, freeze prepared resources, and
|
||||
keep prepared handles bound to their engine. Credentials remain at runtime.
|
||||
3. Test an old engine after configuration/resource edits and a newly constructed
|
||||
engine observing the edits. Test concurrent renders with different Vars/input
|
||||
helpers and no data leakage. Use an FS that rejects reads after construction
|
||||
to demonstrate no configuration/content-file reopening during ordinary runs;
|
||||
do not incorrectly prohibit request artifact/schema reads.
|
||||
4. Preserve appended-message ordering, structural repair, retries, admission,
|
||||
cancellation, observers, and result metadata through existing regression
|
||||
tests; add only distinct missing cross-boundary cases.
|
||||
|
||||
Exit: focused race tests pass for assembled rendering/preparation and lifecycle.
|
||||
No retained callbacks capture ConfigKit resolution contexts or request state.
|
||||
|
||||
## Stage 9 — Expose Explicit Configuration Inspection
|
||||
|
||||
**Completed:** 2026-09-30.
|
||||
|
||||
Scope: the read-only facade and retained ConfigKit inspection adapter.
|
||||
|
||||
1. Implement ConfigurationInspection and its specified methods without exposing
|
||||
Result.Config or filesystem capabilities. Retain a private narrow view of the
|
||||
ConfigKit result; no regeneration of origins from runtime values.
|
||||
2. Ensure effective/explanation reports use ConfigKit redaction, including the
|
||||
selected sensitivity predicate. Preserve safe value-free domain diagnostics.
|
||||
3. Test a targeted literal-key path with layered/inherited history, detached
|
||||
returned slices/records, writer failures, zero/nil behavior, and redacted secret
|
||||
shaped provider values. Demonstrate inspection still works after source removal
|
||||
and makes no filesystem/provider calls.
|
||||
4. Document construction inspection versus InspectProfile and prepared details.
|
||||
Consumers choose report disclosure; do not add CLI commands or automatic logs.
|
||||
|
||||
Exit: public inspection tests pass, including concurrent inspection/execution.
|
||||
The runner has no reference to the inspector or ConfigKit metadata types.
|
||||
|
||||
## Stage 10 — Migrate Examples And Canonical Format Guidance
|
||||
|
||||
**Completed:** 2026-09-30.
|
||||
|
||||
Scope: maintained examples and user-facing documentation.
|
||||
|
||||
1. Complete the preparation, execution, and observation example migration begun
|
||||
at public cutover. Preserve their distinct existing semantic assertions; extend
|
||||
the preparation example with a small explicit inspection query if useful.
|
||||
2. Update `docs/formats.md` to own the new keyed syntax, source-group semantics,
|
||||
inheritance/presence/null table, eager validation, snapshot lifetime, and
|
||||
resource-context rules. Link ConfigKit for generic grammar; do not copy its
|
||||
complete parser/inheritance contract.
|
||||
3. Update README and `docs/consumers/pkg-promptkit.md` with minimal local-backend,
|
||||
file/embedded source, programmatic Mapping/ProfileSource, explicit inspection,
|
||||
and engine-replacement workflows. Explain main versus ordered underlays and
|
||||
backend layering in place of registration-only restrictions.
|
||||
4. Add concise old-to-new migration guidance in the consumer guide: removed API
|
||||
options, exact IDs, `base_profile` to `extends`, zero/false/null, recursive maps,
|
||||
no clear via `{}`, reserved `extends` runtime escape, failure timing, and the
|
||||
fact that changing a model/backend does not clear unrelated explicit fields.
|
||||
Remove old automatic reload claims. Release notes later link to this owner.
|
||||
|
||||
Exit: all three offline examples run with their semantic checks; documentation
|
||||
links and snippets match the implemented API. No unreleased version is tagged or
|
||||
published, and no speculative release-note version is invented.
|
||||
|
||||
## Stage 11 — Remove Superseded Machinery And Consolidate Tests
|
||||
|
||||
**Completed:** 2026-09-30.
|
||||
|
||||
Scope: dead loading/resolution code and its obsolete tests, module cleanup, and
|
||||
internal architectural documentation.
|
||||
|
||||
1. Delete lazy prompt/profile metadata prepasses, filesystem repositories,
|
||||
fallback-on-error overlays, custom base-chain resolver/depth cap, zero-sentinel
|
||||
profile merging, and temporary stage adapters once no caller needs them.
|
||||
Remove the separate backend-manifest decoder, source-policy audits, and
|
||||
pre-handoff backend registration validation. Keep final-value domain checks.
|
||||
Retain narrow lookup interfaces only where they still simplify real consumers.
|
||||
2. Retain useful filecatalog/schema utilities and JSON-value ownership/validation.
|
||||
Remove direct yaml.v3 use and dependency if no remaining owner requires it;
|
||||
do not claim all YAML dependencies disappear when ConfigKit still uses YAML.
|
||||
3. Move meaningful regressions from deleted package tests to their new owning
|
||||
boundary. Retire tests about old internals, repeated parser coverage,
|
||||
source-policy enforcement, or superseded semantics. Do not replace lost runtime
|
||||
protection with loader-only tests. Keep private assembly checks separate from
|
||||
external facade tests.
|
||||
4. Update `docs/internal/overview.md`, `docs/internal/sources.md`, relevant renderer
|
||||
and engine documents, and the implemented-structure paragraphs in architecture
|
||||
policy. Describe the new package/dependency boundary without turning names or
|
||||
chosen adapters into permanent invariants. Update OSS notices with module tidy.
|
||||
|
||||
Exit: no legacy source path remains, no internal package imports the public root,
|
||||
runtime/domain packages do not import ConfigKit, and `go test ./...` passes with
|
||||
the reduced implementation/test inventory.
|
||||
|
||||
## Stage 12 — Validate Scale And The Complete Consumer Workflow
|
||||
|
||||
**Completed:** 2026-09-30.
|
||||
|
||||
Scope: final integration evidence and high-confidence simplification, not another
|
||||
feature expansion or release gate.
|
||||
|
||||
1. Add focused benchmarks for construction with many independent definitions,
|
||||
shared inherited bases, and several tiers; add an already-constructed selection
|
||||
benchmark. Report allocations and input/output shape; inherited materialized
|
||||
output/history may dominate. Do not impose timing thresholds or new caps.
|
||||
2. Inspect for repeated full-map copies, per-run provenance calls, rescans,
|
||||
repeated inheritance, and unnecessary double parsing. Fix demonstrated local
|
||||
inefficiencies with straightforward ownership/indexing; no global cache.
|
||||
3. Run the repository's existing complete local check script once on the final
|
||||
candidate, including race tests and offline examples. Investigate failures;
|
||||
do not weaken meaningful tests to get a green result. Stage intended new
|
||||
files before the script's tracked-file checks.
|
||||
4. Review the feature-roadmap capability list against actual public workflows:
|
||||
independent inspection, partial profile completion, optional credentials,
|
||||
source contexts, artifacts/schemas, prepared ownership, backend capacity,
|
||||
overrides, appended messages, repair/retries, observations, and hashes.
|
||||
5. Report completed stages, selected catalog versions, validation evidence, and
|
||||
remaining issues. Minor-release documentation/version selection and roadmap
|
||||
retirement belong to the separately authorized release preparation workflow.
|
||||
|
||||
Exit: the feature roadmap's end state is demonstrably implemented, with one
|
||||
configuration lifecycle, explicit operator inspection, no legacy loader, and
|
||||
no newly invented provider checks or human release gates.
|
||||
|
||||
The catalog-validation placement question is resolved by the configuration-boundary
|
||||
ADR. No design question remains open here. Catalog publication authority and
|
||||
ordinary module availability remain operational checkpoints for the catalog stages.
|
||||
@@ -1,126 +0,0 @@
|
||||
# ConfigKit Adoption Review Findings
|
||||
|
||||
Review date: 2026-10-03. Reviewed implementation commits `2b847ac` through
|
||||
`66c55b7` against the [feature roadmap](configkit.md),
|
||||
[implementation plan](implementation.md), and accepted
|
||||
[configuration-boundary ADR](../adr/0001-configuration-boundary.md).
|
||||
|
||||
## Summary
|
||||
|
||||
The principal architecture is in place: one ConfigKit load assembles backend,
|
||||
profile, and prompt configuration; PromptKit validates effective values, captures
|
||||
resources/templates, and builds execution-owned indexes. Catalog preflight and
|
||||
legacy profile resolution are removed. Operator inspection delegates to the
|
||||
retained ConfigKit result separately from ordinary execution.
|
||||
|
||||
Both findings have been addressed. The original review evidence is retained
|
||||
below; the resolution section records the changes and regression coverage.
|
||||
|
||||
## 1. P2 — Message Normalization Loses Content-File Resource Origins
|
||||
|
||||
Location: `internal/configuration/prompts.go`, particularly `resolvePrompts`
|
||||
lines 69–99 and `readContent`'s effective-origin lookup.
|
||||
|
||||
When a role or cache-control value needs normalization, `resolvePrompts` records
|
||||
a normalization operation for the entire `messages` sequence in addition to the
|
||||
field-level operations. The resulting metadata no longer provides the source
|
||||
location needed by `readContent` for an unchanged `content_file`. Construction
|
||||
then rejects an otherwise valid prompt.
|
||||
|
||||
Confirmed through the public API using FilesystemInputs over an fs.FS containing
|
||||
this document and a readable `body.txt`:
|
||||
|
||||
```yaml
|
||||
prompts:
|
||||
p:
|
||||
v1:
|
||||
messages:
|
||||
- role: " USER "
|
||||
content_file: body.txt
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
```
|
||||
|
||||
NewEngine fails with
|
||||
`/prompts/p/v1/messages/0/content_file: content_file needs one contributing source with a filesystem context`.
|
||||
Changing only the role to `user` succeeds. Role trimming/lowercasing and
|
||||
source-relative message capture are both intended capabilities; requiring
|
||||
already-normalized metadata is an accidental restriction.
|
||||
|
||||
Required outcome:
|
||||
|
||||
- Preserve ConfigKit's effective resource context while normalizing neighboring
|
||||
message metadata. Review the broad sequence normalization record and how the
|
||||
final input origin is retained across domain normalization.
|
||||
- Use effective handoff metadata/context, not historical traversal, raw-source
|
||||
inspection, source rereading, or an invented fallback directory.
|
||||
- Add focused regressions combining file-backed content with role and
|
||||
cache-control normalization, including inherited/imported content. Existing
|
||||
tests cover normalization and resource capture separately but miss this
|
||||
interaction. Verify inspection remains accurate for untouched content fields.
|
||||
|
||||
## 2. P2 — Profiles Gain An Unrequested Environment-Name Restriction
|
||||
|
||||
Location: `internal/configuration/load.go`, profile validation at lines 278–279,
|
||||
using the package's `envName` regular expression.
|
||||
|
||||
Profile `api_key_env` values now must match `[A-Za-z_][A-Za-z0-9_]*`. The previous
|
||||
profile path accepted environment lookup names without this shell-identifier
|
||||
restriction; the regular expression previously governed backend registration.
|
||||
The refactor extends it to profiles even though arbitrary environment names
|
||||
supported by the host remain usable through Go's environment lookup and the
|
||||
request override path.
|
||||
|
||||
Confirmed through NewEngine with an otherwise valid endpoint/model profile:
|
||||
`api_key_env: NORMAL_KEY` succeeds, while `api_key_env: service.api-key` fails
|
||||
construction. The latter is a usable environment name on the reviewed Linux
|
||||
platform. This restriction is unrelated to configuration layering, ConfigKit
|
||||
typing, or optional credential availability.
|
||||
|
||||
The implementation plan's reference to existing naming rules is not evidence
|
||||
of an express user decision to introduce this new profile restriction. The
|
||||
requested compatibility break permits adopting ConfigKit semantics; it does
|
||||
not itself authorize additional unrelated name constraints.
|
||||
|
||||
Required outcome:
|
||||
|
||||
- Remove the newly introduced profile shell-identifier check, preserving the
|
||||
established trimming, explicit clearing, optional environment lookup, and
|
||||
direct-key precedence behavior.
|
||||
- Add a public-path regression for a supported non-shell-identifier environment
|
||||
name and for its unset/empty optional behavior, with controlled test environment
|
||||
state and a synthetic credential. Keep any prerequisite validation limited to
|
||||
actual supported environment lookup requirements.
|
||||
- Correct the plan's profile-field guidance so later work does not reintroduce
|
||||
this restriction. Treat any broader change to existing backend naming policy
|
||||
separately rather than silently extending or weakening it in this fix.
|
||||
|
||||
## Validation Performed
|
||||
|
||||
- `go test ./...` passed on Go 1.26.8.
|
||||
- `GOTOOLCHAIN=go1.25.5 go test ./...` passed on the declared minimum toolchain.
|
||||
- `go test -race ./...` passed.
|
||||
- Separate offline public-API reproductions confirmed both findings. Missing
|
||||
message/session variables also remained runtime errors in targeted checks.
|
||||
|
||||
The passing suite does not cover the two failing cases above. No provider
|
||||
requests were made. Beyond these findings, the reviewed implementation did not
|
||||
show another actionable ownership-boundary violation or strengthened guarantee.
|
||||
|
||||
## Resolution
|
||||
|
||||
1. Message role/cache-control normalization now occurs on execution-owned copies
|
||||
during materialization. ConfigKit's captured message sequence and its origins
|
||||
remain intact. The imported/inherited content regression now includes role
|
||||
and cache-control normalization and asserts the unchanged field's source in
|
||||
inspection metadata. This also removes duplicate cache-control normalization.
|
||||
2. Removed the profile environment-name regular expression check, leaving backend
|
||||
validation unchanged. A public engine/HTTP-transport regression covers a
|
||||
non-shell environment name, unset/empty/whitespace values, explicit clearing,
|
||||
trimming, and direct-key precedence. The implementation plan now states this
|
||||
distinction explicitly.
|
||||
|
||||
Both regressions failed against the reviewed implementation before the fixes.
|
||||
After remediation, the full ordinary and race-enabled suites, affected packages
|
||||
on Go 1.25.5, `go vet ./...`, and documentation/whitespace checks passed.
|
||||
@@ -82,12 +82,12 @@ var (
|
||||
ErrValidation = errors.New("failed to validate output")
|
||||
)
|
||||
|
||||
// Engine inspects prompts and profiles and prepares and runs Promptkit prompt
|
||||
// requests.
|
||||
// Engine inspects prompts and profiles, enumerates prompt requirements, and
|
||||
// prepares and runs Promptkit prompt requests.
|
||||
//
|
||||
// An Engine is safe for concurrent calls to [Engine.InspectConfiguration],
|
||||
// [Engine.InspectPrompt],
|
||||
// [Engine.InspectProfile], [Engine.Prepare], [Engine.PrepareExecution],
|
||||
// [Engine.InspectRequirements], [Engine.InspectProfile], [Engine.Prepare], [Engine.PrepareExecution],
|
||||
// [Engine.Run], and [Engine.RunPrepared]. Each Engine owns independent
|
||||
// backend-capacity pools that coordinate Run and RunPrepared admission and
|
||||
// model generation. Injected collaborators may still be invoked concurrently
|
||||
|
||||
@@ -36,6 +36,17 @@ func main() {
|
||||
if _, found := view.Explain(configkit.Keys("profiles", "offline-example", "model")); !found {
|
||||
exit(fmt.Errorf("configured profile model was not found"))
|
||||
}
|
||||
requirements, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{
|
||||
PromptID: "example.prepare",
|
||||
})
|
||||
if err != nil {
|
||||
exit(err)
|
||||
}
|
||||
if !requirements.Complete || len(requirements.Inputs) != 1 ||
|
||||
requirements.Inputs[0].Name != "note" ||
|
||||
requirements.Inputs[0].Kind != promptkit.RequirementRequired {
|
||||
exit(fmt.Errorf("required note input was not discovered before preparation"))
|
||||
}
|
||||
|
||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||
PromptID: "example.prepare",
|
||||
|
||||
9
go.mod
9
go.mod
@@ -1,15 +1,20 @@
|
||||
module gitea.maximumdirect.net/eric/promptkit
|
||||
|
||||
go 1.25.5
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
gitea.maximumdirect.net/eric/ConfigKit v0.5.0
|
||||
gitea.maximumdirect.net/eric/ConfigKit v0.7.1
|
||||
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v3 v3.20260930.0
|
||||
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v3 v3.20260930.0
|
||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2
|
||||
)
|
||||
|
||||
require (
|
||||
gitea.maximumdirect.net/eric/ReleaseKit v0.4.1 // indirect
|
||||
github.com/yuin/goldmark/v2 v2.1.6 // indirect
|
||||
go.yaml.in/yaml/v4 v4.0.0-rc.6 // indirect
|
||||
golang.org/x/mod v0.40.0 // indirect
|
||||
golang.org/x/text v0.14.0 // indirect
|
||||
)
|
||||
|
||||
tool gitea.maximumdirect.net/eric/ReleaseKit/cmd/releasekit
|
||||
|
||||
10
go.sum
10
go.sum
@@ -1,5 +1,7 @@
|
||||
gitea.maximumdirect.net/eric/ConfigKit v0.5.0 h1:2KwG10+kApWM3avOoYA+p768xcbHUp7tQ2rqizBNpyg=
|
||||
gitea.maximumdirect.net/eric/ConfigKit v0.5.0/go.mod h1:tYWpsrbLyu05r8+1MapWp5fN6ZrX9I8hU2xEPedYmYk=
|
||||
gitea.maximumdirect.net/eric/ConfigKit v0.7.1 h1:q3HrX4WWCPt1WfMydqFEaAmCf9s6Dq+vknJj0C/mdcI=
|
||||
gitea.maximumdirect.net/eric/ConfigKit v0.7.1/go.mod h1:C82sbyKHYMV1OwaD2GJanFrwTfcZfjlW2JsGze5b0kA=
|
||||
gitea.maximumdirect.net/eric/ReleaseKit v0.4.1 h1:Qn+mdebYOzCVCEXgCbuGSO/WHmJtItwaQ8e3XwY3qtI=
|
||||
gitea.maximumdirect.net/eric/ReleaseKit v0.4.1/go.mod h1:LVodqNQ3OIKv9NBq+fFeyMByd1lHY25YOmmpah/NaNI=
|
||||
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v3 v3.20260930.0 h1:lzfCDbzJQRGzIlRPRprnAV9hVVL5zUQy80VMQ0kbYa0=
|
||||
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v3 v3.20260930.0/go.mod h1:HjS0f+3q0IdHCTEz+VVAEgIwIWJFAZUjuKdAtvtqKdg=
|
||||
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v3 v3.20260930.0 h1:h12ZTRldAmpLAomj0oQRGteFxHLN1LPE6XDl9zJNbm8=
|
||||
@@ -8,8 +10,12 @@ github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxK
|
||||
github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
|
||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ=
|
||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU=
|
||||
github.com/yuin/goldmark/v2 v2.1.6 h1:igy0q9JcqovoFXFx/XjtU8sHVhcUHavPj2sA3EoKJUE=
|
||||
github.com/yuin/goldmark/v2 v2.1.6/go.mod h1:G6M4/qOFtfn01/o14BU1UR2Lo5N3S9Qo7xCuz/sHjGQ=
|
||||
go.yaml.in/yaml/v4 v4.0.0-rc.6 h1:1h7H1ohdUh93/FyE4YaDa1Zh64K6VVbjF4K6WUxMtH4=
|
||||
go.yaml.in/yaml/v4 v4.0.0-rc.6/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0=
|
||||
golang.org/x/mod v0.40.0 h1:hUv+3cXcdRHz08UmSiOob7sadHig73uo5bkXxQ/tvUs=
|
||||
golang.org/x/mod v0.40.0/go.mod h1:0/weTWkPWGBikyTWAX3dkjVztMmBA5hM0DH6BElSupE=
|
||||
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
|
||||
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
|
||||
@@ -21,6 +21,7 @@ var (
|
||||
)
|
||||
|
||||
const fileReadChunkSize = 64 * 1024
|
||||
const fileReadInitialCapacityLimit = 1 << 20
|
||||
|
||||
// Reader resolves artifact references into actual artifacts.
|
||||
type Reader interface {
|
||||
@@ -133,7 +134,11 @@ func readFileArtifact(ctx context.Context, path string, open func(string) (artif
|
||||
return nil, fmt.Errorf("%w: %s", ErrUnsupportedFile, path)
|
||||
}
|
||||
|
||||
data := make([]byte, 0)
|
||||
capacity := 0
|
||||
if size := openedInfo.Size(); size > 0 {
|
||||
capacity = int(min(size, int64(fileReadInitialCapacityLimit)))
|
||||
}
|
||||
data := make([]byte, 0, capacity)
|
||||
chunk := make([]byte, fileReadChunkSize)
|
||||
for {
|
||||
if err := ctx.Err(); err != nil {
|
||||
@@ -153,6 +158,13 @@ func readFileArtifact(ctx context.Context, path string, open func(string) (artif
|
||||
return nil, fmt.Errorf("failed to read file %s: %w", path, readErr)
|
||||
}
|
||||
}
|
||||
if cap(data)-len(data) > len(data) {
|
||||
// A stale size hint should not make a small completed artifact retain a
|
||||
// disproportionately large backing array.
|
||||
compact := make([]byte, len(data))
|
||||
copy(compact, data)
|
||||
data = compact
|
||||
}
|
||||
|
||||
contentType := mime.TypeByExtension(filepath.Ext(path))
|
||||
if contentType == "" {
|
||||
|
||||
180
internal/artifact/reader_benchmark_test.go
Normal file
180
internal/artifact/reader_benchmark_test.go
Normal file
@@ -0,0 +1,180 @@
|
||||
package artifact
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"fmt"
|
||||
"mime"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
)
|
||||
|
||||
var artifactFileReadBenchmarkSink *domain.Artifact
|
||||
|
||||
func BenchmarkArtifactFileRead(b *testing.B) {
|
||||
for _, fixture := range makeArtifactReadBenchmarkFiles(b) {
|
||||
b.Run(fixture.name, func(b *testing.B) {
|
||||
reader := &fileReader{open: openArtifactFile}
|
||||
ref := domain.ArtifactRef{Type: domain.ArtifactRefFile, URI: fixture.path}
|
||||
checkArtifactFileRead(b, reader, ref, fixture.body)
|
||||
|
||||
b.ReportAllocs()
|
||||
b.SetBytes(int64(len(fixture.body)))
|
||||
b.ResetTimer()
|
||||
for range b.N {
|
||||
artifact, err := reader.Read(context.Background(), ref)
|
||||
if err != nil {
|
||||
b.Fatalf("read artifact: %v", err)
|
||||
}
|
||||
artifactFileReadBenchmarkSink = artifact
|
||||
}
|
||||
b.StopTimer()
|
||||
runtime.KeepAlive(artifactFileReadBenchmarkSink)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// BenchmarkRetainedArtifactFileRead measures live artifact bodies when an
|
||||
// opened file reports a size larger than the content that reaches EOF.
|
||||
func BenchmarkRetainedArtifactFileRead(b *testing.B) {
|
||||
for _, fixture := range []struct {
|
||||
name string
|
||||
body []byte
|
||||
}{
|
||||
{name: "small", body: bytes.Repeat([]byte("x"), 256<<10)},
|
||||
{name: "empty", body: []byte{}},
|
||||
} {
|
||||
b.Run(fixture.name, func(b *testing.B) {
|
||||
benchmarkRetainedArtifactFileRead(b, fixture.body)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func benchmarkRetainedArtifactFileRead(b *testing.B, body []byte) {
|
||||
path := filepath.Join(b.TempDir(), "overreported.txt")
|
||||
if err := os.WriteFile(path, body, 0o600); err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
mime.TypeByExtension(filepath.Ext(path))
|
||||
|
||||
reader := &fileReader{open: func(path string) (artifactFile, error) {
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &sizeReportingArtifactFile{artifactFile: file, reportedSize: 16 << 20}, nil
|
||||
}}
|
||||
ref := domain.ArtifactRef{Type: domain.ArtifactRefFile, URI: path}
|
||||
checkArtifactFileRead(b, reader, ref, body)
|
||||
|
||||
const retainedBatchSize = 4
|
||||
retained := make([]*domain.Artifact, retainedBatchSize)
|
||||
artifactFileReadBenchmarkSink = nil
|
||||
runtime.GC()
|
||||
var setupMemory runtime.MemStats
|
||||
runtime.ReadMemStats(&setupMemory)
|
||||
runtime.KeepAlive(reader)
|
||||
runtime.KeepAlive(ref)
|
||||
|
||||
b.ReportAllocs()
|
||||
b.SetBytes(int64(len(body)))
|
||||
b.ResetTimer()
|
||||
for iteration := 0; iteration < b.N; iteration++ {
|
||||
artifact, err := reader.Read(context.Background(), ref)
|
||||
if err != nil {
|
||||
b.Fatalf("read retained artifact: %v", err)
|
||||
}
|
||||
retained[iteration%len(retained)] = artifact
|
||||
}
|
||||
b.StopTimer()
|
||||
for i, artifact := range retained {
|
||||
if artifact != nil {
|
||||
continue
|
||||
}
|
||||
artifact, err := reader.Read(context.Background(), ref)
|
||||
if err != nil {
|
||||
b.Fatalf("fill retained artifact batch: %v", err)
|
||||
}
|
||||
retained[i] = artifact
|
||||
}
|
||||
runtime.GC()
|
||||
var retainedMemory runtime.MemStats
|
||||
runtime.ReadMemStats(&retainedMemory)
|
||||
b.ReportMetric(float64(int64(retainedMemory.HeapAlloc)-int64(setupMemory.HeapAlloc)), "retained-B")
|
||||
runtime.KeepAlive(retained)
|
||||
runtime.KeepAlive(reader)
|
||||
runtime.KeepAlive(ref)
|
||||
}
|
||||
|
||||
type artifactReadBenchmarkFile struct {
|
||||
name string
|
||||
path string
|
||||
body []byte
|
||||
}
|
||||
|
||||
func makeArtifactReadBenchmarkFiles(b *testing.B) []artifactReadBenchmarkFile {
|
||||
b.Helper()
|
||||
fixtures := []struct {
|
||||
name string
|
||||
size int
|
||||
}{
|
||||
{name: "small_4KiB", size: 4 << 10},
|
||||
{name: "1MiB", size: 1 << 20},
|
||||
{name: "large_4MiB", size: 4 << 20},
|
||||
}
|
||||
results := make([]artifactReadBenchmarkFile, 0, len(fixtures))
|
||||
for _, fixture := range fixtures {
|
||||
body := bytes.Repeat([]byte("x"), fixture.size)
|
||||
path := filepath.Join(b.TempDir(), fixture.name+".txt")
|
||||
if err := os.WriteFile(path, body, 0o600); err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
mime.TypeByExtension(filepath.Ext(path))
|
||||
results = append(results, artifactReadBenchmarkFile{
|
||||
name: fixture.name,
|
||||
path: path,
|
||||
body: body,
|
||||
})
|
||||
}
|
||||
return results
|
||||
}
|
||||
|
||||
func checkArtifactFileRead(b *testing.B, reader Reader, ref domain.ArtifactRef, want []byte) {
|
||||
b.Helper()
|
||||
artifact, err := reader.Read(context.Background(), ref)
|
||||
if err != nil {
|
||||
b.Fatalf("check artifact read: %v", err)
|
||||
}
|
||||
if !bytes.Equal(artifact.Body, want) || artifact.Size != int64(len(want)) {
|
||||
b.Fatalf("artifact size=%d body length=%d, want %d", artifact.Size, len(artifact.Body), len(want))
|
||||
}
|
||||
wantHash := fmt.Sprintf("%x", sha256.Sum256(want))
|
||||
if artifact.Hash != wantHash {
|
||||
b.Fatalf("artifact hash=%q, want %q", artifact.Hash, wantHash)
|
||||
}
|
||||
}
|
||||
|
||||
type sizeReportingArtifactFile struct {
|
||||
artifactFile
|
||||
reportedSize int64
|
||||
}
|
||||
|
||||
func (file *sizeReportingArtifactFile) Stat() (os.FileInfo, error) {
|
||||
info, err := file.artifactFile.Stat()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return reportedArtifactFileInfo{FileInfo: info, size: file.reportedSize}, nil
|
||||
}
|
||||
|
||||
type reportedArtifactFileInfo struct {
|
||||
os.FileInfo
|
||||
size int64
|
||||
}
|
||||
|
||||
func (info reportedArtifactFileInfo) Size() int64 { return info.size }
|
||||
@@ -3,7 +3,9 @@ package artifact
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
@@ -232,6 +234,64 @@ func TestFileReaderFailuresAndMetadata(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
func TestFileReaderUsesOpenedSizeOnlyAsACapacityHint(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
body []byte
|
||||
reportedSize int64
|
||||
}{
|
||||
{name: "overreported size", body: []byte("actual file content"), reportedSize: 128},
|
||||
{name: "overreported size with empty content", body: []byte{}, reportedSize: 16 << 20},
|
||||
{name: "underreported size", body: bytes.Repeat([]byte("x"), fileReadChunkSize+17), reportedSize: 1},
|
||||
{name: "very large hint with small content", body: []byte("small actual content"), reportedSize: int64(^uint64(0) >> 1)},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
filePath := filepath.Join(t.TempDir(), "artifact.txt")
|
||||
if err := os.WriteFile(filePath, tc.body, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reader := &fileReader{open: func(path string) (artifactFile, error) {
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
info, err := file.Stat()
|
||||
if err != nil {
|
||||
file.Close()
|
||||
return nil, err
|
||||
}
|
||||
return &reportedInfoFile{
|
||||
artifactFile: file,
|
||||
info: reportedFileSizeInfo{FileInfo: info, size: tc.reportedSize},
|
||||
}, nil
|
||||
}}
|
||||
|
||||
artifact, err := reader.Read(context.Background(), domain.ArtifactRef{
|
||||
Type: domain.ArtifactRefFile,
|
||||
URI: filePath,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("read artifact: %v", err)
|
||||
}
|
||||
if !bytes.Equal(artifact.Body, tc.body) {
|
||||
t.Fatalf("artifact body length=%d, want %d", len(artifact.Body), len(tc.body))
|
||||
}
|
||||
if artifact.Size != int64(len(tc.body)) {
|
||||
t.Fatalf("artifact size=%d, want %d", artifact.Size, len(tc.body))
|
||||
}
|
||||
if artifact.Name != filepath.Base(filePath) || artifact.URI != filePath || artifact.ContentType == "" {
|
||||
t.Fatalf("unexpected artifact metadata: name=%q URI=%q content type=%q", artifact.Name, artifact.URI, artifact.ContentType)
|
||||
}
|
||||
wantHash := fmt.Sprintf("%x", sha256.Sum256(tc.body))
|
||||
if artifact.Hash != wantHash {
|
||||
t.Fatalf("artifact hash=%q, want %q", artifact.Hash, wantHash)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileReaderCancelsAfterReadProgress(t *testing.T) {
|
||||
filePath := filepath.Join(t.TempDir(), "artifact.bin")
|
||||
content := bytes.Repeat([]byte("x"), fileReadChunkSize*2)
|
||||
@@ -268,6 +328,13 @@ func (f *reportedInfoFile) Stat() (os.FileInfo, error) {
|
||||
return f.info, nil
|
||||
}
|
||||
|
||||
type reportedFileSizeInfo struct {
|
||||
os.FileInfo
|
||||
size int64
|
||||
}
|
||||
|
||||
func (info reportedFileSizeInfo) Size() int64 { return info.size }
|
||||
|
||||
type cancelAfterProgressFile struct {
|
||||
artifactFile
|
||||
cancel context.CancelFunc
|
||||
|
||||
@@ -13,9 +13,12 @@ import (
|
||||
const benchmarkDefinitionCount = 128
|
||||
|
||||
var (
|
||||
constructionResult *loaded
|
||||
selectedPrompt *domain.PromptDefinition
|
||||
selectedProfile *domain.ExecutionProfile
|
||||
constructionResult *loaded
|
||||
selectedPrompt *domain.PromptDefinition
|
||||
selectedPromptHash string
|
||||
selectedPromptInspection *domain.PromptInspection
|
||||
selectedProfile *domain.ExecutionProfile
|
||||
selectedProfileIdentity string
|
||||
)
|
||||
|
||||
type configurationShape struct {
|
||||
@@ -177,3 +180,116 @@ func BenchmarkConfigurationSelection(b *testing.B) {
|
||||
b.ReportMetric(float64(shape.effectiveEntries), "effective-entries/op")
|
||||
})
|
||||
}
|
||||
|
||||
func inspectionExtraParams(entries int) map[string]any {
|
||||
providers := make(map[string]any, entries)
|
||||
for index := range entries {
|
||||
providers[fmt.Sprintf("provider-%03d", index)] = map[string]any{
|
||||
"enabled": true,
|
||||
"labels": []any{fmt.Sprintf("tier-%d", index%4), fmt.Sprintf("region-%d", index%3)},
|
||||
"limits": map[string]any{"burst": index + 1, "period": "minute"},
|
||||
"routes": []any{map[string]any{"weight": index + 1, "fallback": true}},
|
||||
}
|
||||
}
|
||||
return map[string]any{"provider_options": providers}
|
||||
}
|
||||
|
||||
func inspectionDefinitions(extraParams map[string]any) configurationShape {
|
||||
profiles := map[string]any{
|
||||
"small": map[string]any{"model": "inspection-model", "extra_params": extraParams},
|
||||
}
|
||||
prompt := benchmarkPrompt("small")
|
||||
prompt["inputs"] = []any{map[string]any{"name": "transcript", "required": true}}
|
||||
prompts := map[string]any{
|
||||
"inspection": map[string]any{"1": prompt},
|
||||
}
|
||||
return configurationShape{
|
||||
main: configkit.Input{Source: configkit.Mapping("inspection definitions", map[string]any{
|
||||
"profiles": profiles,
|
||||
"prompts": prompts,
|
||||
})},
|
||||
inputEntries: 2, effectiveEntries: 2, tiers: 1,
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkInspectionSelection(b *testing.B) {
|
||||
for _, scenario := range []struct {
|
||||
name string
|
||||
entries int
|
||||
}{
|
||||
{name: "small_extra", entries: 2},
|
||||
{name: "nested_extra", entries: 64},
|
||||
} {
|
||||
shape := inspectionDefinitions(inspectionExtraParams(scenario.entries))
|
||||
result, err := load(shape.main, nil)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
ctx := context.Background()
|
||||
definition, hash, err := result.GetPromptSnapshot(ctx, "inspection", "1")
|
||||
if err != nil || definition.ID != "inspection" || definition.Version != "1" || hash == "" ||
|
||||
len(definition.Inputs) != 1 || definition.Inputs[0].Name != "transcript" || !definition.Inputs[0].Required {
|
||||
b.Fatalf("prompt lookup fixture is invalid: definition=%+v hash=%q err=%v", definition, hash, err)
|
||||
}
|
||||
profile, err := result.GetProfile(ctx, "small")
|
||||
if err != nil || profile.ID != "small" || profile.ResolvedTarget.Model != "inspection-model" ||
|
||||
len(profile.ResolvedTarget.ExtraParams["provider_options"].(map[string]any)) != scenario.entries {
|
||||
b.Fatalf("profile lookup fixture is invalid: profile=%+v err=%v", profile, err)
|
||||
}
|
||||
inspection, err := result.GetPromptInspection(ctx, "inspection", "1")
|
||||
if err != nil || inspection.PromptID != "inspection" || inspection.PromptVersion != "1" ||
|
||||
len(inspection.Inputs) != 1 || inspection.Inputs[0].Name != "transcript" {
|
||||
b.Fatalf("prompt inspection fixture is invalid: inspection=%+v err=%v", inspection, err)
|
||||
}
|
||||
identity, err := result.GetProfileIdentity(ctx, "small")
|
||||
if err != nil || identity != "small" {
|
||||
b.Fatalf("profile identity fixture is invalid: identity=%q err=%v", identity, err)
|
||||
}
|
||||
|
||||
b.Run("prompt/"+scenario.name, func(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
for b.Loop() {
|
||||
definition, hash, err := result.GetPromptSnapshot(ctx, "inspection", "1")
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
selectedPrompt = definition
|
||||
selectedPromptHash = hash
|
||||
}
|
||||
b.ReportMetric(float64(scenario.entries), "extra-entries/op")
|
||||
})
|
||||
b.Run("prompt_inspection/"+scenario.name, func(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
for b.Loop() {
|
||||
inspection, err := result.GetPromptInspection(ctx, "inspection", "1")
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
selectedPromptInspection = inspection
|
||||
}
|
||||
b.ReportMetric(float64(scenario.entries), "extra-entries/op")
|
||||
})
|
||||
b.Run("profile/"+scenario.name, func(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
for b.Loop() {
|
||||
profile, err := result.GetProfile(ctx, "small")
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
selectedProfile = profile
|
||||
}
|
||||
b.ReportMetric(float64(scenario.entries), "extra-entries/op")
|
||||
})
|
||||
b.Run("profile_identity/"+scenario.name, func(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
for b.Loop() {
|
||||
identity, err := result.GetProfileIdentity(ctx, "small")
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
selectedProfileIdentity = identity
|
||||
}
|
||||
b.ReportMetric(float64(scenario.entries), "extra-entries/op")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,6 +16,15 @@ type Inspection interface {
|
||||
Explain(configkit.Path) (configkit.Explanation, bool)
|
||||
WriteEffective(io.Writer) error
|
||||
WriteExplanation(io.Writer, configkit.Path) error
|
||||
capturedResult() *configkit.Result[document]
|
||||
}
|
||||
|
||||
type inspectionResult struct {
|
||||
*configkit.Result[document]
|
||||
}
|
||||
|
||||
func (inspection inspectionResult) capturedResult() *configkit.Result[document] {
|
||||
return inspection.Result
|
||||
}
|
||||
|
||||
// Inspection returns only the retained result's inspection operations.
|
||||
@@ -23,7 +32,19 @@ func (state *loaded) Inspection() Inspection {
|
||||
if state == nil {
|
||||
return nil
|
||||
}
|
||||
return state.result
|
||||
return inspectionResult{Result: state.result}
|
||||
}
|
||||
|
||||
// Diff compares two captured ConfigKit results without reconstructing them.
|
||||
func Diff(left, right Inspection) ([]configkit.Difference, error) {
|
||||
var leftResult, rightResult *configkit.Result[document]
|
||||
if left != nil {
|
||||
leftResult = left.capturedResult()
|
||||
}
|
||||
if right != nil {
|
||||
rightResult = right.capturedResult()
|
||||
}
|
||||
return configkit.Diff(leftResult, rightResult)
|
||||
}
|
||||
|
||||
func secretProviderPath(path configkit.Path) bool {
|
||||
|
||||
29
internal/configuration/listing.go
Normal file
29
internal/configuration/listing.go
Normal file
@@ -0,0 +1,29 @@
|
||||
package configuration
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
)
|
||||
|
||||
func (state *loaded) ListProfiles(ctx context.Context) ([]string, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
profiles := append([]string(nil), state.profileIDs...)
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return profiles, nil
|
||||
}
|
||||
|
||||
func (state *loaded) ListPrompts(ctx context.Context) ([]domain.PromptIdentity, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
prompts := append([]domain.PromptIdentity(nil), state.promptIDs...)
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return prompts, nil
|
||||
}
|
||||
142
internal/configuration/listing_test.go
Normal file
142
internal/configuration/listing_test.go
Normal file
@@ -0,0 +1,142 @@
|
||||
package configuration
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
)
|
||||
|
||||
func TestLoadedListsSortedEffectiveIdentities(t *testing.T) {
|
||||
state, err := Load(mapping("identities", map[string]any{
|
||||
"backends": map[string]any{
|
||||
"local": map[string]any{"endpoint": "http://localhost:8080/v1"},
|
||||
},
|
||||
"profiles": map[string]any{
|
||||
"z/profile": map[string]any{"backend": "local", "model": "z-model"},
|
||||
"partial/profile": map[string]any{"backend": "local"},
|
||||
"a/profile": map[string]any{"backend": "local", "model": "a-model"},
|
||||
" z/profile ": map[string]any{"backend": "local", "model": "spaced-model"},
|
||||
},
|
||||
"prompts": map[string]any{
|
||||
"z/prompt": map[string]any{"v1": listingPromptDefinition()},
|
||||
"a/prompt": map[string]any{"v1": listingPromptDefinition()},
|
||||
" z/prompt ": map[string]any{
|
||||
"2": listingPromptDefinition(),
|
||||
"10": listingPromptDefinition(),
|
||||
},
|
||||
},
|
||||
}), nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
profiles, err := state.ListProfiles(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wantProfiles := []string{" z/profile ", "a/profile", "partial/profile", "z/profile"}
|
||||
if !reflect.DeepEqual(profiles, wantProfiles) {
|
||||
t.Fatalf("profile IDs = %#v, want %#v", profiles, wantProfiles)
|
||||
}
|
||||
|
||||
prompts, err := state.ListPrompts(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wantPrompts := []domain.PromptIdentity{
|
||||
{ID: " z/prompt ", Version: "10"},
|
||||
{ID: " z/prompt ", Version: "2"},
|
||||
{ID: "a/prompt", Version: "v1"},
|
||||
{ID: "z/prompt", Version: "v1"},
|
||||
}
|
||||
if !reflect.DeepEqual(prompts, wantPrompts) {
|
||||
t.Fatalf("prompt identities = %#v, want %#v", prompts, wantPrompts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadedListsAreIndependent(t *testing.T) {
|
||||
state, err := Load(mapping("identities", map[string]any{
|
||||
"profiles": map[string]any{"profile": map[string]any{}},
|
||||
"prompts": map[string]any{"example": map[string]any{"v1": listingPromptDefinition()}},
|
||||
}), nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
profiles, err := state.ListProfiles(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
profiles = append(profiles, "caller profile")
|
||||
profiles[0] = "changed profile"
|
||||
|
||||
prompts, err := state.ListPrompts(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
prompts[0].ID = "changed prompt"
|
||||
prompts[0].Version = "changed version"
|
||||
|
||||
profiles, err = state.ListProfiles(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(profiles, []string{"profile"}) {
|
||||
t.Fatalf("caller mutation changed profile inventory: %#v", profiles)
|
||||
}
|
||||
prompts, err = state.ListPrompts(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wantPrompts := []domain.PromptIdentity{{ID: "example", Version: "v1"}}
|
||||
if !reflect.DeepEqual(prompts, wantPrompts) {
|
||||
t.Fatalf("caller mutation changed prompt inventory: %#v", prompts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadedListsEmptyInventory(t *testing.T) {
|
||||
state, err := Load(configkit.Input{Source: configkit.NoSource()}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
profiles, err := state.ListProfiles(context.Background())
|
||||
if err != nil || len(profiles) != 0 {
|
||||
t.Fatalf("empty profile listing = %#v, %v", profiles, err)
|
||||
}
|
||||
prompts, err := state.ListPrompts(context.Background())
|
||||
if err != nil || len(prompts) != 0 {
|
||||
t.Fatalf("empty prompt listing = %#v, %v", prompts, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadedListsHonorCanceledContext(t *testing.T) {
|
||||
state, err := Load(mapping("identities", map[string]any{
|
||||
"profiles": map[string]any{"partial": map[string]any{}},
|
||||
"prompts": map[string]any{"example": map[string]any{"v1": listingPromptDefinition()}},
|
||||
}), nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
|
||||
profiles, err := state.ListProfiles(ctx)
|
||||
if !errors.Is(err, context.Canceled) || len(profiles) != 0 {
|
||||
t.Fatalf("canceled profile listing = %#v, %v", profiles, err)
|
||||
}
|
||||
prompts, err := state.ListPrompts(ctx)
|
||||
if !errors.Is(err, context.Canceled) || len(prompts) != 0 {
|
||||
t.Fatalf("canceled prompt listing = %#v, %v", prompts, err)
|
||||
}
|
||||
}
|
||||
|
||||
func listingPromptDefinition() map[string]any {
|
||||
return map[string]any{
|
||||
"messages": []any{map[string]any{"role": "user", "content": "Hello."}},
|
||||
"output": map[string]any{"format": "text", "validation_mode": "basic"},
|
||||
}
|
||||
}
|
||||
@@ -54,10 +54,10 @@ type promptConfig struct {
|
||||
}
|
||||
|
||||
type promptInputConfig struct {
|
||||
Name string `yaml:"name"`
|
||||
Required bool `yaml:"required"`
|
||||
ContentType string `yaml:"content_type"`
|
||||
Description string `yaml:"description"`
|
||||
Name string `yaml:"name"`
|
||||
Required bool `yaml:"required"`
|
||||
ContentType *string `yaml:"content_type"`
|
||||
Description *string `yaml:"description"`
|
||||
}
|
||||
|
||||
type promptMessageConfig struct {
|
||||
@@ -93,10 +93,13 @@ type loaded struct {
|
||||
result *configkit.Result[document]
|
||||
backends map[string]map[string]fieldState
|
||||
profiles map[string]map[string]fieldState
|
||||
promptMetadata map[string]map[string][]promptInputPresence
|
||||
prompts map[string]map[string]*promptEntry
|
||||
ambiguous map[string]bool
|
||||
registry *backend.Registry
|
||||
indexedProfiles map[string]domain.ExecutionProfile
|
||||
profileIDs []string
|
||||
promptIDs []domain.PromptIdentity
|
||||
}
|
||||
|
||||
var envName = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`)
|
||||
@@ -112,8 +115,9 @@ func Load(main configkit.Input, underlays []configkit.Underlay) (*loaded, error)
|
||||
// load performs one ConfigKit lifecycle over the selected source tiers.
|
||||
func load(main configkit.Input, underlays []configkit.Underlay) (*loaded, error) {
|
||||
state := &loaded{
|
||||
backends: make(map[string]map[string]fieldState),
|
||||
profiles: make(map[string]map[string]fieldState),
|
||||
backends: make(map[string]map[string]fieldState),
|
||||
profiles: make(map[string]map[string]fieldState),
|
||||
promptMetadata: make(map[string]map[string][]promptInputPresence),
|
||||
}
|
||||
result, err := configkit.LoadLayered(main, configkit.LayeredOptions[document]{
|
||||
Underlays: underlays,
|
||||
@@ -122,6 +126,7 @@ func load(main configkit.Input, underlays []configkit.Underlay) (*loaded, error)
|
||||
Resolve: func(config *document, resolution *configkit.Resolution) error {
|
||||
captureFields(config.Backends, "backends", backendFields, resolution, state.backends)
|
||||
captureFields(config.Profiles, "profiles", profileFields, resolution, state.profiles)
|
||||
capturePromptMetadata(config.Prompts, resolution, state.promptMetadata)
|
||||
resolveBackends(config.Backends, resolution)
|
||||
resolveProfiles(config.Profiles, resolution)
|
||||
resolvePrompts(config.Prompts, resolution)
|
||||
@@ -140,9 +145,53 @@ func load(main configkit.Input, underlays []configkit.Underlay) (*loaded, error)
|
||||
if err := state.buildExecutionProfiles(); err != nil {
|
||||
return nil, fmt.Errorf("load configuration: %w", err)
|
||||
}
|
||||
state.profileIDs = sortedKeys(state.indexedProfiles)
|
||||
for _, id := range sortedKeys(state.prompts) {
|
||||
for _, version := range sortedKeys(state.prompts[id]) {
|
||||
state.promptIDs = append(state.promptIDs, domain.PromptIdentity{ID: id, Version: version})
|
||||
}
|
||||
}
|
||||
return state, nil
|
||||
}
|
||||
|
||||
type promptInputPresence struct {
|
||||
contentType fieldState
|
||||
description fieldState
|
||||
}
|
||||
|
||||
func capturePromptMetadata(prompts map[string]map[string]promptConfig, resolution *configkit.Resolution, destination map[string]map[string][]promptInputPresence) {
|
||||
for _, id := range sortedKeys(prompts) {
|
||||
if destination[id] == nil {
|
||||
destination[id] = make(map[string][]promptInputPresence, len(prompts[id]))
|
||||
}
|
||||
for _, version := range sortedKeys(prompts[id]) {
|
||||
inputs := prompts[id][version].Inputs
|
||||
states := make([]promptInputPresence, len(inputs))
|
||||
base := configkit.Keys("prompts", id, version, "inputs")
|
||||
for index := range inputs {
|
||||
path := base.Index(index)
|
||||
states[index] = promptInputPresence{
|
||||
contentType: captureFieldState(path.Key("content_type"), resolution),
|
||||
description: captureFieldState(path.Key("description"), resolution),
|
||||
}
|
||||
}
|
||||
destination[id][version] = states
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func captureFieldState(path configkit.Path, resolution *configkit.Resolution) fieldState {
|
||||
info, exists := resolution.InputInfo(path)
|
||||
switch {
|
||||
case !exists || !info.Explicit:
|
||||
return fieldAbsent
|
||||
case info.Null:
|
||||
return fieldNull
|
||||
default:
|
||||
return fieldValue
|
||||
}
|
||||
}
|
||||
|
||||
func captureFields[T any](entries map[string]T, collection string, fields []string, resolution *configkit.Resolution, destination map[string]map[string]fieldState) {
|
||||
for id := range entries {
|
||||
states := make(map[string]fieldState, len(fields))
|
||||
|
||||
@@ -21,11 +21,12 @@ var (
|
||||
)
|
||||
|
||||
type promptEntry struct {
|
||||
definition domain.PromptDefinition
|
||||
inspection domain.PromptInspection
|
||||
hash string
|
||||
session *template.Template
|
||||
messages []*template.Template
|
||||
definition domain.PromptDefinition
|
||||
inspection domain.PromptInspection
|
||||
requirementInputs []prompt.InputMetadata
|
||||
hash string
|
||||
session *template.Template
|
||||
messages []*template.Template
|
||||
}
|
||||
|
||||
func resolvePrompts(prompts map[string]map[string]promptConfig, resolution *configkit.Resolution) {
|
||||
@@ -48,18 +49,24 @@ func resolvePrompts(prompts map[string]map[string]promptConfig, resolution *conf
|
||||
for index := range definition.Inputs {
|
||||
input := &definition.Inputs[index]
|
||||
inputPath := base.Key("inputs").Index(index)
|
||||
normalizedName := strings.TrimSpace(input.Name)
|
||||
if normalizedName != input.Name {
|
||||
configkit.Normalize(resolution, inputPath.Key("name"), &input.Name, normalizedName, "normalize input metadata")
|
||||
inputsChanged = true
|
||||
}
|
||||
for _, field := range []struct {
|
||||
name string
|
||||
value *string
|
||||
value **string
|
||||
}{
|
||||
{"name", &input.Name},
|
||||
{"content_type", &input.ContentType},
|
||||
{"description", &input.Description},
|
||||
} {
|
||||
normalized := strings.TrimSpace(*field.value)
|
||||
if normalized != *field.value {
|
||||
configkit.Normalize(resolution, inputPath.Key(field.name), field.value, normalized, "normalize input metadata")
|
||||
inputsChanged = true
|
||||
if *field.value != nil {
|
||||
normalized := strings.TrimSpace(**field.value)
|
||||
if normalized != **field.value {
|
||||
configkit.Normalize(resolution, inputPath.Key(field.name), field.value, &normalized, "normalize input metadata")
|
||||
inputsChanged = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -132,6 +139,10 @@ func (state *loaded) materializePrompt(id, version string, raw promptConfig, loo
|
||||
}
|
||||
seenInputs := make(map[string]bool, len(raw.Inputs))
|
||||
definition.Inputs = make([]domain.PromptInput, 0, len(raw.Inputs))
|
||||
entry := &promptEntry{
|
||||
messages: make([]*template.Template, 0, len(raw.Messages)),
|
||||
requirementInputs: make([]prompt.InputMetadata, 0, len(raw.Inputs)),
|
||||
}
|
||||
for index, input := range raw.Inputs {
|
||||
path := base.Key("inputs").Index(index)
|
||||
if input.Name == "" {
|
||||
@@ -141,11 +152,27 @@ func (state *loaded) materializePrompt(id, version string, raw promptConfig, loo
|
||||
return nil, fmt.Errorf("%s: input name is duplicated", path.Key("name"))
|
||||
}
|
||||
seenInputs[input.Name] = true
|
||||
contentType := ""
|
||||
if input.ContentType != nil {
|
||||
contentType = *input.ContentType
|
||||
}
|
||||
description := ""
|
||||
if input.Description != nil {
|
||||
description = *input.Description
|
||||
}
|
||||
definition.Inputs = append(definition.Inputs, domain.PromptInput{
|
||||
Name: input.Name, Required: input.Required, ContentType: input.ContentType, Description: input.Description,
|
||||
Name: input.Name, Required: input.Required, ContentType: contentType, Description: description,
|
||||
})
|
||||
presence := state.promptMetadata[id][version][index]
|
||||
metadata := prompt.InputMetadata{Name: input.Name, Required: input.Required}
|
||||
if presence.contentType == fieldValue {
|
||||
metadata.ContentType = stringPointer(contentType)
|
||||
}
|
||||
if presence.description == fieldValue {
|
||||
metadata.Description = stringPointer(description)
|
||||
}
|
||||
entry.requirementInputs = append(entry.requirementInputs, metadata)
|
||||
}
|
||||
entry := &promptEntry{messages: make([]*template.Template, 0, len(raw.Messages))}
|
||||
definition.Templates = make([]domain.PromptMessageTemplate, 0, len(raw.Messages))
|
||||
for index, message := range raw.Messages {
|
||||
path := base.Key("messages").Index(index)
|
||||
@@ -233,6 +260,11 @@ func (state *loaded) materializePrompt(id, version string, raw promptConfig, loo
|
||||
return entry, nil
|
||||
}
|
||||
|
||||
func stringPointer(value string) *string {
|
||||
copy := value
|
||||
return ©
|
||||
}
|
||||
|
||||
func (state *loaded) readContent(fieldPath configkit.Path, name string, lookupOrigin func(configkit.Path) (configkit.FieldInfo, bool)) (string, error) {
|
||||
if strings.TrimSpace(name) == "" {
|
||||
return "", fmt.Errorf("%s: content_file path must not be blank", fieldPath)
|
||||
@@ -265,14 +297,14 @@ func (state *loaded) readContent(fieldPath configkit.Path, name string, lookupOr
|
||||
return string(content), nil
|
||||
}
|
||||
|
||||
func (state *loaded) prompt(id, version string) (*domain.PromptDefinition, domain.PromptInspection, error) {
|
||||
func (state *loaded) lookupPromptEntry(id, version string) (*promptEntry, error) {
|
||||
if state == nil || strings.TrimSpace(id) == "" || (version != "" && strings.TrimSpace(version) == "") {
|
||||
return nil, domain.PromptInspection{}, errPromptNotFound
|
||||
return nil, errPromptNotFound
|
||||
}
|
||||
versions := state.prompts[id]
|
||||
if version == "" {
|
||||
if state.ambiguous[id] {
|
||||
return nil, domain.PromptInspection{}, errPromptAmbiguous
|
||||
return nil, errPromptAmbiguous
|
||||
}
|
||||
for selected := range versions {
|
||||
version = selected
|
||||
@@ -280,9 +312,12 @@ func (state *loaded) prompt(id, version string) (*domain.PromptDefinition, domai
|
||||
}
|
||||
entry := versions[version]
|
||||
if entry == nil {
|
||||
return nil, domain.PromptInspection{}, errPromptNotFound
|
||||
return nil, errPromptNotFound
|
||||
}
|
||||
definition := entry.definition
|
||||
return entry, nil
|
||||
}
|
||||
|
||||
func ownPromptDefinition(definition domain.PromptDefinition) *domain.PromptDefinition {
|
||||
definition.Inputs = append([]domain.PromptInput(nil), definition.Inputs...)
|
||||
definition.Templates = append([]domain.PromptMessageTemplate(nil), definition.Templates...)
|
||||
for index := range definition.Templates {
|
||||
@@ -291,9 +326,27 @@ func (state *loaded) prompt(id, version string) (*domain.PromptDefinition, domai
|
||||
definition.Templates[index].CacheControl = ©
|
||||
}
|
||||
}
|
||||
inspection := entry.inspection
|
||||
return &definition
|
||||
}
|
||||
|
||||
func ownPromptInspection(inspection domain.PromptInspection) *domain.PromptInspection {
|
||||
inspection.Inputs = append([]domain.PromptInput(nil), inspection.Inputs...)
|
||||
return &definition, inspection, nil
|
||||
return &inspection
|
||||
}
|
||||
|
||||
func promptLookupError(err error) error {
|
||||
if errors.Is(err, errPromptNotFound) {
|
||||
return promptdef.ErrPromptDefinitionNotFound
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (state *loaded) prompt(id, version string) (*domain.PromptDefinition, domain.PromptInspection, error) {
|
||||
entry, err := state.lookupPromptEntry(id, version)
|
||||
if err != nil {
|
||||
return nil, domain.PromptInspection{}, err
|
||||
}
|
||||
return ownPromptDefinition(entry.definition), *ownPromptInspection(entry.inspection), nil
|
||||
}
|
||||
|
||||
// GetPromptSnapshot returns an independently owned captured definition and its
|
||||
@@ -302,14 +355,24 @@ func (state *loaded) GetPromptSnapshot(ctx context.Context, id, version string)
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
definition, inspection, err := state.prompt(id, version)
|
||||
if errors.Is(err, errPromptNotFound) {
|
||||
return nil, "", promptdef.ErrPromptDefinitionNotFound
|
||||
}
|
||||
entry, err := state.lookupPromptEntry(id, version)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
return nil, "", promptLookupError(err)
|
||||
}
|
||||
return definition, inspection.PromptHash, nil
|
||||
return ownPromptDefinition(entry.definition), entry.hash, nil
|
||||
}
|
||||
|
||||
// GetPromptInspection returns an independently owned inspection of one
|
||||
// captured prompt without copying its execution templates.
|
||||
func (state *loaded) GetPromptInspection(ctx context.Context, id, version string) (*domain.PromptInspection, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
entry, err := state.lookupPromptEntry(id, version)
|
||||
if err != nil {
|
||||
return nil, promptLookupError(err)
|
||||
}
|
||||
return ownPromptInspection(entry.inspection), nil
|
||||
}
|
||||
|
||||
// GetPromptDefinition implements the runner's prompt lookup boundary.
|
||||
@@ -329,3 +392,33 @@ func (state *loaded) GetCompiledTemplates(id, version string) (prompt.CompiledTe
|
||||
}
|
||||
return prompt.CompiledTemplates{Session: entry.session, Messages: entry.messages}, true
|
||||
}
|
||||
|
||||
// GetRequirementSnapshot returns captured templates and input metadata for
|
||||
// one exact prompt identity. Returned metadata and slices are independently
|
||||
// owned; the compiled templates are shared immutable construction-time parses.
|
||||
func (state *loaded) GetRequirementSnapshot(id, version string) (prompt.RequirementSnapshot, bool) {
|
||||
if state == nil {
|
||||
return prompt.RequirementSnapshot{}, false
|
||||
}
|
||||
entry := state.prompts[id][version]
|
||||
if entry == nil {
|
||||
return prompt.RequirementSnapshot{}, false
|
||||
}
|
||||
snapshot := prompt.RequirementSnapshot{
|
||||
Templates: prompt.CompiledTemplates{
|
||||
Session: entry.session,
|
||||
Messages: append([]*template.Template(nil), entry.messages...),
|
||||
},
|
||||
Inputs: make([]prompt.InputMetadata, len(entry.requirementInputs)),
|
||||
}
|
||||
for index, input := range entry.requirementInputs {
|
||||
snapshot.Inputs[index] = input
|
||||
if input.ContentType != nil {
|
||||
snapshot.Inputs[index].ContentType = stringPointer(*input.ContentType)
|
||||
}
|
||||
if input.Description != nil {
|
||||
snapshot.Inputs[index].Description = stringPointer(*input.Description)
|
||||
}
|
||||
}
|
||||
return snapshot, true
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package configuration
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io/fs"
|
||||
"os"
|
||||
@@ -10,6 +11,7 @@ import (
|
||||
"testing/fstest"
|
||||
|
||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/promptdef"
|
||||
)
|
||||
|
||||
func TestLoadCapturesInheritedImportedContent(t *testing.T) {
|
||||
@@ -134,6 +136,94 @@ func TestLoadRejectsMalformedStaticSyntaxAndAmbiguousSelection(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestPromptInspectionLookupMatchesSnapshotSelectionAndOwnsInputs(t *testing.T) {
|
||||
makePrompt := func(content string) map[string]any {
|
||||
return map[string]any{
|
||||
"default_profile": "partial",
|
||||
"inputs": []any{map[string]any{
|
||||
"name": "document", "required": true,
|
||||
"content_type": " text/plain ", "description": " source document ",
|
||||
}},
|
||||
"messages": []any{map[string]any{"role": "user", "content": content}},
|
||||
"output": map[string]any{"format": "text", "validation_mode": "none"},
|
||||
}
|
||||
}
|
||||
state, err := load(mapping("prompt inspection lookup", map[string]any{
|
||||
"profiles": map[string]any{"partial": map[string]any{}},
|
||||
"prompts": map[string]any{
|
||||
"multi": map[string]any{"v1": makePrompt("first"), "v2": makePrompt("second")},
|
||||
"single": map[string]any{"v1": makePrompt("only")},
|
||||
},
|
||||
}), nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
for _, selection := range []struct {
|
||||
id, version string
|
||||
}{
|
||||
{id: "multi", version: "v2"},
|
||||
{id: "single"},
|
||||
} {
|
||||
inspection, err := state.GetPromptInspection(context.Background(), selection.id, selection.version)
|
||||
if err != nil {
|
||||
t.Fatalf("inspect %q/%q: %v", selection.id, selection.version, err)
|
||||
}
|
||||
definition, hash, err := state.GetPromptSnapshot(context.Background(), selection.id, selection.version)
|
||||
if err != nil {
|
||||
t.Fatalf("snapshot %q/%q: %v", selection.id, selection.version, err)
|
||||
}
|
||||
if inspection.PromptID != definition.ID || inspection.PromptVersion != definition.Version ||
|
||||
inspection.PromptHash != hash || inspection.DefaultProfileID != "partial" ||
|
||||
len(inspection.Inputs) != 1 || inspection.Inputs[0].Name != "document" ||
|
||||
inspection.Inputs[0].ContentType != "text/plain" || inspection.Inputs[0].Description != "source document" {
|
||||
t.Fatalf("inspection did not match the selected snapshot: inspection=%+v definition=%+v hash=%q", inspection, definition, hash)
|
||||
}
|
||||
if len(definition.Inputs) != len(inspection.Inputs) || definition.Inputs[0].Name != inspection.Inputs[0].Name {
|
||||
t.Fatalf("definition and inspection inputs diverged: definition=%+v inspection=%+v", definition.Inputs, inspection.Inputs)
|
||||
}
|
||||
}
|
||||
|
||||
owned, err := state.GetPromptInspection(context.Background(), "single", "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
owned.Inputs[0].Name = "caller mutation"
|
||||
owned.Inputs[0].Description = "caller mutation"
|
||||
again, err := state.GetPromptInspection(context.Background(), "single", "v1")
|
||||
if err != nil || again.Inputs[0].Name != "document" || again.Inputs[0].Description != "source document" {
|
||||
t.Fatalf("inspection mutation escaped into captured state: inspection=%+v err=%v", again, err)
|
||||
}
|
||||
|
||||
for _, selection := range []struct {
|
||||
id, version string
|
||||
}{
|
||||
{id: "missing", version: "v1"},
|
||||
{id: "multi", version: "V2"},
|
||||
{id: " ", version: "v1"},
|
||||
} {
|
||||
inspection, err := state.GetPromptInspection(context.Background(), selection.id, selection.version)
|
||||
if inspection != nil || !errors.Is(err, promptdef.ErrPromptDefinitionNotFound) {
|
||||
t.Errorf("inspection selection %q/%q=(%+v, %v), want not found", selection.id, selection.version, inspection, err)
|
||||
}
|
||||
definition, _, err := state.GetPromptSnapshot(context.Background(), selection.id, selection.version)
|
||||
if definition != nil || !errors.Is(err, promptdef.ErrPromptDefinitionNotFound) {
|
||||
t.Errorf("snapshot selection %q/%q=(%+v, %v), want not found", selection.id, selection.version, definition, err)
|
||||
}
|
||||
}
|
||||
if _, err := state.GetPromptInspection(context.Background(), "multi", ""); !errors.Is(err, errPromptAmbiguous) {
|
||||
t.Errorf("ambiguous inspection selection error=%v", err)
|
||||
}
|
||||
if _, _, err := state.GetPromptSnapshot(context.Background(), "multi", ""); !errors.Is(err, errPromptAmbiguous) {
|
||||
t.Errorf("ambiguous snapshot selection error=%v", err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
if _, err := state.GetPromptInspection(ctx, "single", "v1"); !errors.Is(err, context.Canceled) {
|
||||
t.Errorf("canceled inspection selection error=%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadAcceptsEmptyInlineContentAndRejectsInvalidPromptFields(t *testing.T) {
|
||||
state, err := load(mapping("empty-content", map[string]any{"prompts": map[string]any{" exact.id ": map[string]any{"v1": map[string]any{
|
||||
"inputs": []any{map[string]any{"name": " note ", "required": true}},
|
||||
@@ -170,6 +260,113 @@ func TestLoadAcceptsEmptyInlineContentAndRejectsInvalidPromptFields(t *testing.T
|
||||
}
|
||||
}
|
||||
|
||||
func TestRequirementSnapshotCapturesMetadataPresenceAndNormalizedValues(t *testing.T) {
|
||||
state, err := load(mapping("requirement-metadata", map[string]any{
|
||||
"prompts": map[string]any{"example": map[string]any{"v1": map[string]any{
|
||||
"session_id": "{{.session}}",
|
||||
"inputs": []any{
|
||||
map[string]any{"name": "absent"},
|
||||
map[string]any{"name": "null", "content_type": nil, "description": nil},
|
||||
map[string]any{"name": "empty", "content_type": "", "description": ""},
|
||||
map[string]any{"name": "normalized", "content_type": " text/plain ", "description": " final text "},
|
||||
},
|
||||
"messages": []any{map[string]any{"role": "user", "content": "{{input \"absent\"}}"}},
|
||||
"output": map[string]any{"format": "text", "validation_mode": "basic"},
|
||||
}}},
|
||||
}), nil)
|
||||
if err != nil {
|
||||
var loadError *configkit.LoadError
|
||||
if errors.As(err, &loadError) {
|
||||
t.Fatalf("%v: %+v", err, loadError.Diagnostics())
|
||||
}
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
snapshot, ok := state.GetRequirementSnapshot("example", "v1")
|
||||
if !ok || len(snapshot.Inputs) != 4 || len(snapshot.Templates.Messages) != 1 || snapshot.Templates.Session == nil {
|
||||
t.Fatalf("requirement snapshot was not captured: %+v, %t", snapshot, ok)
|
||||
}
|
||||
for _, index := range []int{0, 1} {
|
||||
if snapshot.Inputs[index].ContentType != nil || snapshot.Inputs[index].Description != nil {
|
||||
t.Errorf("input %q should not have explicit string metadata: %+v", snapshot.Inputs[index].Name, snapshot.Inputs[index])
|
||||
}
|
||||
}
|
||||
if snapshot.Inputs[2].ContentType == nil || *snapshot.Inputs[2].ContentType != "" ||
|
||||
snapshot.Inputs[2].Description == nil || *snapshot.Inputs[2].Description != "" {
|
||||
t.Errorf("explicit empty metadata presence was lost: %+v", snapshot.Inputs[2])
|
||||
}
|
||||
if snapshot.Inputs[3].ContentType == nil || *snapshot.Inputs[3].ContentType != "text/plain" ||
|
||||
snapshot.Inputs[3].Description == nil || *snapshot.Inputs[3].Description != "final text" {
|
||||
t.Errorf("snapshot did not use normalized final metadata: %+v", snapshot.Inputs[3])
|
||||
}
|
||||
|
||||
definition, inspection, err := state.prompt("example", "v1")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(definition.Inputs) != 4 || len(inspection.Inputs) != 4 || inspection.Inputs[2].ContentType != "" || inspection.Inputs[2].Description != "" ||
|
||||
inspection.Inputs[3].ContentType != "text/plain" || inspection.Inputs[3].Description != "final text" {
|
||||
t.Fatalf("existing prompt inspection shape or normalized values changed: %+v", inspection)
|
||||
}
|
||||
|
||||
*snapshot.Inputs[2].ContentType = "caller mutation"
|
||||
snapshot.Inputs[3].Name = "caller mutation"
|
||||
snapshot.Templates.Messages[0] = nil
|
||||
again, ok := state.GetRequirementSnapshot("example", "v1")
|
||||
if !ok || *again.Inputs[2].ContentType != "" || again.Inputs[3].Name != "normalized" || again.Templates.Messages[0] == nil {
|
||||
t.Fatalf("snapshot mutation escaped into captured state: %+v, %t", again, ok)
|
||||
}
|
||||
if _, ok := state.GetRequirementSnapshot("example", "V1"); ok {
|
||||
t.Fatal("snapshot lookup unexpectedly changed exact version matching")
|
||||
}
|
||||
}
|
||||
|
||||
func TestInputMetadataPresenceDoesNotChangePromptHashOrInspection(t *testing.T) {
|
||||
variants := []struct {
|
||||
name string
|
||||
fields map[string]any
|
||||
}{
|
||||
{name: "absent"},
|
||||
{name: "null", fields: map[string]any{"content_type": nil, "description": nil}},
|
||||
{name: "empty", fields: map[string]any{"content_type": "", "description": ""}},
|
||||
}
|
||||
var expectedHash string
|
||||
for _, variant := range variants {
|
||||
t.Run(variant.name, func(t *testing.T) {
|
||||
input := map[string]any{"name": "document"}
|
||||
for name, value := range variant.fields {
|
||||
input[name] = value
|
||||
}
|
||||
state, err := load(mapping("hash-"+variant.name, map[string]any{
|
||||
"prompts": map[string]any{"same": map[string]any{"v1": map[string]any{
|
||||
"inputs": []any{input},
|
||||
"messages": []any{map[string]any{"role": "user", "content": "hello"}},
|
||||
"output": map[string]any{"format": "text", "validation_mode": "basic"},
|
||||
}}},
|
||||
}), nil)
|
||||
if err != nil {
|
||||
var loadError *configkit.LoadError
|
||||
if errors.As(err, &loadError) {
|
||||
t.Fatalf("%v: %+v", err, loadError.Diagnostics())
|
||||
}
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, inspection, err := state.prompt("same", "v1")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(inspection.Inputs) != 1 || inspection.Inputs[0].ContentType != "" || inspection.Inputs[0].Description != "" {
|
||||
t.Fatalf("old inspection metadata changed: %+v", inspection.Inputs)
|
||||
}
|
||||
if expectedHash == "" {
|
||||
expectedHash = inspection.PromptHash
|
||||
} else if inspection.PromptHash != expectedHash {
|
||||
t.Fatalf("metadata presence changed the definition hash: got %q, want %q", inspection.PromptHash, expectedHash)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func promptWithFile(name string) map[string]any {
|
||||
return map[string]any{"prompts": map[string]any{"example": map[string]any{"v1": map[string]any{
|
||||
"messages": []any{map[string]any{"role": "user", "content_file": name}},
|
||||
|
||||
@@ -147,18 +147,26 @@ func overlayProviderMap(lower, upper map[string]any) (map[string]any, error) {
|
||||
return merged, nil
|
||||
}
|
||||
|
||||
// GetProfile returns an independently owned indexed profile for the internal
|
||||
// runner's lookup boundary.
|
||||
func (state *loaded) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) {
|
||||
func (state *loaded) lookupProfile(ctx context.Context, id string) (domain.ExecutionProfile, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
return domain.ExecutionProfile{}, err
|
||||
}
|
||||
if state == nil {
|
||||
return nil, fmt.Errorf("%w: %q", profilepkg.ErrProfileNotFound, id)
|
||||
return domain.ExecutionProfile{}, fmt.Errorf("%w: %q", profilepkg.ErrProfileNotFound, id)
|
||||
}
|
||||
profile, found := state.indexedProfiles[id]
|
||||
if !found {
|
||||
return nil, fmt.Errorf("%w: %q", profilepkg.ErrProfileNotFound, id)
|
||||
return domain.ExecutionProfile{}, fmt.Errorf("%w: %q", profilepkg.ErrProfileNotFound, id)
|
||||
}
|
||||
return profile, nil
|
||||
}
|
||||
|
||||
// GetProfile returns an independently owned indexed profile for the internal
|
||||
// runner's lookup boundary.
|
||||
func (state *loaded) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) {
|
||||
profile, err := state.lookupProfile(ctx, id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
target := *profile.ResolvedTarget
|
||||
params, err := jsonvalue.CopyMap(target.ExtraParams)
|
||||
@@ -170,6 +178,15 @@ func (state *loaded) GetProfile(ctx context.Context, id string) (*domain.Executi
|
||||
return &profile, nil
|
||||
}
|
||||
|
||||
// GetProfileIdentity returns the exact lookup ID for a captured profile
|
||||
// without copying its execution target.
|
||||
func (state *loaded) GetProfileIdentity(ctx context.Context, id string) (string, error) {
|
||||
if _, err := state.lookupProfile(ctx, id); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return id, nil
|
||||
}
|
||||
|
||||
// GetBackend returns an independently owned effective backend definition.
|
||||
func (state *loaded) GetBackend(id string) (domain.Backend, error) {
|
||||
if state == nil {
|
||||
|
||||
@@ -3,6 +3,7 @@ package configuration
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"reflect"
|
||||
@@ -11,6 +12,7 @@ import (
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/defaults"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/llm"
|
||||
profilepkg "gitea.maximumdirect.net/eric/promptkit/internal/profile"
|
||||
)
|
||||
|
||||
func TestLoadBuildsEffectiveTargetsAfterInheritance(t *testing.T) {
|
||||
@@ -89,6 +91,18 @@ func TestLoadBuildsEffectiveTargetsAfterInheritance(t *testing.T) {
|
||||
if partial.ResolvedTarget.Endpoint != "" || partial.ResolvedTarget.Model != "" {
|
||||
t.Fatalf("partial profile gained a runnable target: %+v", partial.ResolvedTarget)
|
||||
}
|
||||
identity, err := state.GetProfileIdentity(context.Background(), "partial")
|
||||
if err != nil || identity != "partial" {
|
||||
t.Fatalf("partial profile identity=(%q, %v)", identity, err)
|
||||
}
|
||||
if _, err := state.GetProfileIdentity(context.Background(), "missing"); !errors.Is(err, profilepkg.ErrProfileNotFound) {
|
||||
t.Fatalf("missing profile identity error=%v", err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
if _, err := state.GetProfileIdentity(ctx, "partial"); !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("canceled profile identity lookup error=%v", err)
|
||||
}
|
||||
endpointOnly, _ := state.GetProfile(context.Background(), "endpoint-only")
|
||||
if endpointOnly.ResolvedTarget.Endpoint != "https://standalone.example/v1" ||
|
||||
endpointOnly.ResolvedTarget.BackendID != "" || endpointOnly.ResolvedTarget.Model != "" {
|
||||
|
||||
@@ -135,6 +135,12 @@ type Artifact struct {
|
||||
Hash string
|
||||
}
|
||||
|
||||
// PromptIdentity identifies one configured prompt version.
|
||||
type PromptIdentity struct {
|
||||
ID string
|
||||
Version string
|
||||
}
|
||||
|
||||
// PromptDefinition represents a configured prompt execution definition.
|
||||
type PromptDefinition struct {
|
||||
ID string `yaml:"id"`
|
||||
|
||||
58
internal/domain/requirements.go
Normal file
58
internal/domain/requirements.go
Normal file
@@ -0,0 +1,58 @@
|
||||
package domain
|
||||
|
||||
// RequirementKind classifies how a prompt may depend on an input or variable.
|
||||
type RequirementKind string
|
||||
|
||||
const (
|
||||
RequirementRequired RequirementKind = "required"
|
||||
RequirementConditional RequirementKind = "conditional"
|
||||
RequirementOptional RequirementKind = "optional"
|
||||
)
|
||||
|
||||
// InputRequirement describes one declared or template-referenced input.
|
||||
type InputRequirement struct {
|
||||
Name string
|
||||
Kind RequirementKind
|
||||
Declared bool
|
||||
ContentType *string
|
||||
Description *string
|
||||
Explanation string
|
||||
}
|
||||
|
||||
// VariableRequirement describes one caller variable referenced by a prompt.
|
||||
type VariableRequirement struct {
|
||||
Name string
|
||||
Kind RequirementKind
|
||||
Supplied bool
|
||||
Explanation string
|
||||
}
|
||||
|
||||
// RequirementIssueCode identifies a limit that may affect requirement
|
||||
// discovery.
|
||||
type RequirementIssueCode string
|
||||
|
||||
const (
|
||||
RequirementIssueMissingVariable RequirementIssueCode = "missing_variable"
|
||||
RequirementIssueInputContent RequirementIssueCode = "input_content"
|
||||
RequirementIssueDynamicInputName RequirementIssueCode = "dynamic_input_name"
|
||||
RequirementIssueUnsupportedConstruct RequirementIssueCode = "unsupported_construct"
|
||||
)
|
||||
|
||||
// RequirementIssue describes one limitation on requirement discovery.
|
||||
type RequirementIssue struct {
|
||||
Code RequirementIssueCode
|
||||
InputName string
|
||||
VariableName string
|
||||
Description string
|
||||
}
|
||||
|
||||
// RequirementInspection contains the inventories discovered for one prompt.
|
||||
type RequirementInspection struct {
|
||||
PromptID string
|
||||
PromptVersion string
|
||||
ProfileID string
|
||||
Inputs []InputRequirement
|
||||
Variables []VariableRequirement
|
||||
Complete bool
|
||||
Issues []RequirementIssue
|
||||
}
|
||||
@@ -83,6 +83,12 @@ func (o *Options) SetSchemaFile(path string) error {
|
||||
type Engine struct {
|
||||
runner *usecase.Runner
|
||||
inspection configuration.Inspection
|
||||
listing identityInventory
|
||||
}
|
||||
|
||||
type identityInventory interface {
|
||||
ListProfiles(context.Context) ([]string, error)
|
||||
ListPrompts(context.Context) ([]domain.PromptIdentity, error)
|
||||
}
|
||||
|
||||
// New assembles an engine from facade-selected configuration and collaborators.
|
||||
@@ -135,6 +141,7 @@ func New(cfg Config, options Options) (*Engine, error) {
|
||||
return &Engine{
|
||||
runner: usecase.NewRunnerWithRepairer(snapshot, snapshot, reader, prompt.NewCompiledRenderer(snapshot), client, validator, usecase.NewDefaultOutputRepairer(client), manager, defaultRetries),
|
||||
inspection: snapshot.Inspection(),
|
||||
listing: snapshot,
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -178,9 +185,23 @@ func fileSource(name string) (fs.FS, string, string, error) {
|
||||
func (e *Engine) InspectPrompt(ctx context.Context, id, version string) (*domain.PromptInspection, error) {
|
||||
return e.runner.InspectPrompt(ctx, id, version)
|
||||
}
|
||||
func (e *Engine) InspectRequirements(
|
||||
ctx context.Context,
|
||||
request usecase.RequirementInspectionRequest,
|
||||
) (*domain.RequirementInspection, error) {
|
||||
return e.runner.InspectRequirements(ctx, request)
|
||||
}
|
||||
func (e *Engine) InspectProfile(ctx context.Context, id string) (*domain.ProfileInspection, error) {
|
||||
return e.runner.InspectProfile(ctx, id)
|
||||
}
|
||||
|
||||
func (e *Engine) ListProfiles(ctx context.Context) ([]string, error) {
|
||||
return e.listing.ListProfiles(ctx)
|
||||
}
|
||||
|
||||
func (e *Engine) ListPrompts(ctx context.Context) ([]domain.PromptIdentity, error) {
|
||||
return e.listing.ListPrompts(ctx)
|
||||
}
|
||||
func (e *Engine) Prepare(ctx context.Context, request domain.RunRequest) (*domain.PreparedRun, error) {
|
||||
return e.runner.Prepare(ctx, request)
|
||||
}
|
||||
|
||||
@@ -152,6 +152,11 @@ func (c *OpenAICompatibleClient) PrepareGeneration(req domain.GenerateRequest) (
|
||||
|
||||
payload, err := json.Marshal(wirePayload)
|
||||
if err != nil {
|
||||
for key, value := range req.Target.ExtraParams {
|
||||
if _, extraErr := json.Marshal(value); extraErr != nil {
|
||||
return nil, fmt.Errorf("%w: extra_params.%s must be JSON-serializable: %v", ErrInvalidRequest, key, extraErr)
|
||||
}
|
||||
}
|
||||
return nil, fmt.Errorf("%w: failed to encode request: %v", ErrRequestFailed, err)
|
||||
}
|
||||
apiKey := strings.TrimSpace(req.Target.APIKey)
|
||||
@@ -446,9 +451,6 @@ func openAIChatRequestPayload(req openAIChatRequest) (map[string]any, error) {
|
||||
if IsReservedOpenAIChatRequestField(key) {
|
||||
return nil, fmt.Errorf("extra_params key %q collides with reserved request field", key)
|
||||
}
|
||||
if _, err := json.Marshal(value); err != nil {
|
||||
return nil, fmt.Errorf("extra_params.%s must be JSON-serializable: %w", key, err)
|
||||
}
|
||||
out[key] = value
|
||||
}
|
||||
|
||||
|
||||
@@ -1113,6 +1113,13 @@ func TestOpenAICompatibleClientRejectsInvalidExtraParamsBeforeProviderCall(t *te
|
||||
}{
|
||||
{name: "empty key", extraParams: map[string]any{"": "empty"}, want: "key must not be empty"},
|
||||
{name: "unserializable value", extraParams: map[string]any{"bad": math.Inf(1)}, want: "JSON-serializable"},
|
||||
{
|
||||
name: "nested unserializable value",
|
||||
extraParams: map[string]any{
|
||||
"nested": map[string]any{"values": []any{"valid", math.NaN()}},
|
||||
},
|
||||
want: "extra_params.nested",
|
||||
},
|
||||
}
|
||||
for _, key := range []string{
|
||||
"model",
|
||||
@@ -1161,6 +1168,56 @@ func TestOpenAICompatibleClientRejectsInvalidExtraParamsBeforeProviderCall(t *te
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAICompatibleClientClassifiesSchemaEncodingFailures(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
extraParams map[string]any
|
||||
want error
|
||||
unwanted error
|
||||
}{
|
||||
{
|
||||
name: "serializable extra keeps full-request failure",
|
||||
extraParams: map[string]any{"client_option": "accepted"},
|
||||
want: ErrRequestFailed,
|
||||
unwanted: ErrInvalidRequest,
|
||||
},
|
||||
{
|
||||
name: "invalid extra keeps invalid-request failure",
|
||||
extraParams: map[string]any{"nested": map[string]any{"bad": math.Inf(1)}},
|
||||
want: ErrInvalidRequest,
|
||||
unwanted: ErrRequestFailed,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
provider := newRecordingProvider(t)
|
||||
client := newProviderClient(t, provider, OpenAICompatibleConfig{})
|
||||
request := ordinaryGenerateRequest()
|
||||
request.Target.ExtraParams = tc.extraParams
|
||||
request.StructuredOutput = &domain.StructuredOutputSpec{
|
||||
Type: domain.StructuredOutputJSONSchema,
|
||||
JSONSchema: &domain.StructuredOutputJSONSpec{
|
||||
Name: "invalid-schema",
|
||||
Strict: true,
|
||||
Schema: json.RawMessage("{"),
|
||||
},
|
||||
}
|
||||
|
||||
_, err := client.Generate(context.Background(), request)
|
||||
if err == nil || !errors.Is(err, tc.want) {
|
||||
t.Fatalf("error = %v, want identity %v", err, tc.want)
|
||||
}
|
||||
if errors.Is(err, tc.unwanted) {
|
||||
t.Fatalf("error = %v, unexpectedly has identity %v", err, tc.unwanted)
|
||||
}
|
||||
if calls := provider.requestCount(); calls != 0 {
|
||||
t.Fatalf("provider calls = %d, want 0", calls)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func checkConfiguredModelFallback(t *testing.T) {
|
||||
provider := newRecordingProvider(t)
|
||||
client := newProviderClient(t, provider, OpenAICompatibleConfig{Model: "default-model"})
|
||||
|
||||
119
internal/llm/prepare_generation_benchmark_test.go
Normal file
119
internal/llm/prepare_generation_benchmark_test.go
Normal file
@@ -0,0 +1,119 @@
|
||||
package llm
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
)
|
||||
|
||||
var prepareGenerationBenchmarkSink *PreparedGeneration
|
||||
|
||||
func BenchmarkPrepareGeneration(b *testing.B) {
|
||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||
BaseURL: "https://api.example.test/v1",
|
||||
Model: "benchmark-model",
|
||||
})
|
||||
if err != nil {
|
||||
b.Fatalf("construct benchmark client: %v", err)
|
||||
}
|
||||
|
||||
prompt := domain.RenderedPrompt{
|
||||
Messages: []domain.RenderedMessage{{Role: "user", Content: "Summarize this request."}},
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
request domain.GenerateRequest
|
||||
}{
|
||||
{
|
||||
name: "no_extras",
|
||||
request: domain.GenerateRequest{
|
||||
Prompt: prompt,
|
||||
Target: domain.ExecutionTarget{Model: "benchmark-model"},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "small_extras",
|
||||
request: domain.GenerateRequest{
|
||||
Prompt: prompt,
|
||||
Target: domain.ExecutionTarget{Model: "benchmark-model", ExtraParams: map[string]any{
|
||||
"priority": "low",
|
||||
"attempt": 3,
|
||||
"enabled": true,
|
||||
}},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "1MiB_string_extra",
|
||||
request: domain.GenerateRequest{
|
||||
Prompt: prompt,
|
||||
Target: domain.ExecutionTarget{Model: "benchmark-model", ExtraParams: map[string]any{
|
||||
"payload": strings.Repeat("x", 1<<20),
|
||||
}},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "nested_json_extras",
|
||||
request: domain.GenerateRequest{
|
||||
Prompt: prompt,
|
||||
Target: domain.ExecutionTarget{Model: "benchmark-model", ExtraParams: map[string]any{
|
||||
"options": map[string]any{
|
||||
"enabled": true,
|
||||
"limits": []any{2, 4, 8},
|
||||
"tags": []string{"summary", "offline"},
|
||||
},
|
||||
}},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "structured_output",
|
||||
request: domain.GenerateRequest{
|
||||
Prompt: prompt,
|
||||
Target: domain.ExecutionTarget{Model: "benchmark-model"},
|
||||
StructuredOutput: &domain.StructuredOutputSpec{
|
||||
Type: domain.StructuredOutputJSONSchema,
|
||||
JSONSchema: &domain.StructuredOutputJSONSpec{
|
||||
Name: "summary",
|
||||
Strict: true,
|
||||
Schema: map[string]any{
|
||||
"type": "object",
|
||||
"properties": map[string]any{
|
||||
"summary": map[string]any{"type": "string"},
|
||||
"count": map[string]any{"type": "integer"},
|
||||
},
|
||||
"required": []string{"summary", "count"},
|
||||
"additionalProperties": false,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
b.Run(tc.name, func(b *testing.B) {
|
||||
prepared, err := client.PrepareGeneration(tc.request)
|
||||
if err != nil {
|
||||
b.Fatalf("check prepared request: %v", err)
|
||||
}
|
||||
if !json.Valid(prepared.payload) {
|
||||
b.Fatal("prepared request is not valid JSON")
|
||||
}
|
||||
|
||||
b.ReportAllocs()
|
||||
b.SetBytes(int64(len(prepared.payload)))
|
||||
b.ResetTimer()
|
||||
for range b.N {
|
||||
prepared, err := client.PrepareGeneration(tc.request)
|
||||
if err != nil {
|
||||
b.Fatalf("prepare generation: %v", err)
|
||||
}
|
||||
prepareGenerationBenchmarkSink = prepared
|
||||
}
|
||||
b.StopTimer()
|
||||
runtime.KeepAlive(prepareGenerationBenchmarkSink)
|
||||
})
|
||||
}
|
||||
}
|
||||
254
internal/prompt/compiled_renderer_benchmark_test.go
Normal file
254
internal/prompt/compiled_renderer_benchmark_test.go
Normal file
@@ -0,0 +1,254 @@
|
||||
package prompt
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"testing"
|
||||
"text/template"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
)
|
||||
|
||||
const rendererBenchmarkArtifactSize = 256 << 10
|
||||
|
||||
var (
|
||||
renderedPromptBenchmarkSink *domain.RenderedPrompt
|
||||
renderedPromptsBenchmarkSink []*domain.RenderedPrompt
|
||||
)
|
||||
|
||||
type rendererBenchmarkIdentity struct {
|
||||
id string
|
||||
version string
|
||||
}
|
||||
|
||||
type rendererBenchmarkLookup struct {
|
||||
identity rendererBenchmarkIdentity
|
||||
templates CompiledTemplates
|
||||
}
|
||||
|
||||
func (lookup rendererBenchmarkLookup) GetCompiledTemplates(id, version string) (CompiledTemplates, bool) {
|
||||
if id != lookup.identity.id || version != lookup.identity.version {
|
||||
return CompiledTemplates{}, false
|
||||
}
|
||||
return lookup.templates, true
|
||||
}
|
||||
|
||||
type compiledRenderingBenchmarkCase struct {
|
||||
name string
|
||||
renderer Renderer
|
||||
definition *domain.PromptDefinition
|
||||
inputs map[string]*domain.Artifact
|
||||
vars map[string]string
|
||||
ctx context.Context
|
||||
outputBytes int
|
||||
check func(*testing.B, *domain.RenderedPrompt)
|
||||
}
|
||||
|
||||
func BenchmarkCompiledRendering(b *testing.B) {
|
||||
body := bytes.Repeat([]byte("x"), rendererBenchmarkArtifactSize)
|
||||
bodyText := string(body)
|
||||
|
||||
small := newCompiledRenderingBenchmarkCase(b,
|
||||
"small_message", "", []string{"Summary for {{.audience}}."},
|
||||
&domain.PromptDefinition{
|
||||
ID: "compiled-render-benchmark", Version: "v1",
|
||||
Templates: []domain.PromptMessageTemplate{{Role: "user", Content: "Summary for {{.audience}}."}},
|
||||
}, nil, map[string]string{"audience": "project leads"},
|
||||
len("Summary for project leads."),
|
||||
func(b *testing.B, rendered *domain.RenderedPrompt) {
|
||||
if rendered.SessionID != "" || len(rendered.Messages) != 1 || rendered.Messages[0].Content != "Summary for project leads." {
|
||||
b.Fatalf("unexpected small compiled render: %+v", rendered)
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
artifactInputs := map[string]*domain.Artifact{
|
||||
"document": {Name: "document", Body: body},
|
||||
}
|
||||
largeContent := `{{input "document"}}`
|
||||
largeDefinition := &domain.PromptDefinition{
|
||||
ID: "compiled-render-benchmark", Version: "v1",
|
||||
Inputs: []domain.PromptInput{{Name: "document"}},
|
||||
Templates: []domain.PromptMessageTemplate{{Role: "user", Content: largeContent}},
|
||||
}
|
||||
large := newCompiledRenderingBenchmarkCase(b,
|
||||
"artifact_backed_256KiB", "", []string{largeContent},
|
||||
largeDefinition, artifactInputs, nil, len(bodyText),
|
||||
func(b *testing.B, rendered *domain.RenderedPrompt) {
|
||||
if len(rendered.Messages) != 1 {
|
||||
b.Fatalf("artifact render message count=%d, want 1", len(rendered.Messages))
|
||||
}
|
||||
if rendered.Messages[0].Content != bodyText {
|
||||
b.Fatalf("artifact render length=%d, want %d", len(rendered.Messages[0].Content), len(bodyText))
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
repeatedSession := `document-size={{len (input "document")}}`
|
||||
repeatedMessages := []string{
|
||||
`{{input "document"}}`,
|
||||
`{{input "document"}}|{{input "document"}}`,
|
||||
}
|
||||
repeatedDefinition := &domain.PromptDefinition{
|
||||
ID: "compiled-render-benchmark", Version: "v1", SessionID: repeatedSession,
|
||||
Inputs: []domain.PromptInput{{Name: "document"}},
|
||||
Templates: []domain.PromptMessageTemplate{
|
||||
{Role: "system", Content: repeatedMessages[0]},
|
||||
{Role: "user", Content: repeatedMessages[1]},
|
||||
},
|
||||
}
|
||||
repeatedOutputBytes := len(bodyText)*3 + 1
|
||||
repeated := newCompiledRenderingBenchmarkCase(b,
|
||||
"repeated_across_session_and_messages", repeatedSession, repeatedMessages,
|
||||
repeatedDefinition, artifactInputs, nil, repeatedOutputBytes,
|
||||
func(b *testing.B, rendered *domain.RenderedPrompt) {
|
||||
if rendered.SessionID != "document-size="+strconv.Itoa(len(bodyText)) || len(rendered.Messages) != 2 ||
|
||||
rendered.Messages[0].Content != bodyText || rendered.Messages[1].Content != bodyText+"|"+bodyText {
|
||||
b.Fatalf("unexpected repeated compiled render: session=%q message lengths=%v", rendered.SessionID, renderedMessageLengths(rendered))
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
for _, benchmark := range []compiledRenderingBenchmarkCase{small, large, repeated} {
|
||||
b.Run(benchmark.name, func(b *testing.B) {
|
||||
rendered, err := benchmark.renderer.Render(benchmark.ctx, benchmark.definition, benchmark.inputs, benchmark.vars)
|
||||
if err != nil {
|
||||
b.Fatalf("check compiled renderer output: %v", err)
|
||||
}
|
||||
benchmark.check(b, rendered)
|
||||
|
||||
b.ReportAllocs()
|
||||
b.SetBytes(int64(benchmark.outputBytes))
|
||||
b.ResetTimer()
|
||||
for range b.N {
|
||||
rendered, err := benchmark.renderer.Render(benchmark.ctx, benchmark.definition, benchmark.inputs, benchmark.vars)
|
||||
if err != nil {
|
||||
b.Fatalf("render compiled template: %v", err)
|
||||
}
|
||||
renderedPromptBenchmarkSink = rendered
|
||||
}
|
||||
b.StopTimer()
|
||||
runtime.KeepAlive(renderedPromptBenchmarkSink)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// BenchmarkRetainedRenderedOutput reports post-GC live heap for a fixed batch
|
||||
// of rendered outputs above the same fixture's setup-only baseline.
|
||||
func BenchmarkRetainedRenderedOutput(b *testing.B) {
|
||||
body := bytes.Repeat([]byte("x"), rendererBenchmarkArtifactSize)
|
||||
bodyText := string(body)
|
||||
content := `{{input "document"}}`
|
||||
definition := &domain.PromptDefinition{
|
||||
ID: "retained-render-benchmark", Version: "v1",
|
||||
Inputs: []domain.PromptInput{{Name: "document"}},
|
||||
Templates: []domain.PromptMessageTemplate{{Role: "user", Content: content}},
|
||||
}
|
||||
compiled := CompiledTemplates{Messages: []*template.Template{mustCompileRendererBenchmarkTemplate(b, content)}}
|
||||
renderer := NewCompiledRenderer(rendererBenchmarkLookup{
|
||||
identity: rendererBenchmarkIdentity{id: definition.ID, version: definition.Version},
|
||||
templates: compiled,
|
||||
})
|
||||
inputs := map[string]*domain.Artifact{"document": {Name: "document", Body: body}}
|
||||
ctx := context.Background()
|
||||
const retainedBatchSize = 4
|
||||
retained := make([]*domain.RenderedPrompt, retainedBatchSize)
|
||||
|
||||
check, err := renderer.Render(ctx, definition, inputs, nil)
|
||||
if err != nil {
|
||||
b.Fatalf("check retained-output fixture: %v", err)
|
||||
}
|
||||
if len(check.Messages) != 1 || check.Messages[0].Content != bodyText {
|
||||
b.Fatalf("retained-output fixture returned unexpected content")
|
||||
}
|
||||
check = nil
|
||||
// Repeated -count runs must start from the same setup-only heap baseline.
|
||||
renderedPromptBenchmarkSink = nil
|
||||
renderedPromptsBenchmarkSink = nil
|
||||
|
||||
runtime.GC()
|
||||
var setupMemory runtime.MemStats
|
||||
runtime.ReadMemStats(&setupMemory)
|
||||
runtime.KeepAlive(renderer)
|
||||
runtime.KeepAlive(definition)
|
||||
runtime.KeepAlive(inputs)
|
||||
|
||||
b.ReportAllocs()
|
||||
b.SetBytes(int64(len(bodyText)))
|
||||
b.ResetTimer()
|
||||
for iteration := 0; iteration < b.N; iteration++ {
|
||||
rendered, err := renderer.Render(ctx, definition, inputs, nil)
|
||||
if err != nil {
|
||||
b.Fatalf("render retained-output fixture: %v", err)
|
||||
}
|
||||
retained[iteration%len(retained)] = rendered
|
||||
}
|
||||
b.StopTimer()
|
||||
for i, rendered := range retained {
|
||||
if rendered != nil {
|
||||
continue
|
||||
}
|
||||
rendered, err := renderer.Render(ctx, definition, inputs, nil)
|
||||
if err != nil {
|
||||
b.Fatalf("fill retained-output fixture: %v", err)
|
||||
}
|
||||
retained[i] = rendered
|
||||
}
|
||||
renderedPromptsBenchmarkSink = retained
|
||||
runtime.GC()
|
||||
var retainedMemory runtime.MemStats
|
||||
runtime.ReadMemStats(&retainedMemory)
|
||||
b.ReportMetric(float64(int64(retainedMemory.HeapAlloc)-int64(setupMemory.HeapAlloc)), "retained-B")
|
||||
runtime.KeepAlive(retained)
|
||||
runtime.KeepAlive(renderer)
|
||||
runtime.KeepAlive(definition)
|
||||
runtime.KeepAlive(inputs)
|
||||
}
|
||||
|
||||
func newCompiledRenderingBenchmarkCase(
|
||||
b *testing.B,
|
||||
name string,
|
||||
session string,
|
||||
messages []string,
|
||||
definition *domain.PromptDefinition,
|
||||
inputs map[string]*domain.Artifact,
|
||||
vars map[string]string,
|
||||
outputBytes int,
|
||||
check func(*testing.B, *domain.RenderedPrompt),
|
||||
) compiledRenderingBenchmarkCase {
|
||||
compiled := CompiledTemplates{}
|
||||
if session != "" {
|
||||
compiled.Session = mustCompileRendererBenchmarkTemplate(b, session)
|
||||
}
|
||||
for _, content := range messages {
|
||||
compiled.Messages = append(compiled.Messages, mustCompileRendererBenchmarkTemplate(b, content))
|
||||
}
|
||||
return compiledRenderingBenchmarkCase{
|
||||
name: name,
|
||||
renderer: NewCompiledRenderer(rendererBenchmarkLookup{
|
||||
identity: rendererBenchmarkIdentity{id: definition.ID, version: definition.Version},
|
||||
templates: compiled,
|
||||
}),
|
||||
definition: definition, inputs: inputs, vars: vars, ctx: context.Background(),
|
||||
outputBytes: outputBytes, check: check,
|
||||
}
|
||||
}
|
||||
|
||||
func mustCompileRendererBenchmarkTemplate(b *testing.B, source string) *template.Template {
|
||||
b.Helper()
|
||||
compiled, err := CompileTemplate("benchmark", source)
|
||||
if err != nil {
|
||||
b.Fatalf("compile benchmark template: %v", err)
|
||||
}
|
||||
return compiled
|
||||
}
|
||||
|
||||
func renderedMessageLengths(rendered *domain.RenderedPrompt) []int {
|
||||
lengths := make([]int, len(rendered.Messages))
|
||||
for i, message := range rendered.Messages {
|
||||
lengths[i] = len(message.Content)
|
||||
}
|
||||
return lengths
|
||||
}
|
||||
@@ -1,7 +1,6 @@
|
||||
package prompt
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
@@ -126,7 +125,7 @@ func (r *goRenderer) Render(ctx context.Context, definition *domain.PromptDefini
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var buf bytes.Buffer
|
||||
var buf strings.Builder
|
||||
executeErr := tmpl.Execute(&buf, vars)
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
@@ -235,7 +234,7 @@ func executeSessionTemplate(ctx context.Context, tmpl *template.Template, vars m
|
||||
if err := ctx.Err(); err != nil {
|
||||
return "", err
|
||||
}
|
||||
var buf bytes.Buffer
|
||||
var buf strings.Builder
|
||||
executeErr := tmpl.Execute(&buf, vars)
|
||||
if err := ctx.Err(); err != nil {
|
||||
return "", err
|
||||
|
||||
@@ -4,8 +4,11 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"text/template"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
)
|
||||
@@ -501,6 +504,77 @@ func TestGoRendererArtifactTextLifecycle(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompiledRendererKeepsConcurrentRequestsIndependent(t *testing.T) {
|
||||
const (
|
||||
promptID = "concurrent-render"
|
||||
version = "v1"
|
||||
)
|
||||
sessionTemplate, err := CompileTemplate("session", "session-{{.request}}")
|
||||
if err != nil {
|
||||
t.Fatalf("compile session template: %v", err)
|
||||
}
|
||||
messageTemplate, err := CompileTemplate("message", `request={{.request}};document={{input "document"}}`)
|
||||
if err != nil {
|
||||
t.Fatalf("compile message template: %v", err)
|
||||
}
|
||||
renderer := NewCompiledRenderer(compiledRendererTestLookup{
|
||||
id: promptID,
|
||||
version: version,
|
||||
templates: CompiledTemplates{
|
||||
Session: sessionTemplate,
|
||||
Messages: []*template.Template{messageTemplate},
|
||||
},
|
||||
})
|
||||
definition := &domain.PromptDefinition{
|
||||
ID: promptID, Version: version, SessionID: "session-{{.request}}",
|
||||
Inputs: []domain.PromptInput{{Name: "document"}},
|
||||
Templates: []domain.PromptMessageTemplate{{Role: "user", Content: `request={{.request}};document={{input "document"}}`}},
|
||||
}
|
||||
|
||||
const requestCount = 24
|
||||
errs := make(chan error, requestCount)
|
||||
var wait sync.WaitGroup
|
||||
wait.Add(requestCount)
|
||||
for i := 0; i < requestCount; i++ {
|
||||
go func(i int) {
|
||||
defer wait.Done()
|
||||
request := fmt.Sprintf("request-%d", i)
|
||||
document := fmt.Sprintf("document-%d", i)
|
||||
rendered, err := renderer.Render(context.Background(), definition,
|
||||
map[string]*domain.Artifact{"document": {Body: []byte(document)}},
|
||||
map[string]string{"request": request},
|
||||
)
|
||||
if err != nil {
|
||||
errs <- fmt.Errorf("render %s: %w", request, err)
|
||||
return
|
||||
}
|
||||
if rendered.SessionID != "session-"+request || len(rendered.Messages) != 1 ||
|
||||
rendered.Messages[0].Content != "request="+request+";document="+document {
|
||||
errs <- fmt.Errorf("render %s returned session %q and messages %#v", request, rendered.SessionID, rendered.Messages)
|
||||
}
|
||||
}(i)
|
||||
}
|
||||
wait.Wait()
|
||||
close(errs)
|
||||
for err := range errs {
|
||||
t.Error(err)
|
||||
}
|
||||
}
|
||||
|
||||
type compiledRendererTestLookup struct {
|
||||
id, version string
|
||||
templates CompiledTemplates
|
||||
}
|
||||
|
||||
func (lookup compiledRendererTestLookup) GetCompiledTemplates(id, version string) (CompiledTemplates, bool) {
|
||||
if id != lookup.id || version != lookup.version {
|
||||
return CompiledTemplates{}, false
|
||||
}
|
||||
return lookup.templates, true
|
||||
}
|
||||
|
||||
// BenchmarkGoRendererArtifactReferences measures NewGoRenderer's per-call
|
||||
// template-parsing fallback rather than the captured compiled path.
|
||||
func BenchmarkGoRendererArtifactReferences(b *testing.B) {
|
||||
body := bytes.Repeat([]byte("document content "), (artifactTextChunkSize*4)/len("document content "))
|
||||
inputs := map[string]*domain.Artifact{
|
||||
|
||||
978
internal/prompt/requirements.go
Normal file
978
internal/prompt/requirements.go
Normal file
@@ -0,0 +1,978 @@
|
||||
package prompt
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sort"
|
||||
"text/template"
|
||||
"text/template/parse"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
)
|
||||
|
||||
type requirementReachability uint8
|
||||
|
||||
const (
|
||||
requirementDefinite requirementReachability = iota
|
||||
requirementPossible
|
||||
requirementInactive
|
||||
)
|
||||
|
||||
type dotIdentity uint8
|
||||
|
||||
const (
|
||||
dotRoot dotIdentity = iota
|
||||
dotNonRoot
|
||||
dotUnknown
|
||||
)
|
||||
|
||||
type inputEvidence struct {
|
||||
name string
|
||||
kind domain.RequirementKind
|
||||
declared bool
|
||||
applicable bool
|
||||
contentType *string
|
||||
description *string
|
||||
explanation string
|
||||
}
|
||||
|
||||
type variableEvidence struct {
|
||||
name string
|
||||
kind domain.RequirementKind
|
||||
applicable bool
|
||||
explanation string
|
||||
}
|
||||
|
||||
type requirementIssueKey struct {
|
||||
code domain.RequirementIssueCode
|
||||
inputName string
|
||||
variableName string
|
||||
}
|
||||
|
||||
type conditionEvaluation struct {
|
||||
value any
|
||||
known bool
|
||||
unsupported bool
|
||||
inputNameUnknown bool
|
||||
missingVariables map[string]struct{}
|
||||
contentInputs map[string]struct{}
|
||||
}
|
||||
|
||||
func (evaluation conditionEvaluation) truth() bool {
|
||||
truth, ok := template.IsTrue(evaluation.value)
|
||||
return evaluation.known && ok && truth
|
||||
}
|
||||
|
||||
func (walker *requirementWalker) evaluateCondition(pipe *parse.PipeNode, reach requirementReachability, dot dotIdentity) conditionEvaluation {
|
||||
if pipe == nil {
|
||||
return conditionEvaluation{unsupported: true}
|
||||
}
|
||||
if len(pipe.Cmds) == 1 && len(pipe.Decl) == 0 && !pipe.IsAssign {
|
||||
return walker.evaluateCommand(pipe.Cmds[0], reach, dot)
|
||||
}
|
||||
if name, ok := directInputPipeline(pipe); ok {
|
||||
return walker.evaluateInputName(name, reach, dot)
|
||||
}
|
||||
before := walker.dependencies
|
||||
visitReach := requirementPossible
|
||||
if reach == requirementInactive {
|
||||
visitReach = requirementInactive
|
||||
}
|
||||
if err := walker.visitPipe(pipe, visitReach, dot); err != nil {
|
||||
return conditionEvaluation{unsupported: true}
|
||||
}
|
||||
return conditionEvaluation{unsupported: walker.dependencies != before}
|
||||
}
|
||||
|
||||
func (walker *requirementWalker) evaluateCommand(command *parse.CommandNode, reach requirementReachability, dot dotIdentity) conditionEvaluation {
|
||||
if command == nil || len(command.Args) == 0 {
|
||||
return conditionEvaluation{unsupported: true}
|
||||
}
|
||||
identifier, isIdentifier := command.Args[0].(*parse.IdentifierNode)
|
||||
if isIdentifier {
|
||||
switch identifier.Ident {
|
||||
case "input":
|
||||
if reach == requirementInactive {
|
||||
if len(command.Args) == 2 {
|
||||
if name, ok := literalStringValue(command.Args[1]); ok {
|
||||
walker.recordInput(name, requirementInactive)
|
||||
return conditionEvaluation{}
|
||||
}
|
||||
}
|
||||
for _, argument := range command.Args[1:] {
|
||||
_ = walker.visit(argument, requirementInactive, dot)
|
||||
}
|
||||
return conditionEvaluation{}
|
||||
}
|
||||
if len(command.Args) == 2 {
|
||||
if name, ok := command.Args[1].(*parse.StringNode); ok {
|
||||
walker.recordInput(name.Text, reach)
|
||||
return conditionEvaluation{contentInputs: map[string]struct{}{name.Text: {}}}
|
||||
}
|
||||
return walker.evaluateInputName(command.Args[1], reach, dot)
|
||||
}
|
||||
for _, argument := range command.Args[1:] {
|
||||
_ = walker.visit(argument, requirementPossible, dot)
|
||||
}
|
||||
return walker.unresolvedInputName(conditionEvaluation{unsupported: true})
|
||||
case "index":
|
||||
if len(command.Args) == 3 {
|
||||
if key, ok := command.Args[2].(*parse.StringNode); ok && isRootMap(command.Args[1], dot, walker.rootKnown) {
|
||||
walker.recordOptionalVariable(key.Text, reach)
|
||||
value, supplied := walker.vars[key.Text]
|
||||
if !supplied {
|
||||
value = ""
|
||||
}
|
||||
return conditionEvaluation{value: value, known: true}
|
||||
}
|
||||
}
|
||||
case "not":
|
||||
if len(command.Args) == 2 {
|
||||
evaluation := walker.evaluateValue(command.Args[1], reach, dot)
|
||||
if evaluation.known {
|
||||
truth, ok := template.IsTrue(evaluation.value)
|
||||
if ok {
|
||||
evaluation.value = !truth
|
||||
} else {
|
||||
evaluation.known = false
|
||||
evaluation.unsupported = true
|
||||
}
|
||||
}
|
||||
return evaluation
|
||||
}
|
||||
case "eq", "ne":
|
||||
if len(command.Args) == 3 {
|
||||
left := walker.evaluateValue(command.Args[1], reach, dot)
|
||||
right := walker.evaluateValue(command.Args[2], reach, dot)
|
||||
result := mergeConditionEvaluations(left, right)
|
||||
if left.known && right.known {
|
||||
leftString, leftOK := left.value.(string)
|
||||
rightString, rightOK := right.value.(string)
|
||||
if leftOK && rightOK {
|
||||
equal := leftString == rightString
|
||||
if identifier.Ident == "ne" {
|
||||
equal = !equal
|
||||
}
|
||||
result.value = equal
|
||||
result.known = true
|
||||
return result
|
||||
}
|
||||
result.unsupported = true
|
||||
}
|
||||
return result
|
||||
}
|
||||
}
|
||||
}
|
||||
if !isIdentifier && len(command.Args) == 1 {
|
||||
return walker.evaluateValue(command.Args[0], reach, dot)
|
||||
}
|
||||
if reach == requirementInactive {
|
||||
_ = walker.visitCommand(command, reach, dot)
|
||||
return conditionEvaluation{}
|
||||
}
|
||||
before := walker.dependencies
|
||||
_ = walker.visitCommand(command, requirementPossible, dot)
|
||||
return conditionEvaluation{unsupported: walker.dependencies != before}
|
||||
}
|
||||
|
||||
func (walker *requirementWalker) evaluateValue(node parse.Node, reach requirementReachability, dot dotIdentity) conditionEvaluation {
|
||||
switch node := node.(type) {
|
||||
case *parse.DotNode:
|
||||
if dot == dotRoot {
|
||||
return conditionEvaluation{value: walker.vars, known: true}
|
||||
}
|
||||
if dot == dotUnknown {
|
||||
walker.unresolvedVariableScope(reach)
|
||||
}
|
||||
return conditionEvaluation{unsupported: true}
|
||||
case *parse.StringNode:
|
||||
return conditionEvaluation{value: node.Text, known: true}
|
||||
case *parse.BoolNode:
|
||||
return conditionEvaluation{value: node.True, known: true}
|
||||
case *parse.NilNode:
|
||||
return conditionEvaluation{value: nil, known: true}
|
||||
case *parse.FieldNode:
|
||||
if dot != dotRoot || len(node.Ident) == 0 {
|
||||
if dot == dotUnknown {
|
||||
walker.unresolvedVariableScope(reach)
|
||||
}
|
||||
return conditionEvaluation{unsupported: true}
|
||||
}
|
||||
name := node.Ident[0]
|
||||
walker.recordVariable(name, reach)
|
||||
if len(node.Ident) > 1 {
|
||||
walker.addIssue("", name, "A chained field access is outside the direct-key scope.")
|
||||
return conditionEvaluation{unsupported: true}
|
||||
}
|
||||
value, supplied := walker.vars[name]
|
||||
if !supplied {
|
||||
return conditionEvaluation{missingVariables: map[string]struct{}{name: {}}}
|
||||
}
|
||||
return conditionEvaluation{value: value, known: true}
|
||||
case *parse.VariableNode:
|
||||
if len(node.Ident) == 1 && node.Ident[0] == "$" && walker.rootKnown {
|
||||
return conditionEvaluation{value: walker.vars, known: true}
|
||||
}
|
||||
if len(node.Ident) < 2 || node.Ident[0] != "$" || !walker.rootKnown {
|
||||
walker.visitVariable(node, reach)
|
||||
return conditionEvaluation{unsupported: true}
|
||||
}
|
||||
name := node.Ident[1]
|
||||
walker.recordVariable(name, reach)
|
||||
if len(node.Ident) > 2 {
|
||||
walker.addIssue("", name, "A chained field access is outside the direct-key scope.")
|
||||
return conditionEvaluation{unsupported: true}
|
||||
}
|
||||
value, supplied := walker.vars[name]
|
||||
if !supplied {
|
||||
return conditionEvaluation{missingVariables: map[string]struct{}{name: {}}}
|
||||
}
|
||||
return conditionEvaluation{value: value, known: true}
|
||||
case *parse.CommandNode:
|
||||
return walker.evaluateCommand(node, reach, dot)
|
||||
case *parse.PipeNode:
|
||||
return walker.evaluateCondition(node, reach, dot)
|
||||
case *parse.ChainNode:
|
||||
before := walker.dependencies
|
||||
visitReach := requirementPossible
|
||||
if reach == requirementInactive {
|
||||
visitReach = requirementInactive
|
||||
}
|
||||
_ = walker.visit(node.Node, visitReach, dot)
|
||||
return conditionEvaluation{unsupported: walker.dependencies != before}
|
||||
default:
|
||||
return conditionEvaluation{unsupported: true}
|
||||
}
|
||||
}
|
||||
|
||||
func (walker *requirementWalker) evaluateInputName(node parse.Node, reach requirementReachability, dot dotIdentity) conditionEvaluation {
|
||||
evaluation := walker.evaluateValue(node, reach, dot)
|
||||
if evaluation.known {
|
||||
if name, ok := evaluation.value.(string); ok {
|
||||
walker.recordInput(name, reach)
|
||||
return conditionEvaluation{contentInputs: map[string]struct{}{name: {}}}
|
||||
}
|
||||
evaluation.unsupported = true
|
||||
}
|
||||
// Value analysis has already visited recognizable references, including
|
||||
// the conservative fallback for unsupported expressions. Do not rescan
|
||||
// the same subtree: nested computed names would multiply that work.
|
||||
return walker.unresolvedInputName(evaluation)
|
||||
}
|
||||
|
||||
func (walker *requirementWalker) unresolvedInputName(evaluation conditionEvaluation) conditionEvaluation {
|
||||
for name := range evaluation.missingVariables {
|
||||
walker.addIssueCode(domain.RequirementIssueMissingVariable, "", name, "A missing variable prevents resolving an input name.")
|
||||
}
|
||||
for name := range evaluation.contentInputs {
|
||||
walker.addIssueCode(domain.RequirementIssueInputContent, name, "", "Captured input content affects an input name.")
|
||||
}
|
||||
walker.addIssueCode(domain.RequirementIssueDynamicInputName, "", "", "An input helper name is not statically available.")
|
||||
walker.unknownInputEffect = true
|
||||
evaluation.inputNameUnknown = true
|
||||
return evaluation
|
||||
}
|
||||
|
||||
func directInputPipeline(pipe *parse.PipeNode) (parse.Node, bool) {
|
||||
if pipe == nil || len(pipe.Cmds) != 2 || len(pipe.Decl) != 0 || pipe.IsAssign {
|
||||
return nil, false
|
||||
}
|
||||
input := pipe.Cmds[1]
|
||||
if len(input.Args) != 1 {
|
||||
return nil, false
|
||||
}
|
||||
identifier, ok := input.Args[0].(*parse.IdentifierNode)
|
||||
if !ok || identifier.Ident != "input" {
|
||||
return nil, false
|
||||
}
|
||||
value := pipe.Cmds[0]
|
||||
if len(value.Args) > 0 {
|
||||
valueIdentifier, isValueIdentifier := value.Args[0].(*parse.IdentifierNode)
|
||||
if isValueIdentifier && valueIdentifier.Ident == "input" && len(value.Args) == 2 {
|
||||
return value, true
|
||||
}
|
||||
}
|
||||
if len(value.Args) == 1 {
|
||||
return value.Args[0], true
|
||||
}
|
||||
if len(value.Args) == 3 {
|
||||
function, ok := value.Args[0].(*parse.IdentifierNode)
|
||||
if ok && function.Ident == "index" {
|
||||
return value, true
|
||||
}
|
||||
}
|
||||
return nil, false
|
||||
}
|
||||
|
||||
func literalStringValue(node parse.Node) (string, bool) {
|
||||
switch node := node.(type) {
|
||||
case *parse.StringNode:
|
||||
return node.Text, true
|
||||
case *parse.PipeNode:
|
||||
if len(node.Cmds) == 1 && len(node.Cmds[0].Args) == 1 && len(node.Decl) == 0 && !node.IsAssign {
|
||||
return literalStringValue(node.Cmds[0].Args[0])
|
||||
}
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
func mergeConditionEvaluations(left, right conditionEvaluation) conditionEvaluation {
|
||||
result := conditionEvaluation{
|
||||
unsupported: left.unsupported || right.unsupported,
|
||||
inputNameUnknown: left.inputNameUnknown || right.inputNameUnknown,
|
||||
missingVariables: make(map[string]struct{}, len(left.missingVariables)+len(right.missingVariables)),
|
||||
contentInputs: make(map[string]struct{}, len(left.contentInputs)+len(right.contentInputs)),
|
||||
}
|
||||
for name := range left.missingVariables {
|
||||
result.missingVariables[name] = struct{}{}
|
||||
}
|
||||
for name := range right.missingVariables {
|
||||
result.missingVariables[name] = struct{}{}
|
||||
}
|
||||
for name := range left.contentInputs {
|
||||
result.contentInputs[name] = struct{}{}
|
||||
}
|
||||
for name := range right.contentInputs {
|
||||
result.contentInputs[name] = struct{}{}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func branchReach(parent requirementReachability, known, truth, selected bool) requirementReachability {
|
||||
if parent == requirementInactive {
|
||||
return requirementInactive
|
||||
}
|
||||
if !known {
|
||||
return requirementPossible
|
||||
}
|
||||
if truth == selected {
|
||||
if parent == requirementPossible {
|
||||
return requirementPossible
|
||||
}
|
||||
return requirementDefinite
|
||||
}
|
||||
return requirementInactive
|
||||
}
|
||||
|
||||
type requirementWalker struct {
|
||||
ctx context.Context
|
||||
vars map[string]string
|
||||
inputs map[string]*inputEvidence
|
||||
variables map[string]*variableEvidence
|
||||
issues map[requirementIssueKey]domain.RequirementIssue
|
||||
complete bool
|
||||
rootKnown bool
|
||||
unknownInputEffect bool
|
||||
unknownVariableEffect bool
|
||||
dependencies int
|
||||
templateSet *template.Template
|
||||
visitedTrees map[*parse.Tree]bool
|
||||
}
|
||||
|
||||
// AnalyzeRequirements collects the prompt dependencies recognizable from its
|
||||
// captured syntax. Branch bodies remain conditional until their conditions are
|
||||
// classified; unsupported dependency-bearing syntax makes the result partial.
|
||||
func AnalyzeRequirements(
|
||||
ctx context.Context,
|
||||
definition *domain.PromptDefinition,
|
||||
snapshot RequirementSnapshot,
|
||||
vars map[string]string,
|
||||
sessionBypassed bool,
|
||||
) (domain.RequirementInspection, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return domain.RequirementInspection{}, err
|
||||
}
|
||||
result := domain.RequirementInspection{Complete: true}
|
||||
if definition != nil {
|
||||
result.PromptID = definition.ID
|
||||
result.PromptVersion = definition.Version
|
||||
}
|
||||
walker := &requirementWalker{
|
||||
ctx: ctx,
|
||||
vars: vars,
|
||||
inputs: make(map[string]*inputEvidence, len(snapshot.Inputs)),
|
||||
variables: make(map[string]*variableEvidence),
|
||||
issues: make(map[requirementIssueKey]domain.RequirementIssue),
|
||||
complete: true,
|
||||
rootKnown: true,
|
||||
}
|
||||
for _, input := range snapshot.Inputs {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return domain.RequirementInspection{}, err
|
||||
}
|
||||
kind := domain.RequirementOptional
|
||||
if input.Required {
|
||||
kind = domain.RequirementRequired
|
||||
}
|
||||
walker.inputs[input.Name] = &inputEvidence{
|
||||
name: input.Name,
|
||||
kind: kind,
|
||||
declared: true,
|
||||
contentType: copyString(input.ContentType),
|
||||
description: copyString(input.Description),
|
||||
}
|
||||
if kind == domain.RequirementOptional {
|
||||
walker.inputs[input.Name].explanation = "Declared as an optional input."
|
||||
}
|
||||
}
|
||||
for _, message := range snapshot.Templates.Messages {
|
||||
if err := walker.walkTemplate(message); err != nil {
|
||||
return domain.RequirementInspection{}, err
|
||||
}
|
||||
}
|
||||
if !sessionBypassed && snapshot.Templates.Session != nil {
|
||||
if err := walker.walkTemplate(snapshot.Templates.Session); err != nil {
|
||||
return domain.RequirementInspection{}, err
|
||||
}
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return domain.RequirementInspection{}, err
|
||||
}
|
||||
return walker.result(result), nil
|
||||
}
|
||||
|
||||
func (walker *requirementWalker) walkTemplate(compiled *template.Template) error {
|
||||
if err := walker.ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if compiled == nil || compiled.Tree == nil || compiled.Tree.Root == nil {
|
||||
walker.addIssue("", "", "A captured template has no parsed root.")
|
||||
return nil
|
||||
}
|
||||
previousSet := walker.templateSet
|
||||
previousVisited := walker.visitedTrees
|
||||
walker.templateSet = compiled
|
||||
walker.visitedTrees = map[*parse.Tree]bool{compiled.Tree: true}
|
||||
walker.rootKnown = true
|
||||
err := walker.visit(compiled.Tree.Root, requirementDefinite, dotRoot)
|
||||
walker.templateSet = previousSet
|
||||
walker.visitedTrees = previousVisited
|
||||
return err
|
||||
}
|
||||
|
||||
func (walker *requirementWalker) visit(node parse.Node, reach requirementReachability, dot dotIdentity) error {
|
||||
if err := walker.ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if node == nil {
|
||||
return nil
|
||||
}
|
||||
switch node := node.(type) {
|
||||
case *parse.ListNode:
|
||||
if node == nil {
|
||||
return nil
|
||||
}
|
||||
for _, child := range node.Nodes {
|
||||
if err := walker.visit(child, reach, dot); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
case *parse.ActionNode:
|
||||
return walker.visit(node.Pipe, reach, dot)
|
||||
case *parse.IfNode:
|
||||
if reach == requirementInactive {
|
||||
if err := walker.visit(node.Pipe, requirementInactive, dot); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := walker.visit(node.List, requirementInactive, dot); err != nil {
|
||||
return err
|
||||
}
|
||||
return walker.visit(node.ElseList, requirementInactive, dot)
|
||||
}
|
||||
condition := walker.evaluateCondition(node.Pipe, reach, dot)
|
||||
before := walker.dependencies
|
||||
thenReach := branchReach(reach, condition.known, condition.truth(), true)
|
||||
elseReach := branchReach(reach, condition.known, condition.truth(), false)
|
||||
if err := walker.visit(node.List, thenReach, dot); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := walker.visit(node.ElseList, elseReach, dot); err != nil {
|
||||
return err
|
||||
}
|
||||
if !condition.known && walker.dependencies != before {
|
||||
walker.recordConditionLimitation(condition)
|
||||
}
|
||||
return nil
|
||||
case *parse.WithNode:
|
||||
if reach == requirementInactive {
|
||||
if err := walker.visit(node.Pipe, requirementInactive, dot); err != nil {
|
||||
return err
|
||||
}
|
||||
bodyDot := walker.withDotIdentity(node.Pipe, dot, conditionEvaluation{})
|
||||
if err := walker.visit(node.List, requirementInactive, bodyDot); err != nil {
|
||||
return err
|
||||
}
|
||||
return walker.visit(node.ElseList, requirementInactive, dot)
|
||||
}
|
||||
condition := walker.evaluateCondition(node.Pipe, reach, dot)
|
||||
before := walker.dependencies
|
||||
thenReach := branchReach(reach, condition.known, condition.truth(), true)
|
||||
elseReach := branchReach(reach, condition.known, condition.truth(), false)
|
||||
bodyDot := walker.withDotIdentity(node.Pipe, dot, condition)
|
||||
if err := walker.visit(node.List, thenReach, bodyDot); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := walker.visit(node.ElseList, elseReach, dot); err != nil {
|
||||
return err
|
||||
}
|
||||
if !condition.known && walker.dependencies != before {
|
||||
walker.recordConditionLimitation(condition)
|
||||
}
|
||||
return nil
|
||||
case *parse.RangeNode:
|
||||
if err := walker.visit(node.Pipe, reach, dot); err != nil {
|
||||
return err
|
||||
}
|
||||
before := walker.dependencies
|
||||
bodyReach := requirementPossible
|
||||
if reach == requirementInactive {
|
||||
bodyReach = requirementInactive
|
||||
}
|
||||
if err := walker.visit(node.List, bodyReach, dotNonRoot); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := walker.visit(node.ElseList, bodyReach, dot); err != nil {
|
||||
return err
|
||||
}
|
||||
if reach != requirementInactive && walker.dependencies != before {
|
||||
walker.addIssue("", "", "Dependencies inside a range are only conservatively enumerated.")
|
||||
}
|
||||
case *parse.TemplateNode:
|
||||
pipeReach := requirementPossible
|
||||
if reach == requirementInactive {
|
||||
pipeReach = requirementInactive
|
||||
}
|
||||
if err := walker.visit(node.Pipe, pipeReach, dot); err != nil {
|
||||
return err
|
||||
}
|
||||
if reach == requirementInactive {
|
||||
return nil
|
||||
}
|
||||
called := walker.templateSet.Lookup(node.Name)
|
||||
if called == nil || called.Tree == nil || called.Tree.Root == nil {
|
||||
walker.addIssue("", "", "An invoked named template is unavailable for syntax inspection.")
|
||||
return nil
|
||||
}
|
||||
if walker.visitedTrees[called.Tree] {
|
||||
walker.addIssue("", "", "An invoked named template is scanned conservatively.")
|
||||
return nil
|
||||
}
|
||||
walker.visitedTrees[called.Tree] = true
|
||||
rootKnown := walker.rootKnown
|
||||
walker.rootKnown = false
|
||||
walker.unknownVariableEffect = true
|
||||
err := walker.visit(called.Tree.Root, requirementPossible, dotNonRoot)
|
||||
walker.rootKnown = rootKnown
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
walker.addIssue("", "", "An invoked named template is scanned conservatively.")
|
||||
case *parse.PipeNode:
|
||||
return walker.visitPipe(node, reach, dot)
|
||||
case *parse.CommandNode:
|
||||
return walker.visitCommand(node, reach, dot)
|
||||
case *parse.FieldNode:
|
||||
if dot == dotRoot && len(node.Ident) > 0 {
|
||||
walker.recordVariable(node.Ident[0], reach)
|
||||
if len(node.Ident) > 1 && reach != requirementInactive {
|
||||
walker.addIssue("", node.Ident[0], "A chained field access is outside the direct-key scope.")
|
||||
}
|
||||
}
|
||||
if dot == dotUnknown {
|
||||
walker.unresolvedVariableScope(reach)
|
||||
}
|
||||
case *parse.DotNode:
|
||||
if dot == dotUnknown {
|
||||
walker.unresolvedVariableScope(reach)
|
||||
}
|
||||
case *parse.VariableNode:
|
||||
walker.visitVariable(node, reach)
|
||||
case *parse.ChainNode:
|
||||
before := walker.dependencies
|
||||
chainReach := requirementPossible
|
||||
if reach == requirementInactive {
|
||||
chainReach = requirementInactive
|
||||
}
|
||||
if err := walker.visit(node.Node, chainReach, dot); err != nil {
|
||||
return err
|
||||
}
|
||||
if reach != requirementInactive && walker.dependencies != before {
|
||||
walker.addIssue("", "", "A chained expression is not interpreted.")
|
||||
}
|
||||
case *parse.BreakNode, *parse.ContinueNode:
|
||||
if reach != requirementInactive {
|
||||
walker.addIssue("", "", "Loop control is outside the supported syntax scope.")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (walker *requirementWalker) visitPipe(pipe *parse.PipeNode, reach requirementReachability, dot dotIdentity) error {
|
||||
if pipe == nil {
|
||||
return nil
|
||||
}
|
||||
if len(pipe.Cmds) == 1 && len(pipe.Decl) == 0 && !pipe.IsAssign {
|
||||
return walker.visitCommand(pipe.Cmds[0], reach, dot)
|
||||
}
|
||||
if name, ok := directInputPipeline(pipe); ok {
|
||||
if reach == requirementInactive {
|
||||
if literal, known := literalStringValue(name); known {
|
||||
walker.recordInput(literal, requirementInactive)
|
||||
return nil
|
||||
}
|
||||
return walker.visit(name, requirementInactive, dot)
|
||||
}
|
||||
walker.evaluateInputName(name, reach, dot)
|
||||
return nil
|
||||
}
|
||||
before := walker.dependencies
|
||||
commandReach := requirementPossible
|
||||
if reach == requirementInactive {
|
||||
commandReach = requirementInactive
|
||||
}
|
||||
for _, command := range pipe.Cmds {
|
||||
if err := walker.visitCommand(command, commandReach, dot); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if reach != requirementInactive && walker.dependencies != before {
|
||||
walker.addIssue("", "", "A pipeline or variable assignment is not interpreted.")
|
||||
}
|
||||
if reach != requirementInactive {
|
||||
for _, declaration := range pipe.Decl {
|
||||
if len(declaration.Ident) > 0 && declaration.Ident[0] == "$" {
|
||||
walker.rootKnown = false
|
||||
walker.unknownVariableEffect = true
|
||||
walker.addIssue("", "", "The root template binding is reassigned.")
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (walker *requirementWalker) visitCommand(command *parse.CommandNode, reach requirementReachability, dot dotIdentity) error {
|
||||
if command == nil || len(command.Args) == 0 {
|
||||
return nil
|
||||
}
|
||||
identifier, isIdentifier := command.Args[0].(*parse.IdentifierNode)
|
||||
if isIdentifier && identifier.Ident == "input" {
|
||||
if reach == requirementInactive {
|
||||
if len(command.Args) == 2 {
|
||||
if name, ok := literalStringValue(command.Args[1]); ok {
|
||||
walker.recordInput(name, reach)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
for _, argument := range command.Args[1:] {
|
||||
if err := walker.visit(argument, requirementInactive, dot); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if len(command.Args) == 2 {
|
||||
if name, ok := command.Args[1].(*parse.StringNode); ok {
|
||||
walker.recordInput(name.Text, reach)
|
||||
return nil
|
||||
}
|
||||
walker.evaluateInputName(command.Args[1], reach, dot)
|
||||
return nil
|
||||
}
|
||||
for _, argument := range command.Args[1:] {
|
||||
if err := walker.visit(argument, requirementPossible, dot); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
walker.unresolvedInputName(conditionEvaluation{unsupported: true})
|
||||
return nil
|
||||
}
|
||||
if isIdentifier && identifier.Ident == "index" && len(command.Args) == 3 {
|
||||
if _, ok := command.Args[2].(*parse.StringNode); ok && isRootMap(command.Args[1], dot, walker.rootKnown) {
|
||||
key := command.Args[2].(*parse.StringNode).Text
|
||||
walker.recordOptionalVariable(key, reach)
|
||||
return nil
|
||||
}
|
||||
// A local map may alias the caller root even though its value is not
|
||||
// tracked. A lookup on it must not vanish from complete discovery.
|
||||
if variable, ok := unwrapSingleValue(command.Args[1]).(*parse.VariableNode); ok && len(variable.Ident) == 1 {
|
||||
walker.unresolvedVariableScope(reach)
|
||||
}
|
||||
}
|
||||
if !isIdentifier && len(command.Args) == 1 {
|
||||
return walker.visit(command.Args[0], reach, dot)
|
||||
}
|
||||
before := walker.dependencies
|
||||
for index, argument := range command.Args {
|
||||
if isIdentifier && index == 0 {
|
||||
continue
|
||||
}
|
||||
argumentReach := requirementPossible
|
||||
if reach == requirementInactive {
|
||||
argumentReach = requirementInactive
|
||||
}
|
||||
if err := walker.visit(argument, argumentReach, dot); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if reach != requirementInactive && walker.dependencies != before {
|
||||
walker.addIssue("", "", "A function call or computed expression is not interpreted.")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (walker *requirementWalker) visitVariable(variable *parse.VariableNode, reach requirementReachability) {
|
||||
if len(variable.Ident) < 2 {
|
||||
return
|
||||
}
|
||||
if variable.Ident[0] != "$" {
|
||||
walker.unresolvedVariableScope(reach)
|
||||
return
|
||||
}
|
||||
if !walker.rootKnown {
|
||||
if reach != requirementInactive {
|
||||
walker.unknownVariableEffect = true
|
||||
walker.addIssue("", "", "The root template binding is not known.")
|
||||
}
|
||||
return
|
||||
}
|
||||
walker.recordVariable(variable.Ident[1], reach)
|
||||
if len(variable.Ident) > 2 && reach != requirementInactive {
|
||||
walker.addIssue("", variable.Ident[1], "A chained field access is outside the direct-key scope.")
|
||||
}
|
||||
}
|
||||
|
||||
func (walker *requirementWalker) unresolvedVariableScope(reach requirementReachability) {
|
||||
if reach != requirementInactive {
|
||||
walker.unknownVariableEffect = true
|
||||
walker.addIssue("", "", "A caller-variable scope is not statically known.")
|
||||
}
|
||||
}
|
||||
|
||||
func (walker *requirementWalker) withDotIdentity(pipe *parse.PipeNode, dot dotIdentity, condition conditionEvaluation) dotIdentity {
|
||||
if pipe != nil && len(pipe.Cmds) == 1 && len(pipe.Decl) == 0 && !pipe.IsAssign && len(pipe.Cmds[0].Args) == 1 &&
|
||||
isRootMap(pipe.Cmds[0].Args[0], dot, walker.rootKnown) {
|
||||
return dotRoot
|
||||
}
|
||||
if condition.known {
|
||||
if _, root := condition.value.(map[string]string); root {
|
||||
return dotRoot
|
||||
}
|
||||
return dotNonRoot
|
||||
}
|
||||
if condition.inputNameUnknown ||
|
||||
!condition.unsupported && (len(condition.missingVariables) > 0 || len(condition.contentInputs) > 0) {
|
||||
// Supported caller values and input bodies are strings; supported
|
||||
// comparisons and negation return booleans, never the root map.
|
||||
return dotNonRoot
|
||||
}
|
||||
return dotUnknown
|
||||
}
|
||||
|
||||
func isRootMap(node parse.Node, dot dotIdentity, rootKnown bool) bool {
|
||||
switch node := unwrapSingleValue(node).(type) {
|
||||
case *parse.DotNode:
|
||||
return dot == dotRoot
|
||||
case *parse.VariableNode:
|
||||
return rootKnown && len(node.Ident) == 1 && node.Ident[0] == "$"
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func unwrapSingleValue(node parse.Node) parse.Node {
|
||||
for {
|
||||
pipe, ok := node.(*parse.PipeNode)
|
||||
if !ok || len(pipe.Cmds) != 1 || len(pipe.Decl) != 0 || pipe.IsAssign || len(pipe.Cmds[0].Args) != 1 {
|
||||
return node
|
||||
}
|
||||
node = pipe.Cmds[0].Args[0]
|
||||
}
|
||||
}
|
||||
|
||||
func (walker *requirementWalker) recordInput(name string, reach requirementReachability) {
|
||||
evidence := walker.inputs[name]
|
||||
if evidence == nil {
|
||||
evidence = &inputEvidence{
|
||||
name: name, kind: domain.RequirementOptional,
|
||||
explanation: "Declared as an optional input.",
|
||||
}
|
||||
walker.inputs[name] = evidence
|
||||
}
|
||||
if reach == requirementInactive {
|
||||
if !evidence.applicable && evidence.kind != domain.RequirementRequired {
|
||||
evidence.explanation = "This input is referenced only in a skipped template path."
|
||||
}
|
||||
} else {
|
||||
evidence.applicable = true
|
||||
walker.promoteInput(evidence, kindForReach(reach))
|
||||
if evidence.kind == domain.RequirementRequired {
|
||||
evidence.explanation = ""
|
||||
} else if reach == requirementPossible {
|
||||
evidence.explanation = "This reference is not known to be reached on every execution path."
|
||||
}
|
||||
}
|
||||
if reach != requirementInactive {
|
||||
walker.dependencies++
|
||||
}
|
||||
}
|
||||
|
||||
func (walker *requirementWalker) recordVariable(name string, reach requirementReachability) {
|
||||
evidence := walker.variables[name]
|
||||
if evidence == nil {
|
||||
evidence = &variableEvidence{name: name, kind: domain.RequirementOptional}
|
||||
walker.variables[name] = evidence
|
||||
}
|
||||
if reach == requirementInactive {
|
||||
if !evidence.applicable {
|
||||
evidence.explanation = "This variable is referenced only in a skipped template path."
|
||||
}
|
||||
} else {
|
||||
evidence.applicable = true
|
||||
walker.promoteVariable(evidence, kindForReach(reach))
|
||||
if evidence.kind == domain.RequirementRequired {
|
||||
evidence.explanation = ""
|
||||
} else if reach == requirementPossible {
|
||||
evidence.explanation = "This reference is not known to be reached on every execution path."
|
||||
}
|
||||
}
|
||||
if reach != requirementInactive {
|
||||
walker.dependencies++
|
||||
}
|
||||
}
|
||||
|
||||
func (walker *requirementWalker) recordOptionalVariable(name string, reach requirementReachability) {
|
||||
evidence := walker.variables[name]
|
||||
if evidence == nil {
|
||||
evidence = &variableEvidence{name: name, kind: domain.RequirementOptional}
|
||||
walker.variables[name] = evidence
|
||||
}
|
||||
if reach == requirementInactive {
|
||||
if !evidence.applicable {
|
||||
evidence.explanation = "This variable is referenced only in a skipped template path."
|
||||
}
|
||||
} else {
|
||||
evidence.applicable = true
|
||||
}
|
||||
if evidence.kind == domain.RequirementOptional && reach != requirementInactive {
|
||||
evidence.explanation = "This key is read through an optional map lookup."
|
||||
}
|
||||
if reach != requirementInactive {
|
||||
walker.dependencies++
|
||||
}
|
||||
}
|
||||
|
||||
func (walker *requirementWalker) promoteInput(evidence *inputEvidence, kind domain.RequirementKind) {
|
||||
if requirementRank(kind) > requirementRank(evidence.kind) {
|
||||
evidence.kind = kind
|
||||
}
|
||||
if evidence.kind == domain.RequirementRequired {
|
||||
evidence.explanation = ""
|
||||
}
|
||||
}
|
||||
|
||||
func (walker *requirementWalker) promoteVariable(evidence *variableEvidence, kind domain.RequirementKind) {
|
||||
if requirementRank(kind) > requirementRank(evidence.kind) {
|
||||
evidence.kind = kind
|
||||
}
|
||||
if evidence.kind == domain.RequirementRequired {
|
||||
evidence.explanation = ""
|
||||
}
|
||||
}
|
||||
|
||||
func kindForReach(reach requirementReachability) domain.RequirementKind {
|
||||
if reach == requirementPossible {
|
||||
return domain.RequirementConditional
|
||||
}
|
||||
return domain.RequirementRequired
|
||||
}
|
||||
|
||||
func requirementRank(kind domain.RequirementKind) int {
|
||||
switch kind {
|
||||
case domain.RequirementRequired:
|
||||
return 3
|
||||
case domain.RequirementConditional:
|
||||
return 2
|
||||
default:
|
||||
return 1
|
||||
}
|
||||
}
|
||||
|
||||
func (walker *requirementWalker) addIssue(inputName, variableName, description string) {
|
||||
walker.addIssueCode(domain.RequirementIssueUnsupportedConstruct, inputName, variableName, description)
|
||||
}
|
||||
|
||||
func (walker *requirementWalker) addIssueCode(code domain.RequirementIssueCode, inputName, variableName, description string) {
|
||||
walker.complete = false
|
||||
issue := domain.RequirementIssue{
|
||||
Code: code,
|
||||
InputName: inputName,
|
||||
VariableName: variableName,
|
||||
Description: description,
|
||||
}
|
||||
key := requirementIssueKey{code: issue.Code, inputName: inputName, variableName: variableName}
|
||||
walker.issues[key] = issue
|
||||
}
|
||||
|
||||
func (walker *requirementWalker) recordConditionLimitation(condition conditionEvaluation) {
|
||||
for name := range condition.missingVariables {
|
||||
walker.addIssueCode(domain.RequirementIssueMissingVariable, "", name, "A missing variable prevents selecting a dependency-bearing branch.")
|
||||
}
|
||||
for name := range condition.contentInputs {
|
||||
walker.addIssueCode(domain.RequirementIssueInputContent, name, "", "Captured input content affects branch selection.")
|
||||
}
|
||||
if condition.unsupported && !condition.inputNameUnknown ||
|
||||
(len(condition.missingVariables) == 0 && len(condition.contentInputs) == 0 && !condition.inputNameUnknown) {
|
||||
walker.addIssue("", "", "A dependency-bearing branch condition is not resolved by the supported syntax analysis.")
|
||||
}
|
||||
}
|
||||
|
||||
func (walker *requirementWalker) result(result domain.RequirementInspection) domain.RequirementInspection {
|
||||
result.Complete = walker.complete
|
||||
result.Inputs = make([]domain.InputRequirement, 0, len(walker.inputs))
|
||||
for _, evidence := range walker.inputs {
|
||||
if walker.unknownInputEffect && evidence.kind != domain.RequirementRequired {
|
||||
evidence.kind = domain.RequirementConditional
|
||||
evidence.explanation = "An unresolved input name could refer to this input."
|
||||
}
|
||||
result.Inputs = append(result.Inputs, domain.InputRequirement{
|
||||
Name: evidence.name, Kind: evidence.kind, Declared: evidence.declared,
|
||||
ContentType: copyString(evidence.contentType), Description: copyString(evidence.description),
|
||||
Explanation: evidence.explanation,
|
||||
})
|
||||
}
|
||||
sort.Slice(result.Inputs, func(i, j int) bool { return result.Inputs[i].Name < result.Inputs[j].Name })
|
||||
result.Variables = make([]domain.VariableRequirement, 0, len(walker.variables))
|
||||
for _, evidence := range walker.variables {
|
||||
if walker.unknownVariableEffect && evidence.kind != domain.RequirementRequired {
|
||||
evidence.kind = domain.RequirementConditional
|
||||
evidence.explanation = "An unresolved caller-variable scope could refer to this variable."
|
||||
}
|
||||
_, supplied := walker.vars[evidence.name]
|
||||
result.Variables = append(result.Variables, domain.VariableRequirement{
|
||||
Name: evidence.name, Kind: evidence.kind, Supplied: supplied, Explanation: evidence.explanation,
|
||||
})
|
||||
}
|
||||
sort.Slice(result.Variables, func(i, j int) bool { return result.Variables[i].Name < result.Variables[j].Name })
|
||||
result.Issues = make([]domain.RequirementIssue, 0, len(walker.issues))
|
||||
for _, issue := range walker.issues {
|
||||
result.Issues = append(result.Issues, issue)
|
||||
}
|
||||
sort.Slice(result.Issues, func(i, j int) bool {
|
||||
left, right := result.Issues[i], result.Issues[j]
|
||||
if left.Code != right.Code {
|
||||
return left.Code < right.Code
|
||||
}
|
||||
if left.InputName != right.InputName {
|
||||
return left.InputName < right.InputName
|
||||
}
|
||||
return left.VariableName < right.VariableName
|
||||
})
|
||||
return result
|
||||
}
|
||||
|
||||
func copyString(value *string) *string {
|
||||
if value == nil {
|
||||
return nil
|
||||
}
|
||||
copy := *value
|
||||
return ©
|
||||
}
|
||||
671
internal/prompt/requirements_test.go
Normal file
671
internal/prompt/requirements_test.go
Normal file
@@ -0,0 +1,671 @@
|
||||
package prompt
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"text/template"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
)
|
||||
|
||||
func TestAnalyzeRequirementsDoesNotHideCallerMapScopes(t *testing.T) {
|
||||
for _, source := range []string{
|
||||
`{{with .}}{{.audience}}{{end}}`,
|
||||
`{{with $}}{{.audience}}{{end}}`,
|
||||
`{{with (.)}}{{.audience}}{{end}}`,
|
||||
`{{$alias := .}}{{with $alias}}{{.audience}}{{end}}`,
|
||||
`{{with .}}{{index . "audience"}}{{end}}`,
|
||||
`{{$alias := .}}{{index $alias "audience"}}`,
|
||||
`{{$alias := .}}{{index ($alias) "audience"}}`,
|
||||
`{{$alias := .}}{{$alias.audience}}`,
|
||||
} {
|
||||
t.Run(source, func(t *testing.T) {
|
||||
compiled := compileRequirementTemplate(t, "scope", source)
|
||||
vars := map[string]string{"audience": "visible"}
|
||||
var rendered strings.Builder
|
||||
if err := compiled.Execute(&rendered, vars); err != nil || rendered.String() != "visible" {
|
||||
t.Fatalf("caller-map reference did not render: %q, %v", rendered.String(), err)
|
||||
}
|
||||
result, err := AnalyzeRequirements(context.Background(), &domain.PromptDefinition{}, RequirementSnapshot{
|
||||
Templates: CompiledTemplates{Messages: []*template.Template{compiled}},
|
||||
}, vars, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result.Complete {
|
||||
if len(result.Variables) != 1 || result.Variables[0].Name != "audience" {
|
||||
t.Fatalf("complete discovery lost the caller key: %+v", result)
|
||||
}
|
||||
} else if len(result.Issues) == 0 {
|
||||
t.Fatalf("partial discovery did not explain its limitation: %+v", result)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnalyzeRequirementsUnknownScopesAffectOptionalEntriesOnlyWhenActive(t *testing.T) {
|
||||
for _, access := range []string{`{{with $alias}}{{.audience}}{{end}}`, `{{index $alias "audience"}}`, `{{index ($alias) "audience"}}`, `{{$alias.audience}}`} {
|
||||
for _, active := range []bool{false, true} {
|
||||
source := `{{if false}}{{.optional}}{{end}}{{$alias := .}}` + access
|
||||
if !active {
|
||||
source = `{{if false}}` + source + `{{end}}`
|
||||
}
|
||||
compiled := compileRequirementTemplate(t, "unknown-scope", source)
|
||||
result, err := AnalyzeRequirements(context.Background(), &domain.PromptDefinition{}, RequirementSnapshot{
|
||||
Templates: CompiledTemplates{Messages: []*template.Template{compiled}},
|
||||
}, map[string]string{"audience": "visible"}, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wantKind := domain.RequirementOptional
|
||||
if active {
|
||||
wantKind = domain.RequirementConditional
|
||||
}
|
||||
if result.Complete == active || len(result.Variables) != 1 || result.Variables[0].Name != "optional" || result.Variables[0].Kind != wantKind {
|
||||
t.Fatalf("active=%v, source=%s, result=%+v", active, source, result)
|
||||
}
|
||||
if active && (len(result.Issues) == 0 || result.Issues[0].Code != domain.RequirementIssueUnsupportedConstruct) {
|
||||
t.Fatalf("active unknown scope lacks a structured issue: %+v", result)
|
||||
}
|
||||
if !active && len(result.Issues) != 0 {
|
||||
t.Fatalf("skipped unknown scope introduced uncertainty: %+v", result)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRequirementWalkerDoesNotRescanNestedUnsupportedNames(t *testing.T) {
|
||||
for _, depth := range []int{6, 12} {
|
||||
expression := ".selector"
|
||||
for level := 0; level < depth; level++ {
|
||||
expression = `input (printf "%s" (` + expression + `))`
|
||||
}
|
||||
source := "{{" + expression + "}}"
|
||||
compiled := compileRequirementTemplate(t, "computed-name", source)
|
||||
walker := &requirementWalker{
|
||||
ctx: context.Background(), vars: map[string]string{"selector": "binding"},
|
||||
inputs: make(map[string]*inputEvidence), variables: make(map[string]*variableEvidence),
|
||||
issues: make(map[requirementIssueKey]domain.RequirementIssue), complete: true, rootKnown: true,
|
||||
}
|
||||
if err := walker.walkTemplate(compiled); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// A generous source-relative bound detects exponential rescanning, not
|
||||
// an exact visit count or a production analysis limit.
|
||||
if walker.dependencies > len(source)*8 {
|
||||
t.Fatalf("depth=%d: %d accesses for %d bytes of source", depth, walker.dependencies, len(source))
|
||||
}
|
||||
result := walker.result(domain.RequirementInspection{})
|
||||
if result.Complete || len(result.Variables) != 1 || result.Variables[0].Name != "selector" ||
|
||||
result.Variables[0].Kind != domain.RequirementConditional || len(result.Issues) == 0 {
|
||||
t.Fatalf("nested unsupported names lost discovery metadata: %+v", result)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnalyzeRequirementsCollectsDeclaredAndLiteralInputs(t *testing.T) {
|
||||
description := "retained metadata"
|
||||
definition := &domain.PromptDefinition{ID: "capture", Version: "v1"}
|
||||
snapshot := RequirementSnapshot{
|
||||
Inputs: []InputMetadata{
|
||||
{Name: "optional", ContentType: stringPointerForTest("text/plain"), Description: &description},
|
||||
{Name: "unused", Required: true},
|
||||
{Name: "transcript", Required: true},
|
||||
},
|
||||
Templates: CompiledTemplates{Messages: []*template.Template{
|
||||
compileRequirementTemplate(t, "message", `{{input "transcript"}} {{input "extra"}} {{input "extra"}}`),
|
||||
}},
|
||||
}
|
||||
result, err := AnalyzeRequirements(context.Background(), definition, snapshot, nil, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result.PromptID != "capture" || result.PromptVersion != "v1" || !result.Complete || len(result.Issues) != 0 {
|
||||
t.Fatalf("unexpected result identity or completeness: %+v", result)
|
||||
}
|
||||
wantNames := []string{"extra", "optional", "transcript", "unused"}
|
||||
if got := requirementInputNames(result.Inputs); !reflect.DeepEqual(got, wantNames) {
|
||||
t.Fatalf("input names = %v, want %v", got, wantNames)
|
||||
}
|
||||
if got := result.Inputs[0]; got.Kind != domain.RequirementRequired || got.Declared || got.ContentType != nil || got.Description != nil {
|
||||
t.Fatalf("undeclared literal input was not required independently: %+v", got)
|
||||
}
|
||||
if got := result.Inputs[1]; got.Kind != domain.RequirementOptional || !got.Declared || got.ContentType == nil || *got.ContentType != "text/plain" ||
|
||||
got.Description == nil || *got.Description != description || got.Explanation == "" {
|
||||
t.Fatalf("optional declaration metadata was not retained: %+v", got)
|
||||
}
|
||||
if result.Inputs[2].Kind != domain.RequirementRequired || !result.Inputs[2].Declared || result.Inputs[3].Kind != domain.RequirementRequired {
|
||||
t.Fatalf("required declarations or literal use were lost: %+v", result.Inputs)
|
||||
}
|
||||
*result.Inputs[1].Description = "caller mutation"
|
||||
if *snapshot.Inputs[0].Description != description {
|
||||
t.Fatal("result metadata pointer aliases the captured snapshot")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnalyzeRequirementsDistinguishesDirectAndOptionalCallerKeys(t *testing.T) {
|
||||
result, err := AnalyzeRequirements(context.Background(), &domain.PromptDefinition{}, RequirementSnapshot{
|
||||
Templates: CompiledTemplates{Messages: []*template.Template{
|
||||
compileRequirementTemplate(t, "message", `{{.missing}} {{$.present}} {{index . "lookup"}}`),
|
||||
}},
|
||||
}, map[string]string{"present": ""}, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !result.Complete || len(result.Issues) != 0 {
|
||||
t.Fatalf("plain root and literal lookup syntax should be complete: %+v", result)
|
||||
}
|
||||
wantNames := []string{"lookup", "missing", "present"}
|
||||
if got := requirementVariableNames(result.Variables); !reflect.DeepEqual(got, wantNames) {
|
||||
t.Fatalf("variable names = %v, want %v", got, wantNames)
|
||||
}
|
||||
if result.Variables[0].Kind != domain.RequirementOptional || result.Variables[0].Supplied || result.Variables[0].Explanation == "" {
|
||||
t.Errorf("literal index lookup should be optional and absent: %+v", result.Variables[0])
|
||||
}
|
||||
if result.Variables[1].Kind != domain.RequirementRequired || result.Variables[1].Supplied {
|
||||
t.Errorf("missing direct variable should be required but not supplied: %+v", result.Variables[1])
|
||||
}
|
||||
if result.Variables[2].Kind != domain.RequirementRequired || !result.Variables[2].Supplied {
|
||||
t.Errorf("empty supplied direct variable was not detected by presence: %+v", result.Variables[2])
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnalyzeRequirementsSeparatesRootAndLocalTemplateValues(t *testing.T) {
|
||||
result, err := AnalyzeRequirements(context.Background(), &domain.PromptDefinition{}, RequirementSnapshot{
|
||||
Templates: CompiledTemplates{Messages: []*template.Template{
|
||||
compileRequirementTemplate(t, "message", `{{$local := "local"}}{{with .context}}{{$local}} {{.child}} {{$.root}}{{end}}`),
|
||||
}},
|
||||
}, nil, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wantNames := []string{"context", "root"}
|
||||
if got := requirementVariableNames(result.Variables); !reflect.DeepEqual(got, wantNames) {
|
||||
t.Fatalf("root variable names = %v, want %v", got, wantNames)
|
||||
}
|
||||
if result.Variables[0].Kind != domain.RequirementRequired || result.Variables[1].Kind != domain.RequirementConditional || result.Complete {
|
||||
t.Fatalf("with scope and unresolved branch should be conservative: %+v", result)
|
||||
}
|
||||
if len(result.Issues) != 1 || result.Issues[0].Code != domain.RequirementIssueMissingVariable || result.Issues[0].VariableName != "context" {
|
||||
t.Fatalf("conditional root use should have one associated limitation: %+v", result.Issues)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnalyzeRequirementsMarksBranchAndUnsupportedScopeReferences(t *testing.T) {
|
||||
result, err := AnalyzeRequirements(context.Background(), &domain.PromptDefinition{}, RequirementSnapshot{
|
||||
Templates: CompiledTemplates{Messages: []*template.Template{
|
||||
compileRequirementTemplate(t, "message", `{{if .enabled}}{{input "maybe"}}{{end}}{{range .items}}{{input "inside"}} {{.local}} {{$.root}}{{end}}`),
|
||||
}},
|
||||
}, nil, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wantInputs := []string{"inside", "maybe"}
|
||||
wantVars := []string{"enabled", "items", "root"}
|
||||
if got := requirementInputNames(result.Inputs); !reflect.DeepEqual(got, wantInputs) {
|
||||
t.Fatalf("input names = %v, want %v", got, wantInputs)
|
||||
}
|
||||
if got := requirementVariableNames(result.Variables); !reflect.DeepEqual(got, wantVars) {
|
||||
t.Fatalf("variable names = %v, want %v", got, wantVars)
|
||||
}
|
||||
if result.Complete || len(result.Issues) < 2 {
|
||||
t.Fatalf("branch/range limitations were not reported: %+v", result)
|
||||
}
|
||||
for _, input := range result.Inputs {
|
||||
if input.Kind != domain.RequirementConditional {
|
||||
t.Errorf("input %q should be conservative conditional: %+v", input.Name, input)
|
||||
}
|
||||
}
|
||||
if result.Variables[0].Kind != domain.RequirementRequired || result.Variables[1].Kind != domain.RequirementRequired ||
|
||||
result.Variables[2].Kind != domain.RequirementConditional {
|
||||
t.Errorf("root and loop-scope variables were misclassified: %+v", result.Variables)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnalyzeRequirementsExcludesUnusedDefinitionsAndBypassedSession(t *testing.T) {
|
||||
message := compileRequirementTemplate(t, "message", `{{define "unused"}}{{input "unused"}}{{end}}main`)
|
||||
session := compileRequirementTemplate(t, "session", `{{input "session_only"}}`)
|
||||
result, err := AnalyzeRequirements(context.Background(), &domain.PromptDefinition{}, RequirementSnapshot{
|
||||
Templates: CompiledTemplates{Messages: []*template.Template{message}, Session: session},
|
||||
}, nil, true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !result.Complete || len(result.Inputs) != 0 {
|
||||
t.Fatalf("unused definitions or bypassed session were included: %+v", result)
|
||||
}
|
||||
result, err = AnalyzeRequirements(context.Background(), &domain.PromptDefinition{}, RequirementSnapshot{
|
||||
Templates: CompiledTemplates{Messages: []*template.Template{message}, Session: session},
|
||||
}, nil, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(result.Inputs) != 1 || result.Inputs[0].Name != "session_only" || result.Inputs[0].Kind != domain.RequirementRequired {
|
||||
t.Fatalf("applicable session template was not visited: %+v", result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnalyzeRequirementsScansInvokedDefinitionsConservatively(t *testing.T) {
|
||||
message := compileRequirementTemplate(t, "message", `{{define "unused"}}{{input "unused"}}{{end}}{{define "called"}}{{input "called"}} {{.local}}{{end}}main {{template "called" .}}`)
|
||||
result, err := AnalyzeRequirements(context.Background(), &domain.PromptDefinition{}, RequirementSnapshot{
|
||||
Templates: CompiledTemplates{Messages: []*template.Template{message}},
|
||||
}, nil, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result.Complete || len(result.Inputs) != 1 || result.Inputs[0].Name != "called" ||
|
||||
result.Inputs[0].Kind != domain.RequirementConditional || len(result.Variables) != 0 {
|
||||
t.Fatalf("called definition was not scanned conservatively: %+v", result)
|
||||
}
|
||||
if len(result.Issues) < 1 || result.Issues[0].Code != domain.RequirementIssueUnsupportedConstruct {
|
||||
t.Fatalf("named-template limitation was not reported: %+v", result.Issues)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnalyzeRequirementsClassifiesKnownConditions(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
condition string
|
||||
vars map[string]string
|
||||
wantActive string
|
||||
wantSkipped string
|
||||
}{
|
||||
{name: "string false is truthy", condition: `.flag`, vars: map[string]string{"flag": "false"}, wantActive: "active", wantSkipped: "inactive"},
|
||||
{name: "empty direct value is false", condition: `.flag`, vars: map[string]string{"flag": ""}, wantActive: "inactive", wantSkipped: "active"},
|
||||
{name: "simple equality", condition: `eq .mode "active"`, vars: map[string]string{"mode": "active"}, wantActive: "active", wantSkipped: "inactive"},
|
||||
{name: "simple inequality", condition: `ne .mode "active"`, vars: map[string]string{"mode": "other"}, wantActive: "active", wantSkipped: "inactive"},
|
||||
{name: "not direct value", condition: `not .flag`, vars: map[string]string{"flag": ""}, wantActive: "active", wantSkipped: "inactive"},
|
||||
{name: "literal true", condition: `true`, wantActive: "active", wantSkipped: "inactive"},
|
||||
{name: "literal false", condition: `false`, wantActive: "inactive", wantSkipped: "active"},
|
||||
{name: "missing index is empty", condition: `index . "toggle"`, wantActive: "inactive", wantSkipped: "active"},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
definition := &domain.PromptDefinition{Templates: []domain.PromptMessageTemplate{{
|
||||
Role: "user", Content: `{{if ` + test.condition + `}}{{input "active"}}{{else}}{{input "inactive"}}{{end}}`,
|
||||
}}}
|
||||
snapshot := RequirementSnapshot{Templates: CompiledTemplates{Messages: []*template.Template{
|
||||
compileRequirementTemplate(t, "message", definition.Templates[0].Content),
|
||||
}}}
|
||||
result, err := AnalyzeRequirements(context.Background(), definition, snapshot, test.vars, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !result.Complete || len(result.Issues) != 0 {
|
||||
t.Fatalf("known condition should produce a complete result: %+v", result)
|
||||
}
|
||||
inputs := make(map[string]domain.InputRequirement, len(result.Inputs))
|
||||
for _, input := range result.Inputs {
|
||||
inputs[input.Name] = input
|
||||
}
|
||||
if inputs[test.wantActive].Kind != domain.RequirementRequired || inputs[test.wantSkipped].Kind != domain.RequirementOptional || inputs[test.wantSkipped].Explanation == "" {
|
||||
t.Fatalf("selected/skipped inputs were misclassified: %+v", result.Inputs)
|
||||
}
|
||||
if test.condition == `index . "toggle"` {
|
||||
if len(result.Variables) != 1 || result.Variables[0].Name != "toggle" || result.Variables[0].Kind != domain.RequirementOptional || result.Variables[0].Supplied {
|
||||
t.Fatalf("missing index key was not retained as optional: %+v", result.Variables)
|
||||
}
|
||||
}
|
||||
|
||||
artifacts := map[string]*domain.Artifact{
|
||||
"active": {Name: "active", Body: []byte("active")},
|
||||
"inactive": {Name: "inactive", Body: []byte("inactive")},
|
||||
}
|
||||
rendered, err := NewGoRenderer().Render(context.Background(), definition, artifacts, test.vars)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
selected := "active"
|
||||
if inputs[selected].Kind != domain.RequirementRequired {
|
||||
selected = "inactive"
|
||||
}
|
||||
if len(rendered.Messages) != 1 || rendered.Messages[0].Content != selected {
|
||||
t.Fatalf("analysis and renderer disagree: result=%+v rendered=%+v", result, rendered)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnalyzeRequirementsHandlesWithDotAndNestedReachability(t *testing.T) {
|
||||
definition := &domain.PromptDefinition{Templates: []domain.PromptMessageTemplate{{
|
||||
Role: "user", Content: `{{with .context}}{{input "inside"}} {{.child}} {{$.root}}{{else}}{{input "empty"}}{{end}}`,
|
||||
}}}
|
||||
snapshot := RequirementSnapshot{Templates: CompiledTemplates{Messages: []*template.Template{
|
||||
compileRequirementTemplate(t, "message", definition.Templates[0].Content),
|
||||
}}}
|
||||
result, err := AnalyzeRequirements(context.Background(), definition, snapshot, map[string]string{"context": "value", "root": "root"}, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !result.Complete || len(result.Inputs) != 2 || len(result.Variables) != 2 {
|
||||
t.Fatalf("known with condition should have a complete inventory: %+v", result)
|
||||
}
|
||||
if result.Inputs[0].Name != "empty" || result.Inputs[0].Kind != domain.RequirementOptional ||
|
||||
result.Inputs[1].Name != "inside" || result.Inputs[1].Kind != domain.RequirementRequired {
|
||||
t.Fatalf("with body and else reachability is incorrect: %+v", result.Inputs)
|
||||
}
|
||||
if result.Variables[0].Name != "context" || result.Variables[0].Kind != domain.RequirementRequired ||
|
||||
result.Variables[1].Name != "root" || result.Variables[1].Kind != domain.RequirementRequired {
|
||||
t.Fatalf("with dot was confused with the root caller map: %+v", result.Variables)
|
||||
}
|
||||
|
||||
nested := compileRequirementTemplate(t, "nested", `{{if .outer}}{{if .inner}}{{input "inner"}}{{else}}{{input "inner_else"}}{{end}}{{else}}{{input "outer_else"}}{{end}}`)
|
||||
nestedResult, err := AnalyzeRequirements(context.Background(), &domain.PromptDefinition{}, RequirementSnapshot{
|
||||
Templates: CompiledTemplates{Messages: []*template.Template{nested}},
|
||||
}, map[string]string{"outer": "yes", "inner": "yes"}, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !nestedResult.Complete || nestedResult.Inputs[0].Name != "inner" || nestedResult.Inputs[0].Kind != domain.RequirementRequired ||
|
||||
nestedResult.Inputs[1].Name != "inner_else" || nestedResult.Inputs[1].Kind != domain.RequirementOptional ||
|
||||
nestedResult.Inputs[2].Name != "outer_else" || nestedResult.Inputs[2].Kind != domain.RequirementOptional {
|
||||
t.Fatalf("known nested conditions lost selected or inactive reachability: %+v", nestedResult)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnalyzeRequirementsReportsUnresolvedDecisionAndContentConditions(t *testing.T) {
|
||||
missing := compileRequirementTemplate(t, "missing", `{{if eq .mode "active"}}{{input "yes"}}{{else}}{{input "no"}}{{end}}`)
|
||||
missingResult, err := AnalyzeRequirements(context.Background(), &domain.PromptDefinition{}, RequirementSnapshot{
|
||||
Templates: CompiledTemplates{Messages: []*template.Template{missing}},
|
||||
}, nil, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if missingResult.Complete || len(missingResult.Issues) != 1 || missingResult.Issues[0].Code != domain.RequirementIssueMissingVariable ||
|
||||
missingResult.Issues[0].VariableName != "mode" || missingResult.Inputs[0].Kind != domain.RequirementConditional ||
|
||||
missingResult.Inputs[1].Kind != domain.RequirementConditional {
|
||||
t.Fatalf("missing decision value did not produce conditional dependencies: %+v", missingResult)
|
||||
}
|
||||
|
||||
content := compileRequirementTemplate(t, "content", `{{if input "selector"}}{{input "yes"}}{{else}}{{input "no"}}{{end}}`)
|
||||
contentResult, err := AnalyzeRequirements(context.Background(), &domain.PromptDefinition{}, RequirementSnapshot{
|
||||
Templates: CompiledTemplates{Messages: []*template.Template{content}},
|
||||
}, nil, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
contentInputs := make(map[string]domain.InputRequirement, len(contentResult.Inputs))
|
||||
for _, input := range contentResult.Inputs {
|
||||
contentInputs[input.Name] = input
|
||||
}
|
||||
if contentResult.Complete || len(contentResult.Issues) != 1 || contentResult.Issues[0].Code != domain.RequirementIssueInputContent ||
|
||||
contentResult.Issues[0].InputName != "selector" || contentInputs["selector"].Kind != domain.RequirementRequired ||
|
||||
contentInputs["yes"].Kind != domain.RequirementConditional || contentInputs["no"].Kind != domain.RequirementConditional {
|
||||
t.Fatalf("content-dependent condition did not preserve known dependencies: %+v", contentResult)
|
||||
}
|
||||
|
||||
textOnly := compileRequirementTemplate(t, "text-only", `{{if .missing}}plain text{{else}}also plain text{{end}}`)
|
||||
textResult, err := AnalyzeRequirements(context.Background(), &domain.PromptDefinition{}, RequirementSnapshot{
|
||||
Templates: CompiledTemplates{Messages: []*template.Template{textOnly}},
|
||||
}, nil, false)
|
||||
if err != nil || !textResult.Complete || len(textResult.Issues) != 0 {
|
||||
t.Fatalf("unresolved text-only branch should not affect completeness: %+v, %v", textResult, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnalyzeRequirementsRetainsInactiveEntriesAndPromotesActiveUses(t *testing.T) {
|
||||
description := "declared for a reason"
|
||||
definition := &domain.PromptDefinition{Templates: []domain.PromptMessageTemplate{{
|
||||
Role: "user", Content: `{{if false}}{{input "shared"}} {{.shared_var}} {{input "declared"}}{{input .dynamic}}{{and .left .right}}{{end}}{{input "shared"}} {{.shared_var}}`,
|
||||
}}}
|
||||
snapshot := RequirementSnapshot{
|
||||
Inputs: []InputMetadata{{Name: "declared", Required: true, Description: &description}},
|
||||
Templates: CompiledTemplates{Messages: []*template.Template{
|
||||
compileRequirementTemplate(t, "message", definition.Templates[0].Content),
|
||||
}},
|
||||
}
|
||||
result, err := AnalyzeRequirements(context.Background(), definition, snapshot, map[string]string{"shared_var": "present"}, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !result.Complete || len(result.Issues) != 0 {
|
||||
t.Fatalf("skipped unsupported expressions introduced uncertainty: %+v", result)
|
||||
}
|
||||
inputs := make(map[string]domain.InputRequirement, len(result.Inputs))
|
||||
for _, input := range result.Inputs {
|
||||
inputs[input.Name] = input
|
||||
}
|
||||
if len(inputs) != 2 || inputs["shared"].Kind != domain.RequirementRequired || inputs["shared"].Explanation != "" ||
|
||||
inputs["declared"].Kind != domain.RequirementRequired || inputs["declared"].Description == nil || *inputs["declared"].Description != description {
|
||||
t.Fatalf("active or declared requirements did not override inactivity: %+v", result.Inputs)
|
||||
}
|
||||
variables := make(map[string]domain.VariableRequirement, len(result.Variables))
|
||||
for _, variable := range result.Variables {
|
||||
variables[variable.Name] = variable
|
||||
}
|
||||
for _, name := range []string{"dynamic", "left", "right", "shared_var"} {
|
||||
if _, ok := variables[name]; !ok {
|
||||
t.Errorf("inactive caller key %q was not inventoried", name)
|
||||
}
|
||||
}
|
||||
if variables["shared_var"].Kind != domain.RequirementRequired || variables["shared_var"].Explanation != "" ||
|
||||
variables["dynamic"].Kind != domain.RequirementOptional || variables["dynamic"].Explanation == "" {
|
||||
t.Fatalf("inactive caller metadata was not promoted or explained: %+v", result.Variables)
|
||||
}
|
||||
|
||||
allOptional := compileRequirementTemplate(t, "optional", `{{index . "lookup"}}`)
|
||||
optionalResult, err := AnalyzeRequirements(context.Background(), &domain.PromptDefinition{}, RequirementSnapshot{
|
||||
Inputs: []InputMetadata{{Name: "optional"}}, Templates: CompiledTemplates{Messages: []*template.Template{allOptional}},
|
||||
}, nil, false)
|
||||
if err != nil || !optionalResult.Complete || len(optionalResult.Issues) != 0 || optionalResult.Inputs[0].Kind != domain.RequirementOptional ||
|
||||
optionalResult.Variables[0].Kind != domain.RequirementOptional {
|
||||
t.Fatalf("all-optional inventory should be complete without artifacts: %+v, %v", optionalResult, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnalyzeRequirementsResolvesSimpleInputNames(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
expression string
|
||||
vars map[string]string
|
||||
kind domain.RequirementKind
|
||||
supplied bool
|
||||
}{
|
||||
{name: "direct dot", expression: `{{input .name}}`, vars: map[string]string{"name": " chosen "}, kind: domain.RequirementRequired, supplied: true},
|
||||
{name: "explicit root", expression: `{{input $.name}}`, vars: map[string]string{"name": "chosen"}, kind: domain.RequirementRequired, supplied: true},
|
||||
{name: "parenthesized direct", expression: `{{input (.name)}}`, vars: map[string]string{"name": "chosen"}, kind: domain.RequirementRequired, supplied: true},
|
||||
{name: "literal lookup", expression: `{{input (index . "name")}}`, vars: map[string]string{"name": "chosen"}, kind: domain.RequirementOptional, supplied: true},
|
||||
{name: "direct pipeline", expression: `{{.name | input}}`, vars: map[string]string{"name": "chosen"}, kind: domain.RequirementRequired, supplied: true},
|
||||
{name: "empty supplied value", expression: `{{input .name}}`, vars: map[string]string{"name": ""}, kind: domain.RequirementRequired, supplied: true},
|
||||
{name: "missing lookup has empty zero value", expression: `{{input (index . "name")}}`, vars: nil, kind: domain.RequirementOptional, supplied: false},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
definition := &domain.PromptDefinition{Templates: []domain.PromptMessageTemplate{{Role: "user", Content: test.expression}}}
|
||||
snapshot := RequirementSnapshot{Templates: CompiledTemplates{Messages: []*template.Template{
|
||||
compileRequirementTemplate(t, "message", test.expression),
|
||||
}}}
|
||||
result, err := AnalyzeRequirements(context.Background(), definition, snapshot, test.vars, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
inputName := "chosen"
|
||||
if test.name == "direct dot" {
|
||||
inputName = " chosen "
|
||||
}
|
||||
if test.name == "empty supplied value" || test.name == "missing lookup has empty zero value" {
|
||||
inputName = ""
|
||||
}
|
||||
if !result.Complete || len(result.Issues) != 0 || len(result.Inputs) != 1 || result.Inputs[0].Name != inputName ||
|
||||
result.Inputs[0].Kind != domain.RequirementRequired {
|
||||
t.Fatalf("simple input name was not resolved exactly: %+v", result)
|
||||
}
|
||||
if len(result.Variables) != 1 || result.Variables[0].Name != "name" || result.Variables[0].Kind != test.kind || result.Variables[0].Supplied != test.supplied {
|
||||
t.Fatalf("name-source caller key classification is incorrect: %+v", result.Variables)
|
||||
}
|
||||
artifacts := map[string]*domain.Artifact{inputName: {Name: inputName, Body: []byte("resolved")}}
|
||||
rendered, err := NewGoRenderer().Render(context.Background(), definition, artifacts, test.vars)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(rendered.Messages) != 1 || rendered.Messages[0].Content != "resolved" {
|
||||
t.Fatalf("analyzed input name differs from renderer: %+v", rendered)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnalyzeRequirementsReportsUnresolvedInputNamesAndPromotesUnknownEffects(t *testing.T) {
|
||||
definition := &domain.PromptDefinition{Templates: []domain.PromptMessageTemplate{{
|
||||
Role: "user", Content: `{{input .missing}}`,
|
||||
}}}
|
||||
result, err := AnalyzeRequirements(context.Background(), definition, RequirementSnapshot{
|
||||
Inputs: []InputMetadata{{Name: "optional"}, {Name: "required", Required: true}},
|
||||
Templates: CompiledTemplates{Messages: []*template.Template{
|
||||
compileRequirementTemplate(t, "message", definition.Templates[0].Content),
|
||||
}},
|
||||
}, nil, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result.Complete || len(result.Issues) != 2 || result.Issues[0].Code != domain.RequirementIssueDynamicInputName ||
|
||||
result.Issues[1].Code != domain.RequirementIssueMissingVariable {
|
||||
t.Fatalf("missing dynamic name did not report its limitations: %+v", result)
|
||||
}
|
||||
if result.Inputs[0].Name != "optional" || result.Inputs[0].Kind != domain.RequirementConditional || result.Inputs[0].Explanation == "" ||
|
||||
result.Inputs[1].Name != "required" || result.Inputs[1].Kind != domain.RequirementRequired || len(result.Variables) != 1 ||
|
||||
result.Variables[0].Name != "missing" || result.Variables[0].Kind != domain.RequirementRequired {
|
||||
t.Fatalf("unknown name effects lost known classifications: %+v", result)
|
||||
}
|
||||
|
||||
computed := compileRequirementTemplate(t, "computed", `{{input (printf "%s" .prefix)}}`)
|
||||
computedResult, err := AnalyzeRequirements(context.Background(), &domain.PromptDefinition{}, RequirementSnapshot{
|
||||
Inputs: []InputMetadata{{Name: "declared"}}, Templates: CompiledTemplates{Messages: []*template.Template{computed}},
|
||||
}, map[string]string{"prefix": "SENSITIVE-VALUE"}, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if computedResult.Complete || len(computedResult.Inputs) != 1 || computedResult.Inputs[0].Name != "declared" ||
|
||||
computedResult.Inputs[0].Kind != domain.RequirementConditional || len(computedResult.Variables) != 1 ||
|
||||
computedResult.Variables[0].Name != "prefix" || computedResult.Variables[0].Kind != domain.RequirementConditional {
|
||||
t.Fatalf("computed-name uncertainty did not affect optional entries: %+v", computedResult)
|
||||
}
|
||||
for _, issue := range computedResult.Issues {
|
||||
if strings.Contains(issue.Description, "SENSITIVE-VALUE") || issue.InputName == "SENSITIVE-VALUE" || issue.VariableName == "SENSITIVE-VALUE" {
|
||||
t.Fatalf("input-name source value leaked into diagnostics: %+v", issue)
|
||||
}
|
||||
}
|
||||
for _, input := range computedResult.Inputs {
|
||||
if strings.Contains(input.Explanation, "SENSITIVE-VALUE") {
|
||||
t.Fatalf("input-name source value leaked into an explanation: %+v", input)
|
||||
}
|
||||
}
|
||||
for _, variable := range computedResult.Variables {
|
||||
if strings.Contains(variable.Explanation, "SENSITIVE-VALUE") {
|
||||
t.Fatalf("input-name source value leaked into an explanation: %+v", variable)
|
||||
}
|
||||
}
|
||||
|
||||
for _, expression := range []string{`{{input (input "selector")}}`, `{{input "selector" | input}}`} {
|
||||
t.Run(expression, func(t *testing.T) {
|
||||
contentName := compileRequirementTemplate(t, "content-name", expression)
|
||||
contentResult, err := AnalyzeRequirements(context.Background(), &domain.PromptDefinition{}, RequirementSnapshot{
|
||||
Templates: CompiledTemplates{Messages: []*template.Template{contentName}},
|
||||
}, nil, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if contentResult.Complete || len(contentResult.Issues) != 2 ||
|
||||
contentResult.Issues[0].Code != domain.RequirementIssueDynamicInputName ||
|
||||
contentResult.Issues[1].Code != domain.RequirementIssueInputContent || contentResult.Issues[1].InputName != "selector" || len(contentResult.Inputs) != 1 ||
|
||||
contentResult.Inputs[0].Name != "selector" || contentResult.Inputs[0].Kind != domain.RequirementRequired {
|
||||
t.Fatalf("input-body-derived name did not preserve the known source dependency: %+v", contentResult)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnalyzeRequirementsDoesNotResolveSkippedComputedInputNames(t *testing.T) {
|
||||
compiled := compileRequirementTemplate(t, "inactive-name", `{{if false}}{{input .name}}{{input (printf "%s" .other)}}{{"piped-literal" | input}}{{template "missing" .arg}}{{end}}`)
|
||||
result, err := AnalyzeRequirements(context.Background(), &domain.PromptDefinition{}, RequirementSnapshot{
|
||||
Templates: CompiledTemplates{Messages: []*template.Template{compiled}},
|
||||
}, nil, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !result.Complete || len(result.Issues) != 0 || len(result.Inputs) != 1 || result.Inputs[0].Name != "piped-literal" ||
|
||||
result.Inputs[0].Kind != domain.RequirementOptional || result.Inputs[0].Explanation == "" || len(result.Variables) != 3 {
|
||||
t.Fatalf("skipped computed expressions introduced uncertainty or names: %+v", result)
|
||||
}
|
||||
for _, variable := range result.Variables {
|
||||
if variable.Kind != domain.RequirementOptional || variable.Explanation == "" {
|
||||
t.Errorf("skipped variable %q was not retained as inactive optional: %+v", variable.Name, variable)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnalyzeRequirementsPromotesOptionalVariablesWhenRootScopeIsUnknown(t *testing.T) {
|
||||
compiled := compileRequirementTemplate(t, "scope", `{{index . "optional"}} {{.required}} {{template "called" .}}{{define "called"}}{{$.hidden}}{{end}}`)
|
||||
result, err := AnalyzeRequirements(context.Background(), &domain.PromptDefinition{}, RequirementSnapshot{
|
||||
Templates: CompiledTemplates{Messages: []*template.Template{compiled}},
|
||||
}, map[string]string{"required": "value"}, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result.Complete || len(result.Variables) != 2 || result.Variables[0].Name != "optional" ||
|
||||
result.Variables[0].Kind != domain.RequirementConditional || result.Variables[0].Explanation == "" ||
|
||||
result.Variables[1].Name != "required" || result.Variables[1].Kind != domain.RequirementRequired {
|
||||
t.Fatalf("unknown caller scope did not affect only optional known entries: %+v", result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnalyzeRequirementsLeavesAssignmentDerivedNamesPartial(t *testing.T) {
|
||||
compiled := compileRequirementTemplate(t, "assignment", `{{$alias := .source}}{{input $alias}}`)
|
||||
result, err := AnalyzeRequirements(context.Background(), &domain.PromptDefinition{}, RequirementSnapshot{
|
||||
Inputs: []InputMetadata{{Name: "known-optional"}}, Templates: CompiledTemplates{Messages: []*template.Template{compiled}},
|
||||
}, nil, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result.Complete || len(result.Issues) == 0 || len(result.Inputs) != 1 ||
|
||||
result.Inputs[0].Name != "known-optional" || result.Inputs[0].Kind != domain.RequirementConditional ||
|
||||
len(result.Variables) != 1 || result.Variables[0].Name != "source" || result.Variables[0].Kind != domain.RequirementConditional {
|
||||
t.Fatalf("assignment-derived input name was interpreted or lost: %+v", result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnalyzeRequirementsSupportsEmptyTemplatesAndCancellation(t *testing.T) {
|
||||
result, err := AnalyzeRequirements(context.Background(), &domain.PromptDefinition{}, RequirementSnapshot{
|
||||
Templates: CompiledTemplates{Messages: []*template.Template{compileRequirementTemplate(t, "message", "plain text")}},
|
||||
}, nil, false)
|
||||
if err != nil || !result.Complete || len(result.Inputs) != 0 || len(result.Variables) != 0 {
|
||||
t.Fatalf("input-free template should have a complete empty inventory: %+v, %v", result, err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
if _, err := AnalyzeRequirements(ctx, &domain.PromptDefinition{}, RequirementSnapshot{}, nil, false); !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("canceled context returned %v, want context.Canceled", err)
|
||||
}
|
||||
}
|
||||
|
||||
func compileRequirementTemplate(t *testing.T, name, source string) *template.Template {
|
||||
t.Helper()
|
||||
compiled, err := CompileTemplate(name, source)
|
||||
if err != nil {
|
||||
t.Fatalf("compile template: %v", err)
|
||||
}
|
||||
return compiled
|
||||
}
|
||||
|
||||
func requirementInputNames(inputs []domain.InputRequirement) []string {
|
||||
names := make([]string, len(inputs))
|
||||
for index, input := range inputs {
|
||||
names[index] = input.Name
|
||||
}
|
||||
return names
|
||||
}
|
||||
|
||||
func requirementVariableNames(variables []domain.VariableRequirement) []string {
|
||||
names := make([]string, len(variables))
|
||||
for index, variable := range variables {
|
||||
names[index] = variable.Name
|
||||
}
|
||||
return names
|
||||
}
|
||||
|
||||
func stringPointerForTest(value string) *string {
|
||||
return &value
|
||||
}
|
||||
@@ -9,6 +9,22 @@ type CompiledTemplates struct {
|
||||
Messages []*template.Template
|
||||
}
|
||||
|
||||
// InputMetadata is the captured declaration metadata for one prompt input.
|
||||
// Nil metadata pointers mean the corresponding field was absent or null.
|
||||
type InputMetadata struct {
|
||||
Name string
|
||||
Required bool
|
||||
ContentType *string
|
||||
Description *string
|
||||
}
|
||||
|
||||
// RequirementSnapshot provides the captured syntax and declarations needed
|
||||
// for requirement inspection without reopening prompt resources.
|
||||
type RequirementSnapshot struct {
|
||||
Templates CompiledTemplates
|
||||
Inputs []InputMetadata
|
||||
}
|
||||
|
||||
// TemplateLookup provides parsed templates for a captured prompt identity.
|
||||
type TemplateLookup interface {
|
||||
GetCompiledTemplates(id, version string) (CompiledTemplates, bool)
|
||||
|
||||
@@ -50,10 +50,12 @@ func (r *Runner) PrepareExecution(ctx context.Context, req domain.RunRequest) (*
|
||||
return nil, err
|
||||
}
|
||||
|
||||
executionSnapshot, err := clonePreparedRun(operation.run)
|
||||
executionSnapshot := operation.run
|
||||
extraParams, err := jsonvalue.CopyMap(executionSnapshot.EffectiveModelParams.ExtraParams)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: failed to copy prepared execution: %v", ErrInvalidRequest, err)
|
||||
}
|
||||
executionSnapshot.EffectiveModelParams.ExtraParams = extraParams
|
||||
|
||||
details, err := clonePreparedRun(executionSnapshot)
|
||||
if err != nil {
|
||||
|
||||
@@ -631,6 +631,124 @@ func TestRunnerPreparedExecutionOwnershipUseAndDiscard(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerPreparedExecutionDetachesTransferredSnapshotSources(t *testing.T) {
|
||||
newExtras := func(value string) map[string]any {
|
||||
return map[string]any{"provider_options": map[string]any{
|
||||
"provider": map[string]any{"settings": map[string]any{"labels": []any{value}}},
|
||||
}}
|
||||
}
|
||||
extraLabel := func(extraParams map[string]any) string {
|
||||
return extraParams["provider_options"].(map[string]any)["provider"].(map[string]any)["settings"].(map[string]any)["labels"].([]any)[0].(string)
|
||||
}
|
||||
mutateExtra := func(extraParams map[string]any, value string) {
|
||||
extraParams["provider_options"].(map[string]any)["provider"].(map[string]any)["settings"].(map[string]any)["labels"].([]any)[0] = value
|
||||
}
|
||||
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
requestOverride bool
|
||||
}{
|
||||
{name: "profile target"},
|
||||
{name: "request override", requestOverride: true},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
profileExtras := newExtras("profile source")
|
||||
requestExtras := newExtras("request source")
|
||||
profile := defaultExecutionProfile()
|
||||
profile.ResolvedTarget.ExtraParams = profileExtras
|
||||
|
||||
schemaDocument := map[string]any{
|
||||
"type": "object",
|
||||
"properties": map[string]any{"value": map[string]any{"type": "string"}},
|
||||
}
|
||||
definition := promptDef(domain.FormatJSON, domain.ValidationJSONSchema, 0)
|
||||
definition.Validation.SchemaPath = "schema.json"
|
||||
control := &domain.CacheControl{Type: domain.CacheControlEphemeral, TTL: "1h"}
|
||||
rendered := &domain.RenderedPrompt{Messages: []domain.RenderedMessage{{
|
||||
Role: domain.RoleUser, Content: "captured rendering", CacheControl: control,
|
||||
}}}
|
||||
renderer := &fakeRenderer{rendered: rendered}
|
||||
sourceArtifact := &domain.Artifact{Body: []byte("captured input"), Hash: "captured-input-hash"}
|
||||
reader := &fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{"source": sourceArtifact}}
|
||||
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: `{"value":"ok"}`}}
|
||||
validator := &recordingValidationPreparer{
|
||||
plan: &recordingPreparedValidation{schemaDocument: schemaDocument},
|
||||
}
|
||||
runner := NewRunner(
|
||||
&fakePromptRepo{def: definition},
|
||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": profile}},
|
||||
reader, renderer, llmClient, validator, nil,
|
||||
)
|
||||
request := domain.RunRequest{
|
||||
PromptID: "p", PromptVersion: "1", ProfileID: "exec",
|
||||
Inputs: map[string]domain.ArtifactRef{
|
||||
"transcript": {Type: domain.ArtifactRefFile, URI: "source"},
|
||||
},
|
||||
}
|
||||
wantLabel := "profile source"
|
||||
if test.requestOverride {
|
||||
request.Execution = &domain.ExecutionTargetOverride{ExtraParams: requestExtras}
|
||||
wantLabel = "request source"
|
||||
}
|
||||
|
||||
prepared, err := runner.PrepareExecution(context.Background(), request)
|
||||
if err != nil {
|
||||
t.Fatalf("prepare execution: %v", err)
|
||||
}
|
||||
mutateExtra(profileExtras, "mutated profile source")
|
||||
mutateExtra(requestExtras, "mutated request source")
|
||||
rendered.Messages[0].Content = "mutated renderer source"
|
||||
control.TTL = "mutated renderer cache metadata"
|
||||
schemaDocument["properties"].(map[string]any)["value"].(map[string]any)["type"] = "integer"
|
||||
sourceArtifact.Hash = "mutated source hash"
|
||||
|
||||
details := prepared.Details()
|
||||
if details == nil || len(details.Messages) != 1 || details.Messages[0].Content != "captured rendering" ||
|
||||
details.Messages[0].CacheControl == nil || details.Messages[0].CacheControl.TTL != "1h" ||
|
||||
details.InputHashes["transcript"] != "captured-input-hash" ||
|
||||
extraLabel(details.EffectiveModelParams.ExtraParams) != wantLabel ||
|
||||
details.StructuredOutput == nil || details.StructuredOutput.JSONSchema == nil ||
|
||||
details.StructuredOutput.JSONSchema.Schema.(map[string]any)["properties"].(map[string]any)["value"].(map[string]any)["type"] != "string" {
|
||||
t.Fatalf("prepared details retained a source alias: %#v", details)
|
||||
}
|
||||
|
||||
result, err := runner.RunPrepared(context.Background(), prepared)
|
||||
if err != nil || result == nil || !result.Validation.IsValid || result.InputHashes["transcript"] != "captured-input-hash" {
|
||||
t.Fatalf("run prepared snapshot=(%+v, %v)", result, err)
|
||||
}
|
||||
if len(llmClient.lastReq.Prompt.Messages) != 1 || llmClient.lastReq.StructuredOutput == nil ||
|
||||
llmClient.lastReq.StructuredOutput.JSONSchema == nil ||
|
||||
extraLabel(llmClient.lastReq.Target.ExtraParams) != wantLabel ||
|
||||
llmClient.lastReq.Prompt.Messages[0].Content != "captured rendering" ||
|
||||
llmClient.lastReq.Prompt.Messages[0].CacheControl == nil || llmClient.lastReq.Prompt.Messages[0].CacheControl.TTL != "1h" ||
|
||||
llmClient.lastReq.StructuredOutput.JSONSchema.Schema.(map[string]any)["properties"].(map[string]any)["value"].(map[string]any)["type"] != "string" {
|
||||
t.Fatalf("execution snapshot retained a source alias: %#v", llmClient.lastReq)
|
||||
}
|
||||
if completedDetails := prepared.Details(); completedDetails == nil ||
|
||||
len(completedDetails.Messages) != 1 || completedDetails.Messages[0].Content != "captured rendering" ||
|
||||
extraLabel(completedDetails.EffectiveModelParams.ExtraParams) != wantLabel {
|
||||
t.Fatalf("completed details changed after source mutation: %#v", completedDetails)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerPrepareExecutionClassifiesExtraCopyFailureAsInvalidRequest(t *testing.T) {
|
||||
profile := defaultExecutionProfile()
|
||||
profile.ResolvedTarget.ExtraParams = map[string]any{"unsupported": func() {}}
|
||||
runner := NewRunner(
|
||||
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
|
||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": profile}},
|
||||
defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil, nil,
|
||||
)
|
||||
prepared, err := runner.PrepareExecution(context.Background(), domain.RunRequest{
|
||||
PromptID: "p", ProfileID: "exec",
|
||||
})
|
||||
if prepared != nil || !errors.Is(err, ErrInvalidRequest) {
|
||||
t.Fatalf("prepared execution copy failure=(%v, %v), want ErrInvalidRequest", prepared, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerPreparedExecutionWithoutValidatorSkipsValidation(t *testing.T) {
|
||||
runner := NewRunner(
|
||||
&fakePromptRepo{def: promptDef(domain.FormatJSON, domain.ValidationJSON, 0)},
|
||||
|
||||
237
internal/usecase/prepared_snapshot_benchmark_test.go
Normal file
237
internal/usecase/prepared_snapshot_benchmark_test.go
Normal file
@@ -0,0 +1,237 @@
|
||||
package usecase
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"runtime"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/validate"
|
||||
)
|
||||
|
||||
var preparedSnapshotHandleSink *PreparedExecution
|
||||
|
||||
const preparedSnapshotRetainedBatchSize = 4
|
||||
|
||||
type preparedSnapshotBenchmarkClient struct{}
|
||||
|
||||
func (preparedSnapshotBenchmarkClient) Generate(
|
||||
context.Context,
|
||||
domain.GenerateRequest,
|
||||
) (*domain.GenerateResponse, error) {
|
||||
return &domain.GenerateResponse{Content: "ok"}, nil
|
||||
}
|
||||
|
||||
type preparedSnapshotBenchmarkValidation struct{ schema any }
|
||||
|
||||
func (validation *preparedSnapshotBenchmarkValidation) Validate(
|
||||
context.Context,
|
||||
*domain.Artifact,
|
||||
) (domain.ValidationResult, error) {
|
||||
return domain.ValidationResult{Status: domain.ValidationPassed, IsValid: true}, nil
|
||||
}
|
||||
|
||||
func (validation *preparedSnapshotBenchmarkValidation) SchemaDocument() any {
|
||||
return validation.schema
|
||||
}
|
||||
|
||||
type preparedSnapshotBenchmarkValidator struct{ schema any }
|
||||
|
||||
func (validator preparedSnapshotBenchmarkValidator) Validate(
|
||||
context.Context,
|
||||
*domain.Artifact,
|
||||
domain.OutputContract,
|
||||
) (domain.ValidationResult, error) {
|
||||
return domain.ValidationResult{Status: domain.ValidationPassed, IsValid: true}, nil
|
||||
}
|
||||
|
||||
func (validator preparedSnapshotBenchmarkValidator) PrepareValidation(
|
||||
context.Context,
|
||||
domain.OutputContract,
|
||||
) (validate.PreparedValidation, error) {
|
||||
return &preparedSnapshotBenchmarkValidation{schema: validator.schema}, nil
|
||||
}
|
||||
|
||||
func preparedSnapshotBenchmarkSchema(properties int) map[string]any {
|
||||
fields := make(map[string]any, properties)
|
||||
for index := range properties {
|
||||
fields[fmt.Sprintf("property-%03d", index)] = map[string]any{"type": "string"}
|
||||
}
|
||||
return map[string]any{
|
||||
"type": "object", "properties": fields, "additionalProperties": false,
|
||||
}
|
||||
}
|
||||
|
||||
func preparedSnapshotBenchmarkExtras(entries int) map[string]any {
|
||||
providers := make(map[string]any, entries)
|
||||
for index := range entries {
|
||||
providers[fmt.Sprintf("provider-%03d", index)] = map[string]any{
|
||||
"enabled": true,
|
||||
"labels": []any{fmt.Sprintf("tier-%d", index%4), fmt.Sprintf("region-%d", index%3)},
|
||||
"limits": map[string]any{"burst": index + 1, "period": "minute"},
|
||||
"routes": []any{map[string]any{"weight": index + 1, "fallback": true}},
|
||||
}
|
||||
}
|
||||
return map[string]any{"provider_options": providers}
|
||||
}
|
||||
|
||||
func newPreparedSnapshotBenchmarkRunner(
|
||||
mutable bool,
|
||||
) (*Runner, domain.RunRequest, any) {
|
||||
messageCount := 1
|
||||
extraEntries := 0
|
||||
schemaProperties := 1
|
||||
if mutable {
|
||||
messageCount = 8
|
||||
extraEntries = 64
|
||||
schemaProperties = 256
|
||||
}
|
||||
|
||||
messages := make([]domain.RenderedMessage, messageCount)
|
||||
for index := range messages {
|
||||
messages[index] = domain.RenderedMessage{
|
||||
Role: domain.RoleUser, Content: fmt.Sprintf("rendered message %d", index),
|
||||
CacheControl: &domain.CacheControl{Type: domain.CacheControlEphemeral, TTL: "1h"},
|
||||
}
|
||||
}
|
||||
renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{
|
||||
SessionID: "snapshot-session", Messages: messages,
|
||||
}}
|
||||
|
||||
profile := defaultExecutionProfile()
|
||||
if extraEntries > 0 {
|
||||
profile.ResolvedTarget.ExtraParams = preparedSnapshotBenchmarkExtras(extraEntries)
|
||||
}
|
||||
profileRepository := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": profile}}
|
||||
definition := promptDef(domain.FormatJSON, domain.ValidationJSONSchema, 0)
|
||||
definition.Validation.SchemaPath = "snapshot-schema.json"
|
||||
schema := preparedSnapshotBenchmarkSchema(schemaProperties)
|
||||
validator := preparedSnapshotBenchmarkValidator{schema: schema}
|
||||
runner := NewRunner(
|
||||
&fakePromptRepo{def: definition},
|
||||
profileRepository,
|
||||
staticArtifactReader{},
|
||||
renderer,
|
||||
preparedSnapshotBenchmarkClient{},
|
||||
validator,
|
||||
nil,
|
||||
)
|
||||
request := domain.RunRequest{
|
||||
PromptID: "p", PromptVersion: "1", ProfileID: "exec",
|
||||
Inputs: singleInputRef(), Vars: map[string]string{"name": "Ada"},
|
||||
AppendedMessages: []domain.RenderedMessage{{
|
||||
Role: domain.RoleAssistant, Content: "appended message",
|
||||
CacheControl: &domain.CacheControl{Type: domain.CacheControlEphemeral, TTL: "1h"},
|
||||
}},
|
||||
}
|
||||
return runner, request, schema
|
||||
}
|
||||
|
||||
func checkPreparedSnapshotBenchmarkFixture(b *testing.B, runner *Runner, request domain.RunRequest, schema any) {
|
||||
b.Helper()
|
||||
prepared, err := runner.PrepareExecution(context.Background(), request)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
details := prepared.Details()
|
||||
if details == nil || details.PromptID != request.PromptID || len(details.Messages) < 2 ||
|
||||
details.Messages[0].Content != "rendered message 0" || details.Messages[len(details.Messages)-1].Content != "appended message" ||
|
||||
details.InputHashes["transcript"] == "" || details.StructuredOutput == nil || details.StructuredOutput.JSONSchema == nil {
|
||||
prepared.Discard()
|
||||
b.Fatalf("prepared snapshot fixture is incomplete: %#v", details)
|
||||
}
|
||||
if got := details.StructuredOutput.JSONSchema.Schema; !reflect.DeepEqual(got, schema) {
|
||||
prepared.Discard()
|
||||
b.Fatalf("prepared schema differs from fixture: %#v", got)
|
||||
}
|
||||
result, err := runner.RunPrepared(context.Background(), prepared)
|
||||
if err != nil || result == nil || result.RawOutput != "ok" || !result.Validation.IsValid {
|
||||
b.Fatalf("completed fixture=(%+v, %v)", result, err)
|
||||
}
|
||||
completedDetails := prepared.Details()
|
||||
if completedDetails == nil || completedDetails.Messages[0].Content != "rendered message 0" ||
|
||||
!reflect.DeepEqual(completedDetails.StructuredOutput.JSONSchema.Schema, schema) {
|
||||
b.Fatalf("completed handle did not retain its details: %#v", completedDetails)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkPreparedSnapshot(b *testing.B) {
|
||||
for _, fixture := range []struct {
|
||||
name string
|
||||
mutable bool
|
||||
}{
|
||||
{name: "small"},
|
||||
{name: "mutable_payload", mutable: true},
|
||||
} {
|
||||
b.Run(fixture.name, func(b *testing.B) {
|
||||
runner, request, schema := newPreparedSnapshotBenchmarkRunner(fixture.mutable)
|
||||
checkPreparedSnapshotBenchmarkFixture(b, runner, request, schema)
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for b.Loop() {
|
||||
prepared, err := runner.PrepareExecution(context.Background(), request)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
prepared.Discard()
|
||||
preparedSnapshotHandleSink = prepared
|
||||
}
|
||||
b.StopTimer()
|
||||
preparedSnapshotHandleSink = nil
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkRetainedPreparedHandles(b *testing.B) {
|
||||
for _, state := range []string{"ready", "completed", "discarded"} {
|
||||
b.Run(state, func(b *testing.B) {
|
||||
runner, request, schema := newPreparedSnapshotBenchmarkRunner(true)
|
||||
checkPreparedSnapshotBenchmarkFixture(b, runner, request, schema)
|
||||
retained := make([]*PreparedExecution, preparedSnapshotRetainedBatchSize)
|
||||
runtime.GC()
|
||||
var setupMemory runtime.MemStats
|
||||
runtime.ReadMemStats(&setupMemory)
|
||||
runtime.KeepAlive(runner)
|
||||
runtime.KeepAlive(request)
|
||||
runtime.KeepAlive(retained)
|
||||
|
||||
retainHandle := func() *PreparedExecution {
|
||||
prepared, err := runner.PrepareExecution(context.Background(), request)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
switch state {
|
||||
case "completed":
|
||||
result, err := runner.RunPrepared(context.Background(), prepared)
|
||||
if err != nil || result == nil || result.RawOutput != "ok" || !result.Validation.IsValid {
|
||||
b.Fatalf("complete retained handle=(%+v, %v)", result, err)
|
||||
}
|
||||
case "discarded":
|
||||
prepared.Discard()
|
||||
}
|
||||
return prepared
|
||||
}
|
||||
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for iteration := 0; iteration < b.N; iteration++ {
|
||||
retained[iteration%len(retained)] = retainHandle()
|
||||
}
|
||||
b.StopTimer()
|
||||
for index, prepared := range retained {
|
||||
if prepared == nil {
|
||||
retained[index] = retainHandle()
|
||||
}
|
||||
}
|
||||
runtime.GC()
|
||||
var retainedMemory runtime.MemStats
|
||||
runtime.ReadMemStats(&retainedMemory)
|
||||
b.ReportMetric(float64(int64(retainedMemory.HeapAlloc)-int64(setupMemory.HeapAlloc)), "retained-B")
|
||||
runtime.KeepAlive(runner)
|
||||
runtime.KeepAlive(request)
|
||||
runtime.KeepAlive(retained)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -15,16 +15,27 @@ type resolvedProfileSelection struct {
|
||||
profile *domain.ExecutionProfile
|
||||
}
|
||||
|
||||
type profileIdentityRepository interface {
|
||||
GetProfileIdentity(context.Context, string) (string, error)
|
||||
}
|
||||
|
||||
func (r *Runner) profileLookupID(profileID string) (string, error) {
|
||||
if strings.TrimSpace(profileID) == "" {
|
||||
return "", fmt.Errorf("%w: profile id is required", ErrInvalidRequest)
|
||||
}
|
||||
if r == nil || r.profiles == nil {
|
||||
return "", fmt.Errorf("%w: profile repository is not configured", ErrProfileLoad)
|
||||
}
|
||||
return profileID, nil
|
||||
}
|
||||
|
||||
func (r *Runner) resolveProfileSelection(
|
||||
ctx context.Context,
|
||||
profileID string,
|
||||
) (*resolvedProfileSelection, error) {
|
||||
lookupID := profileID
|
||||
if strings.TrimSpace(lookupID) == "" {
|
||||
return nil, fmt.Errorf("%w: profile id is required", ErrInvalidRequest)
|
||||
}
|
||||
if r == nil || r.profiles == nil {
|
||||
return nil, fmt.Errorf("%w: profile repository is not configured", ErrProfileLoad)
|
||||
lookupID, err := r.profileLookupID(profileID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
selectedProfile, err := r.profiles.GetProfile(ctx, lookupID)
|
||||
if err != nil {
|
||||
@@ -45,6 +56,25 @@ func (r *Runner) resolveProfileSelection(
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (r *Runner) resolveProfileIdentity(ctx context.Context, profileID string) (string, error) {
|
||||
lookupID, err := r.profileLookupID(profileID)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if indexed, ok := r.profiles.(profileIdentityRepository); ok {
|
||||
identity, err := indexed.GetProfileIdentity(ctx, lookupID)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("%w: %w", ErrProfileLoad, err)
|
||||
}
|
||||
return identity, nil
|
||||
}
|
||||
selection, err := r.resolveProfileSelection(ctx, lookupID)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return selection.id, nil
|
||||
}
|
||||
|
||||
func normalizeResolvedExecutionTarget(target domain.ExecutionTarget) (domain.ExecutionTarget, error) {
|
||||
endpoint, err := domain.NormalizeOpenAICompatibleBaseEndpoint(target.Endpoint)
|
||||
if err != nil {
|
||||
|
||||
@@ -13,6 +13,10 @@ type resolvedPromptDefinition struct {
|
||||
hash string
|
||||
}
|
||||
|
||||
type promptInspectionRepository interface {
|
||||
GetPromptInspection(context.Context, string, string) (*domain.PromptInspection, error)
|
||||
}
|
||||
|
||||
func (r *Runner) resolvePromptDefinition(
|
||||
ctx context.Context,
|
||||
promptID string,
|
||||
@@ -51,6 +55,43 @@ func (r *Runner) resolvePromptDefinition(
|
||||
return &resolvedPromptDefinition{definition: definition, hash: hash}, nil
|
||||
}
|
||||
|
||||
func (r *Runner) resolvePromptInspection(
|
||||
ctx context.Context,
|
||||
promptID string,
|
||||
promptVersion string,
|
||||
) (*domain.PromptInspection, error) {
|
||||
if strings.TrimSpace(promptID) == "" {
|
||||
return nil, fmt.Errorf("%w: prompt id is required", ErrInvalidRequest)
|
||||
}
|
||||
if r == nil || r.promptDefs == nil {
|
||||
return nil, fmt.Errorf("%w: prompt repository is not configured", ErrPromptLoad)
|
||||
}
|
||||
if indexed, ok := r.promptDefs.(promptInspectionRepository); ok {
|
||||
inspection, err := indexed.GetPromptInspection(ctx, promptID, promptVersion)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: %w", ErrPromptLoad, err)
|
||||
}
|
||||
if inspection == nil {
|
||||
return nil, fmt.Errorf("%w: prompt repository returned nil inspection", ErrPromptLoad)
|
||||
}
|
||||
return inspection, nil
|
||||
}
|
||||
|
||||
selection, err := r.resolvePromptDefinition(ctx, promptID, promptVersion)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
definition := selection.definition
|
||||
return &domain.PromptInspection{
|
||||
PromptID: definition.ID,
|
||||
PromptVersion: definition.Version,
|
||||
PromptHash: selection.hash,
|
||||
DefaultProfileID: definition.DefaultProfile,
|
||||
Inputs: append([]domain.PromptInput(nil), definition.Inputs...),
|
||||
OutputContract: definition.Validation,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// InspectPrompt resolves one explicit prompt without execution work.
|
||||
func (r *Runner) InspectPrompt(
|
||||
ctx context.Context,
|
||||
@@ -66,19 +107,5 @@ func (r *Runner) InspectPrompt(
|
||||
default:
|
||||
}
|
||||
|
||||
selection, err := r.resolvePromptDefinition(ctx, promptID, promptVersion)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
inputs := make([]domain.PromptInput, len(selection.definition.Inputs))
|
||||
copy(inputs, selection.definition.Inputs)
|
||||
|
||||
return &domain.PromptInspection{
|
||||
PromptID: selection.definition.ID,
|
||||
PromptVersion: selection.definition.Version,
|
||||
PromptHash: selection.hash,
|
||||
DefaultProfileID: selection.definition.DefaultProfile,
|
||||
Inputs: inputs,
|
||||
OutputContract: selection.definition.Validation,
|
||||
}, nil
|
||||
return r.resolvePromptInspection(ctx, promptID, promptVersion)
|
||||
}
|
||||
|
||||
@@ -18,6 +18,13 @@ type inspectionPromptRepository struct {
|
||||
version string
|
||||
}
|
||||
|
||||
type promptMetadataRepository struct {
|
||||
inspection domain.PromptInspection
|
||||
inspectionErr error
|
||||
lookupID string
|
||||
lookupVersion string
|
||||
}
|
||||
|
||||
func (r *inspectionPromptRepository) GetPromptDefinition(
|
||||
_ context.Context,
|
||||
id string,
|
||||
@@ -32,6 +39,22 @@ func (r *inspectionPromptRepository) GetPromptDefinition(
|
||||
return r.definition, nil
|
||||
}
|
||||
|
||||
func (r *promptMetadataRepository) GetPromptDefinition(context.Context, string, string) (*domain.PromptDefinition, error) {
|
||||
return nil, errors.New("full prompt definition should not be requested")
|
||||
}
|
||||
|
||||
func (r *promptMetadataRepository) GetPromptInspection(
|
||||
_ context.Context,
|
||||
id string,
|
||||
version string,
|
||||
) (*domain.PromptInspection, error) {
|
||||
r.lookupID = id
|
||||
r.lookupVersion = version
|
||||
inspection := r.inspection
|
||||
inspection.Inputs = append([]domain.PromptInput(nil), inspection.Inputs...)
|
||||
return &inspection, r.inspectionErr
|
||||
}
|
||||
|
||||
func TestRunnerInspectPromptResolvesOneDefinitionWithoutExecutionCollaborators(t *testing.T) {
|
||||
definition := &domain.PromptDefinition{
|
||||
ID: "normalized.prompt",
|
||||
@@ -81,6 +104,42 @@ func TestRunnerInspectPromptResolvesOneDefinitionWithoutExecutionCollaborators(t
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerInspectPromptUsesOwnedMetadataWhenRepositorySupportsIt(t *testing.T) {
|
||||
repository := &promptMetadataRepository{inspection: domain.PromptInspection{
|
||||
PromptID: "captured.prompt", PromptVersion: "v2", PromptHash: "cached-prompt-hash",
|
||||
DefaultProfileID: "partial-profile",
|
||||
Inputs: []domain.PromptInput{{Name: "document", Required: true, ContentType: "text/plain", Description: "Captured."}},
|
||||
OutputContract: domain.OutputContract{Format: domain.FormatJSON, ValidationMode: domain.ValidationJSON},
|
||||
}}
|
||||
runner := &Runner{promptDefs: repository}
|
||||
inspection, err := runner.InspectPrompt(context.Background(), " requested-id ", " requested-version ")
|
||||
if err != nil || repository.lookupID != " requested-id " || repository.lookupVersion != " requested-version " ||
|
||||
inspection.PromptID != "captured.prompt" || inspection.PromptVersion != "v2" ||
|
||||
inspection.PromptHash != "cached-prompt-hash" || inspection.DefaultProfileID != "partial-profile" ||
|
||||
!reflect.DeepEqual(inspection.Inputs, repository.inspection.Inputs) || inspection.OutputContract != repository.inspection.OutputContract {
|
||||
t.Fatalf("metadata inspection=(%+v, %v), lookup=(%q,%q)", inspection, err, repository.lookupID, repository.lookupVersion)
|
||||
}
|
||||
inspection.Inputs[0].Name = "caller mutation"
|
||||
inspection.OutputContract.SchemaPath = "caller mutation"
|
||||
second, err := runner.InspectPrompt(context.Background(), "requested-id", "requested-version")
|
||||
if err != nil || second.Inputs[0].Name != "document" || second.OutputContract.SchemaPath != "" {
|
||||
t.Fatalf("metadata inspection was not caller-owned: (%+v, %v)", second, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerInspectPromptUsesCapturedHashWhenMetadataCapabilityIsAbsent(t *testing.T) {
|
||||
repository := &requirementInspectionPromptRepository{
|
||||
definition: &domain.PromptDefinition{
|
||||
ID: "captured.prompt", Version: "v1", Inputs: []domain.PromptInput{{Name: "document"}},
|
||||
},
|
||||
}
|
||||
inspection, err := (&Runner{promptDefs: repository}).InspectPrompt(context.Background(), " requested-id ", " requested-version ")
|
||||
if err != nil || inspection.PromptID != "captured.prompt" || inspection.PromptVersion != "v1" ||
|
||||
inspection.PromptHash != "captured-hash" || inspection.Inputs[0].Name != "document" {
|
||||
t.Fatalf("captured-hash fallback inspection=(%+v, %v)", inspection, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerInspectPromptClassifiesFailuresWithoutRepositoryWorkAfterCancellation(t *testing.T) {
|
||||
t.Run("blank ID", func(t *testing.T) {
|
||||
repository := &inspectionPromptRepository{}
|
||||
|
||||
80
internal/usecase/requirements_inspection.go
Normal file
80
internal/usecase/requirements_inspection.go
Normal file
@@ -0,0 +1,80 @@
|
||||
package usecase
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/prompt"
|
||||
)
|
||||
|
||||
// RequirementInspectionRequest selects a captured prompt and supplies the
|
||||
// context used for dependency classification.
|
||||
type RequirementInspectionRequest struct {
|
||||
PromptID string
|
||||
PromptVersion string
|
||||
ProfileID string
|
||||
Vars map[string]string
|
||||
SessionID string
|
||||
}
|
||||
|
||||
type requirementSnapshotRepository interface {
|
||||
GetRequirementSnapshot(id, version string) (prompt.RequirementSnapshot, bool)
|
||||
}
|
||||
|
||||
// InspectRequirements discovers prompt inputs and caller variables without
|
||||
// loading artifacts or performing execution preparation.
|
||||
func (r *Runner) InspectRequirements(
|
||||
ctx context.Context,
|
||||
req RequirementInspectionRequest,
|
||||
) (*domain.RequirementInspection, error) {
|
||||
if strings.TrimSpace(req.PromptID) == "" {
|
||||
return nil, fmt.Errorf("%w: prompt id is required", ErrInvalidRequest)
|
||||
}
|
||||
directSessionID, err := domain.NormalizeSessionID(req.SessionID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: session_id: %v", ErrInvalidRequest, err)
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, fmt.Errorf("%w: %w", ErrPromptLoad, err)
|
||||
}
|
||||
|
||||
selection, err := r.resolvePromptInspection(ctx, req.PromptID, req.PromptVersion)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, fmt.Errorf("%w: %w", ErrProfileLoad, err)
|
||||
}
|
||||
selectedProfileID := req.ProfileID
|
||||
if strings.TrimSpace(selectedProfileID) == "" {
|
||||
selectedProfileID = selection.DefaultProfileID
|
||||
}
|
||||
if strings.TrimSpace(selectedProfileID) == "" {
|
||||
return nil, fmt.Errorf("%w: %w: profile id is required either in request or prompt default_profile", ErrInvalidRequest, ErrProfileRequired)
|
||||
}
|
||||
profileIdentity, err := r.resolveProfileIdentity(ctx, selectedProfileID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, fmt.Errorf("%w: %w", ErrPromptLoad, err)
|
||||
}
|
||||
|
||||
repository, ok := r.promptDefs.(requirementSnapshotRepository)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("%w: prompt repository does not provide captured requirement snapshots", ErrPromptLoad)
|
||||
}
|
||||
snapshot, ok := repository.GetRequirementSnapshot(selection.PromptID, selection.PromptVersion)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("%w: captured requirement snapshot is unavailable", ErrPromptLoad)
|
||||
}
|
||||
definition := &domain.PromptDefinition{ID: selection.PromptID, Version: selection.PromptVersion}
|
||||
inspection, err := prompt.AnalyzeRequirements(ctx, definition, snapshot, req.Vars, directSessionID != "")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: %w", ErrPromptLoad, err)
|
||||
}
|
||||
inspection.ProfileID = profileIdentity
|
||||
return &inspection, nil
|
||||
}
|
||||
372
internal/usecase/requirements_inspection_test.go
Normal file
372
internal/usecase/requirements_inspection_test.go
Normal file
@@ -0,0 +1,372 @@
|
||||
package usecase
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"text/template"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/prompt"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/promptdef"
|
||||
)
|
||||
|
||||
type requirementInspectionPromptRepository struct {
|
||||
definition *domain.PromptDefinition
|
||||
snapshot prompt.RequirementSnapshot
|
||||
lookupErr error
|
||||
snapshotFound bool
|
||||
lookupCalls int
|
||||
snapshotCalls int
|
||||
lookupID string
|
||||
lookupVersion string
|
||||
snapshotID string
|
||||
snapshotVersion string
|
||||
}
|
||||
|
||||
type requirementPromptMetadataRepository struct {
|
||||
*requirementInspectionPromptRepository
|
||||
inspection domain.PromptInspection
|
||||
lookupID string
|
||||
lookupVersion string
|
||||
}
|
||||
|
||||
type inspectionProfileIdentityRepository struct {
|
||||
*inspectionProfileRepository
|
||||
identity string
|
||||
identityErr error
|
||||
identityID string
|
||||
identityCalls int
|
||||
}
|
||||
|
||||
func (repository *inspectionProfileIdentityRepository) GetProfileIdentity(
|
||||
_ context.Context,
|
||||
id string,
|
||||
) (string, error) {
|
||||
repository.identityCalls++
|
||||
repository.identityID = id
|
||||
return repository.identity, repository.identityErr
|
||||
}
|
||||
|
||||
func (repository *requirementInspectionPromptRepository) GetPromptDefinition(
|
||||
context.Context,
|
||||
string,
|
||||
string,
|
||||
) (*domain.PromptDefinition, error) {
|
||||
return repository.definition, repository.lookupErr
|
||||
}
|
||||
|
||||
func (repository *requirementPromptMetadataRepository) GetPromptInspection(
|
||||
_ context.Context,
|
||||
id string,
|
||||
version string,
|
||||
) (*domain.PromptInspection, error) {
|
||||
repository.lookupID = id
|
||||
repository.lookupVersion = version
|
||||
inspection := repository.inspection
|
||||
inspection.Inputs = append([]domain.PromptInput(nil), inspection.Inputs...)
|
||||
return &inspection, nil
|
||||
}
|
||||
|
||||
func (repository *requirementInspectionPromptRepository) GetPromptSnapshot(
|
||||
_ context.Context,
|
||||
id string,
|
||||
version string,
|
||||
) (*domain.PromptDefinition, string, error) {
|
||||
repository.lookupCalls++
|
||||
repository.lookupID = id
|
||||
repository.lookupVersion = version
|
||||
return repository.definition, "captured-hash", repository.lookupErr
|
||||
}
|
||||
|
||||
func (repository *requirementInspectionPromptRepository) GetRequirementSnapshot(
|
||||
id string,
|
||||
version string,
|
||||
) (prompt.RequirementSnapshot, bool) {
|
||||
repository.snapshotCalls++
|
||||
repository.snapshotID = id
|
||||
repository.snapshotVersion = version
|
||||
return repository.snapshot, repository.snapshotFound
|
||||
}
|
||||
|
||||
func TestRunnerInspectRequirementsUsesCapturedSelectionWithoutExecutionWork(t *testing.T) {
|
||||
message, err := prompt.CompileTemplate("message", `{{input "message-input"}} {{.caller}}`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
session, err := prompt.CompileTemplate("session", `{{input "session-only"}}`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
definition := &domain.PromptDefinition{ID: "selected", Version: "v2", DefaultProfile: "default-profile"}
|
||||
repository := &requirementInspectionPromptRepository{
|
||||
definition: definition,
|
||||
snapshotFound: true,
|
||||
snapshot: prompt.RequirementSnapshot{Templates: prompt.CompiledTemplates{
|
||||
Messages: []*template.Template{message}, Session: session,
|
||||
}},
|
||||
}
|
||||
profiles := &inspectionProfileRepository{profile: &domain.ExecutionProfile{
|
||||
ID: "partial-profile", ResolvedTarget: &domain.ExecutionTarget{Endpoint: ""},
|
||||
}}
|
||||
runner := &Runner{promptDefs: repository, profiles: profiles}
|
||||
inspection, err := runner.InspectRequirements(context.Background(), RequirementInspectionRequest{
|
||||
PromptID: " requested ", PromptVersion: " exact-version ", ProfileID: "partial-profile",
|
||||
Vars: map[string]string{"caller": ""}, SessionID: " direct-session ",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("inspect requirements: %v", err)
|
||||
}
|
||||
if repository.lookupCalls != 1 || repository.lookupID != " requested " || repository.lookupVersion != " exact-version " ||
|
||||
repository.snapshotCalls != 1 || repository.snapshotID != definition.ID || repository.snapshotVersion != definition.Version {
|
||||
t.Fatalf("selection lookup=(%d,%q,%q), snapshot=(%d,%q,%q)", repository.lookupCalls, repository.lookupID,
|
||||
repository.lookupVersion, repository.snapshotCalls, repository.snapshotID, repository.snapshotVersion)
|
||||
}
|
||||
if profiles.calls != 1 || profiles.id != "partial-profile" || inspection.PromptID != definition.ID ||
|
||||
inspection.PromptVersion != definition.Version || inspection.ProfileID != "partial-profile" || !inspection.Complete ||
|
||||
len(inspection.Inputs) != 1 || inspection.Inputs[0].Name != "message-input" || len(inspection.Variables) != 1 ||
|
||||
inspection.Variables[0].Name != "caller" || !inspection.Variables[0].Supplied {
|
||||
t.Fatalf("unexpected requirement inspection: %+v", inspection)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerInspectRequirementsUsesPromptMetadataAndExactRequirementSnapshot(t *testing.T) {
|
||||
message, err := prompt.CompileTemplate("message", `{{input "message-input"}} {{.caller}}`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
session, err := prompt.CompileTemplate("session", `{{input "session-only"}}`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
baseRepository := &requirementInspectionPromptRepository{
|
||||
snapshotFound: true,
|
||||
snapshot: prompt.RequirementSnapshot{Templates: prompt.CompiledTemplates{
|
||||
Messages: []*template.Template{message}, Session: session,
|
||||
}},
|
||||
}
|
||||
promptRepository := &requirementPromptMetadataRepository{
|
||||
requirementInspectionPromptRepository: baseRepository,
|
||||
inspection: domain.PromptInspection{
|
||||
PromptID: "captured-id", PromptVersion: "captured-version", PromptHash: "captured-hash",
|
||||
DefaultProfileID: "default-profile",
|
||||
},
|
||||
}
|
||||
profiles := &inspectionProfileIdentityRepository{
|
||||
inspectionProfileRepository: &inspectionProfileRepository{},
|
||||
identity: "default-profile",
|
||||
}
|
||||
inspection, err := (&Runner{promptDefs: promptRepository, profiles: profiles}).InspectRequirements(
|
||||
context.Background(), RequirementInspectionRequest{
|
||||
PromptID: " requested-id ", PromptVersion: " requested-version ",
|
||||
Vars: map[string]string{"caller": "Ada"}, SessionID: " direct-session ",
|
||||
},
|
||||
)
|
||||
if err != nil || promptRepository.lookupID != " requested-id " || promptRepository.lookupVersion != " requested-version " ||
|
||||
baseRepository.snapshotID != "captured-id" || baseRepository.snapshotVersion != "captured-version" ||
|
||||
profiles.identityID != "default-profile" || inspection.PromptID != "captured-id" ||
|
||||
inspection.PromptVersion != "captured-version" || inspection.ProfileID != "default-profile" ||
|
||||
!inspection.Complete || len(inspection.Inputs) != 1 || inspection.Inputs[0].Name != "message-input" ||
|
||||
len(inspection.Variables) != 1 || inspection.Variables[0].Name != "caller" || !inspection.Variables[0].Supplied {
|
||||
t.Fatalf("metadata-based requirement inspection=(%+v, %v), prompt lookup=(%q,%q), snapshot=(%q,%q), profile=%q",
|
||||
inspection, err, promptRepository.lookupID, promptRepository.lookupVersion,
|
||||
baseRepository.snapshotID, baseRepository.snapshotVersion, profiles.identityID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerInspectRequirementsUsesProfileIdentityCapabilityOrFallback(t *testing.T) {
|
||||
message, err := prompt.CompileTemplate("message", `{{.name}}`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
definition := &domain.PromptDefinition{ID: "selected", Version: "v1"}
|
||||
makePromptRepository := func() *requirementInspectionPromptRepository {
|
||||
return &requirementInspectionPromptRepository{
|
||||
definition: definition, snapshotFound: true,
|
||||
snapshot: prompt.RequirementSnapshot{Templates: prompt.CompiledTemplates{Messages: []*template.Template{message}}},
|
||||
}
|
||||
}
|
||||
requestedID := " literal-profile "
|
||||
|
||||
t.Run("optional capability", func(t *testing.T) {
|
||||
profiles := &inspectionProfileIdentityRepository{
|
||||
inspectionProfileRepository: &inspectionProfileRepository{profile: &domain.ExecutionProfile{
|
||||
ID: "repository-profile", ResolvedTarget: &domain.ExecutionTarget{},
|
||||
}},
|
||||
identity: requestedID,
|
||||
}
|
||||
inspection, err := (&Runner{promptDefs: makePromptRepository(), profiles: profiles}).InspectRequirements(
|
||||
context.Background(), RequirementInspectionRequest{
|
||||
PromptID: "selected", ProfileID: requestedID, Vars: map[string]string{"name": "Ada"},
|
||||
},
|
||||
)
|
||||
if err != nil || inspection.ProfileID != requestedID || profiles.identityCalls != 1 ||
|
||||
profiles.identityID != requestedID || profiles.calls != 0 {
|
||||
t.Fatalf("identity capability inspection=(%+v, %v), capability calls=%d id=%q full lookups=%d",
|
||||
inspection, err, profiles.identityCalls, profiles.identityID, profiles.calls)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("full-profile fallback", func(t *testing.T) {
|
||||
profiles := &inspectionProfileRepository{profile: &domain.ExecutionProfile{
|
||||
ID: "repository-profile", ResolvedTarget: &domain.ExecutionTarget{},
|
||||
}}
|
||||
inspection, err := (&Runner{promptDefs: makePromptRepository(), profiles: profiles}).InspectRequirements(
|
||||
context.Background(), RequirementInspectionRequest{
|
||||
PromptID: "selected", ProfileID: requestedID, Vars: map[string]string{"name": "Ada"},
|
||||
},
|
||||
)
|
||||
if err != nil || inspection.ProfileID != requestedID || profiles.calls != 1 || profiles.id != requestedID {
|
||||
t.Fatalf("fallback inspection=(%+v, %v), full lookup=(%d,%q)", inspection, err, profiles.calls, profiles.id)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("capability failure", func(t *testing.T) {
|
||||
lookupErr := errors.New("profile identity lookup failed")
|
||||
profiles := &inspectionProfileIdentityRepository{
|
||||
inspectionProfileRepository: &inspectionProfileRepository{},
|
||||
identityErr: lookupErr,
|
||||
}
|
||||
_, err := (&Runner{promptDefs: makePromptRepository(), profiles: profiles}).InspectRequirements(
|
||||
context.Background(), RequirementInspectionRequest{PromptID: "selected", ProfileID: requestedID},
|
||||
)
|
||||
if !errors.Is(err, ErrProfileLoad) || !errors.Is(err, lookupErr) || profiles.identityCalls != 1 || profiles.calls != 0 {
|
||||
t.Fatalf("capability failure=(%v), capability calls=%d full lookups=%d", err, profiles.identityCalls, profiles.calls)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestRunnerInspectRequirementsUsesPromptDefaultAndIncludesSessionWhenNotBypassed(t *testing.T) {
|
||||
message, err := prompt.CompileTemplate("message", `{{input "message-input"}}`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
session, err := prompt.CompileTemplate("session", `{{input "session-input"}}`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
definition := &domain.PromptDefinition{ID: "selected", Version: "v1", DefaultProfile: "default-profile"}
|
||||
repository := &requirementInspectionPromptRepository{
|
||||
definition: definition, snapshotFound: true,
|
||||
snapshot: prompt.RequirementSnapshot{Templates: prompt.CompiledTemplates{Messages: []*template.Template{message}, Session: session}},
|
||||
}
|
||||
profiles := &inspectionProfileRepository{profile: &domain.ExecutionProfile{
|
||||
ID: "default-profile", ResolvedTarget: &domain.ExecutionTarget{},
|
||||
}}
|
||||
inspection, err := (&Runner{promptDefs: repository, profiles: profiles}).InspectRequirements(
|
||||
context.Background(), RequirementInspectionRequest{PromptID: "selected"},
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("inspect requirements: %v", err)
|
||||
}
|
||||
if profiles.id != "default-profile" || inspection.ProfileID != "default-profile" || len(inspection.Inputs) != 2 ||
|
||||
inspection.Inputs[0].Name != "message-input" || inspection.Inputs[1].Name != "session-input" {
|
||||
t.Fatalf("prompt-default profile or session template was not selected: %+v", inspection)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerInspectRequirementsReturnsIncompleteDiscoveryAsSuccess(t *testing.T) {
|
||||
compiled, err := prompt.CompileTemplate("message", `{{if .missing}}{{input "left"}}{{else}}{{input "right"}}{{end}}`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
repository := &requirementInspectionPromptRepository{
|
||||
definition: &domain.PromptDefinition{ID: "selected", Version: "v1", DefaultProfile: "profile"},
|
||||
snapshotFound: true,
|
||||
snapshot: prompt.RequirementSnapshot{Templates: prompt.CompiledTemplates{Messages: []*template.Template{compiled}}},
|
||||
}
|
||||
profiles := &inspectionProfileRepository{profile: &domain.ExecutionProfile{ID: "profile", ResolvedTarget: &domain.ExecutionTarget{}}}
|
||||
inspection, err := (&Runner{promptDefs: repository, profiles: profiles}).InspectRequirements(
|
||||
context.Background(), RequirementInspectionRequest{PromptID: "selected"},
|
||||
)
|
||||
if err != nil || inspection == nil || inspection.Complete || len(inspection.Issues) != 1 ||
|
||||
inspection.Issues[0].Code != domain.RequirementIssueMissingVariable {
|
||||
t.Fatalf("incomplete discovery returned (%+v, %v), want successful partial result", inspection, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerInspectRequirementsClassifiesSelectionAndSnapshotFailures(t *testing.T) {
|
||||
t.Run("blank prompt", func(t *testing.T) {
|
||||
repository := &requirementInspectionPromptRepository{}
|
||||
_, err := (&Runner{promptDefs: repository}).InspectRequirements(context.Background(), RequirementInspectionRequest{PromptID: " \t "})
|
||||
if !errors.Is(err, ErrInvalidRequest) || repository.lookupCalls != 0 {
|
||||
t.Fatalf("blank prompt error=%v lookups=%d", err, repository.lookupCalls)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("invalid session", func(t *testing.T) {
|
||||
repository := &requirementInspectionPromptRepository{}
|
||||
_, err := (&Runner{promptDefs: repository}).InspectRequirements(context.Background(), RequirementInspectionRequest{
|
||||
PromptID: "selected", SessionID: string([]byte{0xff}),
|
||||
})
|
||||
if !errors.Is(err, ErrInvalidRequest) || repository.lookupCalls != 0 {
|
||||
t.Fatalf("invalid session error=%v lookups=%d", err, repository.lookupCalls)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("cancellation before lookup", func(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
repository := &requirementInspectionPromptRepository{}
|
||||
_, err := (&Runner{promptDefs: repository}).InspectRequirements(ctx, RequirementInspectionRequest{PromptID: "selected"})
|
||||
if !errors.Is(err, ErrPromptLoad) || !errors.Is(err, context.Canceled) || repository.lookupCalls != 0 {
|
||||
t.Fatalf("canceled inspection=%v lookups=%d", err, repository.lookupCalls)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("no profile selection", func(t *testing.T) {
|
||||
repository := &requirementInspectionPromptRepository{definition: &domain.PromptDefinition{ID: "selected"}}
|
||||
_, err := (&Runner{promptDefs: repository}).InspectRequirements(context.Background(), RequirementInspectionRequest{PromptID: "selected"})
|
||||
if !errors.Is(err, ErrInvalidRequest) || !errors.Is(err, ErrProfileRequired) {
|
||||
t.Fatalf("missing profile error=%v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("ambiguous prompt selection", func(t *testing.T) {
|
||||
ambiguous := errors.New("ambiguous omitted version")
|
||||
repository := &requirementInspectionPromptRepository{lookupErr: ambiguous}
|
||||
_, err := (&Runner{promptDefs: repository}).InspectRequirements(context.Background(), RequirementInspectionRequest{PromptID: "selected"})
|
||||
if !errors.Is(err, ErrPromptLoad) || !errors.Is(err, ambiguous) || repository.lookupVersion != "" {
|
||||
t.Fatalf("ambiguous selection error=%v version=%q", err, repository.lookupVersion)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("captured snapshot unavailable", func(t *testing.T) {
|
||||
repository := &requirementInspectionPromptRepository{definition: &domain.PromptDefinition{ID: "selected", DefaultProfile: "profile"}}
|
||||
profiles := &inspectionProfileRepository{profile: &domain.ExecutionProfile{ID: "profile", ResolvedTarget: &domain.ExecutionTarget{}}}
|
||||
_, err := (&Runner{promptDefs: repository, profiles: profiles}).InspectRequirements(context.Background(), RequirementInspectionRequest{PromptID: "selected"})
|
||||
if !errors.Is(err, ErrPromptLoad) || repository.snapshotCalls != 1 {
|
||||
t.Fatalf("missing snapshot error=%v calls=%d", err, repository.snapshotCalls)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("repository lacks snapshot capability", func(t *testing.T) {
|
||||
_, err := (&Runner{promptDefs: &inspectionPromptRepository{definition: &domain.PromptDefinition{ID: "selected", DefaultProfile: "profile"}},
|
||||
profiles: &inspectionProfileRepository{profile: &domain.ExecutionProfile{ID: "profile", ResolvedTarget: &domain.ExecutionTarget{}}},
|
||||
}).InspectRequirements(context.Background(), RequirementInspectionRequest{PromptID: "selected"})
|
||||
if !errors.Is(err, ErrPromptLoad) {
|
||||
t.Fatalf("missing capability error=%v", err)
|
||||
}
|
||||
})
|
||||
if _, err := (*Runner)(nil).InspectRequirements(context.Background(), RequirementInspectionRequest{PromptID: "selected"}); !errors.Is(err, ErrPromptLoad) {
|
||||
t.Fatalf("nil runner error=%v", err)
|
||||
}
|
||||
if _, err := (&Runner{}).InspectRequirements(context.Background(), RequirementInspectionRequest{PromptID: "selected"}); !errors.Is(err, ErrPromptLoad) {
|
||||
t.Fatalf("nil repository error=%v", err)
|
||||
}
|
||||
if _, err := (&Runner{promptDefs: &requirementInspectionPromptRepository{definition: &domain.PromptDefinition{ID: "selected"}}}).InspectRequirements(
|
||||
context.Background(), RequirementInspectionRequest{PromptID: "selected", ProfileID: "explicit"},
|
||||
); !errors.Is(err, ErrProfileLoad) {
|
||||
t.Fatalf("missing profile error=%v", err)
|
||||
}
|
||||
if _, err := (&Runner{promptDefs: &requirementInspectionPromptRepository{definition: &domain.PromptDefinition{ID: "selected", DefaultProfile: "profile"}},
|
||||
profiles: &inspectionProfileRepository{profile: &domain.ExecutionProfile{ID: "profile"}},
|
||||
}).InspectRequirements(context.Background(), RequirementInspectionRequest{PromptID: "selected"}); !errors.Is(err, ErrProfileLoad) {
|
||||
t.Fatalf("unresolved profile error=%v", err)
|
||||
}
|
||||
if _, err := (&Runner{promptDefs: &requirementInspectionPromptRepository{lookupErr: promptdef.ErrPromptDefinitionNotFound}}).InspectRequirements(
|
||||
context.Background(), RequirementInspectionRequest{PromptID: "missing"},
|
||||
); !errors.Is(err, ErrPromptLoad) || !errors.Is(err, promptdef.ErrPromptDefinitionNotFound) {
|
||||
t.Fatalf("missing prompt error=%v", err)
|
||||
}
|
||||
}
|
||||
@@ -188,7 +188,7 @@ func validateArtifact(ctx context.Context, artifact *domain.Artifact, contract d
|
||||
res.IsValid = true
|
||||
return res, nil
|
||||
case domain.ValidationBasic:
|
||||
empty := strings.TrimSpace(string(artifact.Body)) == ""
|
||||
empty := len(bytes.TrimSpace(artifact.Body)) == 0
|
||||
if err := ctx.Err(); err != nil {
|
||||
return domain.ValidationResult{}, err
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package validate
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/url"
|
||||
@@ -35,32 +36,51 @@ func TestStandardValidatorNoneSkipped(t *testing.T) {
|
||||
|
||||
func TestStandardValidatorBasicSuccess(t *testing.T) {
|
||||
v := NewStandardValidator("")
|
||||
|
||||
res, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte("hello")}, domain.OutputContract{
|
||||
ValidationMode: domain.ValidationBasic,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
if res.Status != domain.ValidationPassed || !res.IsValid {
|
||||
t.Fatalf("expected passed/valid, got status=%q valid=%v", res.Status, res.IsValid)
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
body []byte
|
||||
}{
|
||||
{name: "text", body: []byte("hello")},
|
||||
{name: "invalid UTF-8 byte", body: []byte{0xff}},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
res, err := v.Validate(context.Background(), &domain.Artifact{Body: tc.body}, domain.OutputContract{
|
||||
ValidationMode: domain.ValidationBasic,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
if res.Status != domain.ValidationPassed || !res.IsValid {
|
||||
t.Fatalf("expected passed/valid, got status=%q valid=%v", res.Status, res.IsValid)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestStandardValidatorBasicFailureEmpty(t *testing.T) {
|
||||
v := NewStandardValidator("")
|
||||
|
||||
res, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte(" \n\t ")}, domain.OutputContract{
|
||||
ValidationMode: domain.ValidationBasic,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
if res.Status != domain.ValidationFailed || res.IsValid {
|
||||
t.Fatalf("expected failed/invalid, got status=%q valid=%v", res.Status, res.IsValid)
|
||||
}
|
||||
if len(res.Errors) == 0 {
|
||||
t.Fatal("expected validation errors")
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
body []byte
|
||||
}{
|
||||
{name: "empty bytes", body: []byte{}},
|
||||
{name: "ASCII whitespace", body: []byte(" \n\t ")},
|
||||
{name: "Unicode whitespace", body: []byte("\u2003\u00a0\u3000")},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
res, err := v.Validate(context.Background(), &domain.Artifact{Body: tc.body}, domain.OutputContract{
|
||||
ValidationMode: domain.ValidationBasic,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
if res.Status != domain.ValidationFailed || res.IsValid {
|
||||
t.Fatalf("expected failed/invalid, got status=%q valid=%v", res.Status, res.IsValid)
|
||||
}
|
||||
if len(res.Errors) == 0 {
|
||||
t.Fatal("expected validation errors")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -917,6 +937,49 @@ func BenchmarkJSONValidation(b *testing.B) {
|
||||
}
|
||||
}
|
||||
|
||||
var basicValidationBenchmarkResult domain.ValidationResult
|
||||
|
||||
func BenchmarkBasicValidation(b *testing.B) {
|
||||
largeText := bytes.Repeat([]byte("x"), 1<<20)
|
||||
largeWhitespace := bytes.Repeat([]byte(" \t\n"), (1<<20)/3)
|
||||
unicodeWhitespace := []byte("\u2003\u00a0\u3000")
|
||||
benchmarks := []struct {
|
||||
name string
|
||||
body []byte
|
||||
wantValid bool
|
||||
}{
|
||||
{name: "short_text", body: []byte("A concise response."), wantValid: true},
|
||||
{name: "1MiB_nonempty_text", body: largeText, wantValid: true},
|
||||
{name: "whitespace_only_text", body: largeWhitespace},
|
||||
{name: "unicode_whitespace", body: unicodeWhitespace},
|
||||
}
|
||||
validator := NewStandardValidator("")
|
||||
contract := domain.OutputContract{ValidationMode: domain.ValidationBasic}
|
||||
ctx := context.Background()
|
||||
|
||||
for _, benchmark := range benchmarks {
|
||||
b.Run(benchmark.name, func(b *testing.B) {
|
||||
artifact := &domain.Artifact{Body: benchmark.body}
|
||||
result, err := validator.Validate(ctx, artifact, contract)
|
||||
if err != nil || result.IsValid != benchmark.wantValid {
|
||||
b.Fatalf("validate basic output: result=%+v error=%v", result, err)
|
||||
}
|
||||
|
||||
b.ReportAllocs()
|
||||
// Nonempty input can be classified from its boundary bytes without
|
||||
// scanning the body, so report latency rather than bytes per second.
|
||||
b.ResetTimer()
|
||||
for range b.N {
|
||||
result, err := validator.Validate(ctx, artifact, contract)
|
||||
if err != nil || result.IsValid != benchmark.wantValid {
|
||||
b.Fatalf("validate basic output: result=%+v error=%v", result, err)
|
||||
}
|
||||
basicValidationBenchmarkResult = result
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestJSONSchemaDialectIsDraft2020(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
|
||||
67
listing.go
Normal file
67
listing.go
Normal file
@@ -0,0 +1,67 @@
|
||||
package promptkit
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// PromptIdentity identifies one prompt ID and exact version in the effective
|
||||
// configuration captured when an Engine is constructed. It has no stable JSON
|
||||
// representation.
|
||||
type PromptIdentity struct {
|
||||
ID string
|
||||
Version string
|
||||
}
|
||||
|
||||
// ListProfiles returns profile IDs in the effective configuration captured
|
||||
// when the engine was constructed. Maintained and consumer profiles are
|
||||
// included; an inherited profile retains its own ID. IDs are preserved exactly
|
||||
// and sorted lexically. The returned slice is caller-owned and safe to mutate
|
||||
// while the engine is used concurrently. Empty inventories succeed.
|
||||
//
|
||||
// Listing reports membership, not execution readiness. It does not reload
|
||||
// configuration or perform resource, credential, provider, or execution work.
|
||||
// Profile IDs are application data and may be sensitive. A nil or zero-value
|
||||
// engine returns an error matching ErrInvalidConfig. Cancellation or deadline
|
||||
// expiration returns the context error and no partial result.
|
||||
func (e *Engine) ListProfiles(ctx context.Context) ([]string, error) {
|
||||
if e == nil || e.internal == nil {
|
||||
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
|
||||
}
|
||||
|
||||
profiles, err := e.internal.ListProfiles(ctx)
|
||||
if err != nil {
|
||||
return nil, mapPublicError(err)
|
||||
}
|
||||
return profiles, nil
|
||||
}
|
||||
|
||||
// ListPrompts returns every prompt ID and exact version in the effective
|
||||
// configuration captured when the engine was constructed, sorted lexically by
|
||||
// ID and then version. Versions are not interpreted or prioritized. The
|
||||
// returned slice is caller-owned and safe to mutate while the engine is used
|
||||
// concurrently. Empty inventories succeed.
|
||||
//
|
||||
// Listing reports membership, not execution readiness. It does not reload
|
||||
// configuration or perform resource, credential, provider, or execution work.
|
||||
// Prompt identities are application data and may be sensitive. A nil or
|
||||
// zero-value engine returns an error matching ErrInvalidConfig. Cancellation
|
||||
// or deadline expiration returns the context error and no partial result.
|
||||
func (e *Engine) ListPrompts(ctx context.Context) ([]PromptIdentity, error) {
|
||||
if e == nil || e.internal == nil {
|
||||
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
|
||||
}
|
||||
|
||||
identities, err := e.internal.ListPrompts(ctx)
|
||||
if err != nil {
|
||||
return nil, mapPublicError(err)
|
||||
}
|
||||
public := make([]PromptIdentity, len(identities))
|
||||
for i, identity := range identities {
|
||||
public[i] = PromptIdentity{ID: identity.ID, Version: identity.Version}
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, mapPublicError(err)
|
||||
}
|
||||
return public, nil
|
||||
}
|
||||
379
listing_contract_test.go
Normal file
379
listing_contract_test.go
Normal file
@@ -0,0 +1,379 @@
|
||||
package promptkit_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"sync"
|
||||
"testing"
|
||||
"testing/fstest"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
func TestListIdentitiesBasicContract(t *testing.T) {
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
|
||||
Source: configkit.Mapping("listing", map[string]any{
|
||||
"profiles": map[string]any{"configured-profile": map[string]any{"model": "example-model"}},
|
||||
"prompts": map[string]any{"example-prompt": map[string]any{
|
||||
"2": listingContractPrompt(),
|
||||
"10": listingContractPrompt(),
|
||||
}},
|
||||
}),
|
||||
}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
profiles, err := engine.ListProfiles(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !containsListingProfile(profiles, "configured-profile") {
|
||||
t.Fatalf("profile listing %#v does not include configured profile", profiles)
|
||||
}
|
||||
|
||||
prompts, err := engine.ListPrompts(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := []promptkit.PromptIdentity{
|
||||
{ID: "example-prompt", Version: "10"},
|
||||
{ID: "example-prompt", Version: "2"},
|
||||
}
|
||||
if len(prompts) != len(want) {
|
||||
t.Fatalf("prompt listing = %#v, want %#v", prompts, want)
|
||||
}
|
||||
for i := range want {
|
||||
if prompts[i] != want[i] {
|
||||
t.Fatalf("prompt listing = %#v, want %#v", prompts, want)
|
||||
}
|
||||
}
|
||||
|
||||
profile, err := engine.InspectProfile(context.Background(), "configured-profile")
|
||||
if err != nil || profile == nil || profile.ProfileID != "configured-profile" {
|
||||
t.Fatalf("listed profile could not be inspected: (%+v, %v)", profile, err)
|
||||
}
|
||||
prompt, err := engine.InspectPrompt(context.Background(), prompts[0].ID, prompts[0].Version)
|
||||
if err != nil || prompt == nil || prompt.PromptID != prompts[0].ID || prompt.PromptVersion != prompts[0].Version {
|
||||
t.Fatalf("listed prompt could not be inspected: (%+v, %v)", prompt, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListIdentitiesInvalidEngine(t *testing.T) {
|
||||
var nilEngine *promptkit.Engine
|
||||
var zeroEngine promptkit.Engine
|
||||
for _, engine := range []*promptkit.Engine{nilEngine, &zeroEngine} {
|
||||
if _, err := engine.ListProfiles(context.Background()); !errors.Is(err, promptkit.ErrInvalidConfig) {
|
||||
t.Errorf("ListProfiles on invalid engine: %v", err)
|
||||
}
|
||||
if _, err := engine.ListPrompts(context.Background()); !errors.Is(err, promptkit.ErrInvalidConfig) {
|
||||
t.Errorf("ListPrompts on invalid engine: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestListIdentitiesPreserveContextErrorsWithoutPartialResults(t *testing.T) {
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
|
||||
Source: configkit.Mapping("listing", map[string]any{
|
||||
"profiles": map[string]any{"profile": map[string]any{}},
|
||||
"prompts": map[string]any{"example": map[string]any{"v1": listingContractPrompt()}},
|
||||
}),
|
||||
}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
canceled, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
expired, expire := context.WithDeadline(context.Background(), time.Now().Add(-time.Second))
|
||||
defer expire()
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
ctx context.Context
|
||||
want error
|
||||
}{
|
||||
{name: "canceled", ctx: canceled, want: context.Canceled},
|
||||
{name: "expired", ctx: expired, want: context.DeadlineExceeded},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
profiles, err := engine.ListProfiles(test.ctx)
|
||||
if !errors.Is(err, test.want) || len(profiles) != 0 {
|
||||
t.Errorf("ListProfiles = %#v, %v; want no results and %v", profiles, err, test.want)
|
||||
}
|
||||
prompts, err := engine.ListPrompts(test.ctx)
|
||||
if !errors.Is(err, test.want) || len(prompts) != 0 {
|
||||
t.Errorf("ListPrompts = %#v, %v; want no results and %v", prompts, err, test.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestListIdentitiesUseEffectiveLayeredMembership(t *testing.T) {
|
||||
underlay := configkit.Input{Source: configkit.Mapping("consumer defaults", map[string]any{
|
||||
"profiles": map[string]any{
|
||||
"base/profile": map[string]any{"endpoint": "https://example.test/v1", "model": "base-model"},
|
||||
"child/profile": map[string]any{"extends": "base/profile"},
|
||||
"overlay/profile": map[string]any{"endpoint": "https://example.test/v1", "model": "old-model"},
|
||||
},
|
||||
"prompts": map[string]any{
|
||||
"shared/prompt": map[string]any{
|
||||
"v1": listingContractPrompt(),
|
||||
"v2": listingContractPrompt(),
|
||||
},
|
||||
"underlay-only": map[string]any{"v1": listingContractPrompt()},
|
||||
},
|
||||
})}
|
||||
main := configkit.Input{Source: configkit.Mapping("application overlay", map[string]any{
|
||||
"profiles": map[string]any{
|
||||
"overlay/profile": map[string]any{"model": "effective-model"},
|
||||
},
|
||||
"prompts": map[string]any{
|
||||
"shared/prompt": map[string]any{"v1": listingContractPrompt()},
|
||||
"main-only": map[string]any{"v3": listingContractPrompt()},
|
||||
},
|
||||
})}
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{underlay}}},
|
||||
Main: &main,
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
profiles, err := engine.ListProfiles(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, id := range []string{
|
||||
"base/profile", "child/profile", "overlay/profile",
|
||||
"promptkit-openrouter-standard", "promptkit-openrouter-auto",
|
||||
} {
|
||||
if !containsListingProfile(profiles, id) {
|
||||
t.Errorf("profile listing %#v does not include %q", profiles, id)
|
||||
}
|
||||
}
|
||||
for _, id := range []string{"base/profile", "child/profile", "overlay/profile"} {
|
||||
if got := countListingProfile(profiles, id); got != 1 {
|
||||
t.Errorf("profile %q appears %d times, want 1", id, got)
|
||||
}
|
||||
}
|
||||
|
||||
overlay, err := engine.InspectProfile(context.Background(), "overlay/profile")
|
||||
if err != nil || overlay == nil || overlay.EffectiveModelParams.Model != "effective-model" {
|
||||
t.Fatalf("overlay profile did not resolve to its effective definition: (%+v, %v)", overlay, err)
|
||||
}
|
||||
child, err := engine.InspectProfile(context.Background(), "child/profile")
|
||||
if err != nil || child == nil || child.EffectiveModelParams.Model != "base-model" {
|
||||
t.Fatalf("inherited child profile did not resolve its effective definition: (%+v, %v)", child, err)
|
||||
}
|
||||
|
||||
prompts, err := engine.ListPrompts(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wantPrompts := []promptkit.PromptIdentity{
|
||||
{ID: "main-only", Version: "v3"},
|
||||
{ID: "shared/prompt", Version: "v1"},
|
||||
{ID: "shared/prompt", Version: "v2"},
|
||||
{ID: "underlay-only", Version: "v1"},
|
||||
}
|
||||
if !reflect.DeepEqual(prompts, wantPrompts) {
|
||||
t.Fatalf("effective prompt identities = %#v, want %#v", prompts, wantPrompts)
|
||||
}
|
||||
if got := countListingPrompt(prompts, "shared/prompt", "v1"); got != 1 {
|
||||
t.Fatalf("overlaid prompt identity appears %d times, want 1", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListIdentitiesEmptyEffectiveConfiguration(t *testing.T) {
|
||||
main := configkit.Input{Source: configkit.Mapping("clear effective identities", map[string]any{
|
||||
"profiles": nil,
|
||||
"prompts": nil,
|
||||
})}
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &main}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
profiles, err := engine.ListProfiles(context.Background())
|
||||
if err != nil || len(profiles) != 0 {
|
||||
t.Fatalf("cleared profile listing = %#v, %v", profiles, err)
|
||||
}
|
||||
prompts, err := engine.ListPrompts(context.Background())
|
||||
if err != nil || len(prompts) != 0 {
|
||||
t.Fatalf("cleared prompt listing = %#v, %v", prompts, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListIdentitiesAreOwnedAndSafeForConcurrentUse(t *testing.T) {
|
||||
engine, err := newListingContractEngine(t, map[string]any{
|
||||
"profiles": map[string]any{"profile": map[string]any{"model": "partial-model"}},
|
||||
"prompts": map[string]any{"prompt": map[string]any{"v1": listingContractPrompt()}},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
profiles, err := engine.ListProfiles(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
prompts, err := engine.ListPrompts(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i := range profiles {
|
||||
if profiles[i] == "profile" {
|
||||
profiles[i] = "caller mutation"
|
||||
}
|
||||
}
|
||||
for i := range prompts {
|
||||
if prompts[i].ID == "prompt" {
|
||||
prompts[i].ID = "caller mutation"
|
||||
prompts[i].Version = "caller mutation"
|
||||
}
|
||||
}
|
||||
profiles, err = engine.ListProfiles(context.Background())
|
||||
if err != nil || !containsListingProfile(profiles, "profile") {
|
||||
t.Fatalf("profile result mutation escaped into engine: %#v, %v", profiles, err)
|
||||
}
|
||||
prompts, err = engine.ListPrompts(context.Background())
|
||||
if err != nil || countListingPrompt(prompts, "prompt", "v1") != 1 {
|
||||
t.Fatalf("prompt result mutation escaped into engine: %#v, %v", prompts, err)
|
||||
}
|
||||
|
||||
const workers = 4
|
||||
const iterations = 25
|
||||
start := make(chan struct{})
|
||||
results := make(chan error, workers)
|
||||
var wait sync.WaitGroup
|
||||
wait.Add(workers)
|
||||
for worker := 0; worker < workers; worker++ {
|
||||
go func() {
|
||||
defer wait.Done()
|
||||
<-start
|
||||
for iteration := 0; iteration < iterations; iteration++ {
|
||||
profiles, err := engine.ListProfiles(context.Background())
|
||||
if err != nil {
|
||||
results <- fmt.Errorf("list profiles: %w", err)
|
||||
return
|
||||
}
|
||||
if len(profiles) == 0 {
|
||||
results <- errors.New("profile listing is empty")
|
||||
return
|
||||
}
|
||||
profiles[0] = "worker mutation"
|
||||
|
||||
prompts, err := engine.ListPrompts(context.Background())
|
||||
if err != nil {
|
||||
results <- fmt.Errorf("list prompts: %w", err)
|
||||
return
|
||||
}
|
||||
if len(prompts) == 0 {
|
||||
results <- errors.New("prompt listing is empty")
|
||||
return
|
||||
}
|
||||
prompts[0].ID = "worker mutation"
|
||||
prompts[0].Version = "worker mutation"
|
||||
|
||||
if _, err := engine.InspectProfile(context.Background(), "profile"); err != nil {
|
||||
results <- fmt.Errorf("inspect profile: %w", err)
|
||||
return
|
||||
}
|
||||
if _, err := engine.InspectPrompt(context.Background(), "prompt", "v1"); err != nil {
|
||||
results <- fmt.Errorf("inspect prompt: %w", err)
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
close(start)
|
||||
wait.Wait()
|
||||
close(results)
|
||||
for err := range results {
|
||||
t.Error(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListIdentitiesUseCapturedSourcesWithoutExecutionWork(t *testing.T) {
|
||||
files := fstest.MapFS{
|
||||
"config/listing.yaml": &fstest.MapFile{Data: []byte("profiles:\n captured-partial:\n model: partial-model\nprompts:\n captured-prompt:\n v1:\n messages:\n - role: user\n content: Hello.\n output:\n format: text\n validation_mode: basic\n")},
|
||||
}
|
||||
inputs, err := promptkit.FilesystemInputs("application listing", files, "config")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||
Underlays: []configkit.Underlay{{Inputs: inputs}},
|
||||
}}, promptkit.WithLLMClient(rejectingListingLLMClient{}), promptkit.WithArtifactReader(rejectingListingArtifactReader{}))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
files["config/listing.yaml"].Data = []byte("profiles:\n replacement-profile: {}\nprompts:\n replacement-prompt:\n v2:\n messages:\n - role: user\n content: Changed.\n output:\n format: text\n validation_mode: basic\n")
|
||||
|
||||
profiles, err := engine.ListProfiles(context.Background())
|
||||
if err != nil || !containsListingProfile(profiles, "captured-partial") || containsListingProfile(profiles, "replacement-profile") {
|
||||
t.Fatalf("profile listing did not use captured source: %#v, %v", profiles, err)
|
||||
}
|
||||
prompts, err := engine.ListPrompts(context.Background())
|
||||
if err != nil || countListingPrompt(prompts, "captured-prompt", "v1") != 1 || countListingPrompt(prompts, "replacement-prompt", "v2") != 0 {
|
||||
t.Fatalf("prompt listing did not use captured source: %#v, %v", prompts, err)
|
||||
}
|
||||
}
|
||||
|
||||
func listingContractPrompt() map[string]any {
|
||||
return map[string]any{
|
||||
"messages": []any{map[string]any{"role": "user", "content": "Hello."}},
|
||||
"output": map[string]any{"format": "text", "validation_mode": "basic"},
|
||||
}
|
||||
}
|
||||
|
||||
func containsListingProfile(profiles []string, id string) bool {
|
||||
for _, profile := range profiles {
|
||||
if profile == id {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func countListingProfile(profiles []string, id string) int {
|
||||
count := 0
|
||||
for _, profile := range profiles {
|
||||
if profile == id {
|
||||
count++
|
||||
}
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
func countListingPrompt(prompts []promptkit.PromptIdentity, id, version string) int {
|
||||
count := 0
|
||||
for _, prompt := range prompts {
|
||||
if prompt.ID == id && prompt.Version == version {
|
||||
count++
|
||||
}
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
func newListingContractEngine(t *testing.T, values map[string]any, options ...promptkit.Option) (*promptkit.Engine, error) {
|
||||
t.Helper()
|
||||
input := configkit.Input{Source: configkit.Mapping("listing contract", values)}
|
||||
return promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &input}}, options...)
|
||||
}
|
||||
|
||||
type rejectingListingLLMClient struct{}
|
||||
|
||||
func (rejectingListingLLMClient) Generate(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
return nil, errors.New("unexpected model generation")
|
||||
}
|
||||
|
||||
type rejectingListingArtifactReader struct{}
|
||||
|
||||
func (rejectingListingArtifactReader) Read(context.Context, promptkit.ArtifactRef) (*promptkit.Artifact, error) {
|
||||
return nil, errors.New("unexpected artifact read")
|
||||
}
|
||||
29
releasekit.json
Normal file
29
releasekit.json
Normal file
@@ -0,0 +1,29 @@
|
||||
{
|
||||
"product": "Promptkit",
|
||||
"remote": "origin",
|
||||
"validation": {
|
||||
"go": {
|
||||
"race": true,
|
||||
"full_vet": true
|
||||
},
|
||||
"go_format": true,
|
||||
"markdown": {},
|
||||
"module_distribution": {
|
||||
"workspace_files": true,
|
||||
"local_replacements": true
|
||||
}
|
||||
},
|
||||
"checks": [
|
||||
{
|
||||
"args": ["python3", "./scripts/check-examples.py"],
|
||||
"env": {
|
||||
"GOTOOLCHAIN": "local",
|
||||
"GOWORK": "off",
|
||||
"GOFLAGS": "-mod=readonly",
|
||||
"GOPROXY": "off",
|
||||
"GONOPROXY": "none",
|
||||
"GOSUMDB": "off"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
240
requirements.go
Normal file
240
requirements.go
Normal file
@@ -0,0 +1,240 @@
|
||||
package promptkit
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/usecase"
|
||||
)
|
||||
|
||||
// RequirementInspectionRequest selects a prompt and supplies caller context
|
||||
// for requirement discovery. It requires no artifact bindings or credentials.
|
||||
type RequirementInspectionRequest struct {
|
||||
// PromptID is the exact, case-sensitive configured prompt ID.
|
||||
PromptID string
|
||||
// PromptVersion selects an exact version. An empty value uses the prompt
|
||||
// selection rule that requires exactly one configured version.
|
||||
PromptVersion string
|
||||
// ProfileID selects the exact effective profile identity. If empty, the
|
||||
// prompt's default profile is used; the profile need not be ready for
|
||||
// execution.
|
||||
ProfileID string
|
||||
// Vars supplies string values for caller-variable lookups and simple
|
||||
// conditions. Presence is determined by exact map key, including an empty
|
||||
// value. Inspection reports caller keys and presence, not their values.
|
||||
// A supplied value may nevertheless appear as a resolved input identifier.
|
||||
Vars map[string]string
|
||||
// SessionID, when nonblank after the normal session-ID normalization,
|
||||
// bypasses the prompt's session template during discovery.
|
||||
SessionID string
|
||||
}
|
||||
|
||||
// RequirementKind describes how a selected prompt may depend on an input or
|
||||
// caller variable for the supplied context. It is independent of whether a
|
||||
// caller-variable value is present.
|
||||
type RequirementKind string
|
||||
|
||||
const (
|
||||
// RequirementRequired identifies an input declared required or a dependency
|
||||
// reached by a recognized definite access.
|
||||
RequirementRequired RequirementKind = "required"
|
||||
// RequirementConditional identifies a dependency whose reachability or
|
||||
// identity remains unresolved for the supplied context.
|
||||
RequirementConditional RequirementKind = "conditional"
|
||||
// RequirementOptional identifies a dependency known not to be required,
|
||||
// including a nonrequired declaration, optional lookup, or reference used
|
||||
// only on a known skipped path.
|
||||
RequirementOptional RequirementKind = "optional"
|
||||
)
|
||||
|
||||
// InputRequirement describes a declared or template-referenced prompt input.
|
||||
type InputRequirement struct {
|
||||
// Name is the exact input identifier.
|
||||
Name string
|
||||
// Kind is the strongest classification established for this name.
|
||||
Kind RequirementKind
|
||||
// Declared reports whether the name appears in the effective input
|
||||
// declarations; helper-only references have Declared false.
|
||||
Declared bool
|
||||
// ContentType preserves effective metadata presence: nil means absent or
|
||||
// null, while a non-nil pointer may refer to an empty string.
|
||||
ContentType *string
|
||||
// Description preserves effective metadata presence independently of
|
||||
// Explanation, including an explicitly empty string.
|
||||
Description *string
|
||||
// Explanation is safe human-readable context, such as why a reference is
|
||||
// optional because it occurs only on a known skipped path. It is not a
|
||||
// machine-readable contract.
|
||||
Explanation string
|
||||
}
|
||||
|
||||
// VariableRequirement describes a caller-map key referenced by a prompt.
|
||||
type VariableRequirement struct {
|
||||
// Name is the exact caller-map key, not a template-local name.
|
||||
Name string
|
||||
// Kind is the strongest classification established for this key.
|
||||
Kind RequirementKind
|
||||
// Supplied reports exact-key presence in the request Vars map. An empty
|
||||
// string is supplied; the value itself is not returned.
|
||||
Supplied bool
|
||||
// Explanation is safe human-readable context and is not a machine-readable
|
||||
// contract.
|
||||
Explanation string
|
||||
}
|
||||
|
||||
// RequirementIssueCode identifies a limitation that can affect discovery.
|
||||
type RequirementIssueCode string
|
||||
|
||||
const (
|
||||
// RequirementIssueMissingVariable means an absent caller value prevents
|
||||
// resolving a dependency decision or input name.
|
||||
RequirementIssueMissingVariable RequirementIssueCode = "missing_variable"
|
||||
// RequirementIssueInputContent means unavailable input content affects
|
||||
// dependency discovery.
|
||||
RequirementIssueInputContent RequirementIssueCode = "input_content"
|
||||
// RequirementIssueDynamicInputName means a referenced input name cannot be
|
||||
// determined for the supplied context.
|
||||
RequirementIssueDynamicInputName RequirementIssueCode = "dynamic_input_name"
|
||||
// RequirementIssueUnsupportedConstruct means a dependency-relevant
|
||||
// expression or scope is outside the supported discovery analysis.
|
||||
RequirementIssueUnsupportedConstruct RequirementIssueCode = "unsupported_construct"
|
||||
)
|
||||
|
||||
// RequirementIssue describes one discovery limitation using safe summary
|
||||
// text. Descriptions are not intended as machine-readable details.
|
||||
type RequirementIssue struct {
|
||||
// Code is the machine-readable limitation category.
|
||||
Code RequirementIssueCode
|
||||
// InputName identifies an affected input when one is known; empty means no
|
||||
// particular input was identified.
|
||||
InputName string
|
||||
// VariableName identifies an affected caller key when one is known; empty
|
||||
// means no particular key was identified.
|
||||
VariableName string
|
||||
// Description is a safe human-readable summary, not a string to parse.
|
||||
Description string
|
||||
}
|
||||
|
||||
// RequirementInspection contains discovered dependencies for one selected
|
||||
// prompt context. Input and variable entries are sorted by exact name. Issues
|
||||
// are deduplicated by code and associated names, then sorted by that tuple.
|
||||
// Their descriptions and entry explanations are safe summaries, not
|
||||
// machine-readable details.
|
||||
//
|
||||
// Complete describes the supported analysis's discovery coverage, not whether
|
||||
// requirements are supplied, templates will render, artifacts or schemas are
|
||||
// valid, or execution is ready. A complete result can still contain required
|
||||
// inputs or missing required variables. Valid templates using dependency-
|
||||
// relevant constructs outside the deliberately small recognition scope may
|
||||
// produce useful partial inventories with Complete false; this is not a
|
||||
// template-definition error or an authoring prohibition.
|
||||
type RequirementInspection struct {
|
||||
// PromptID is the selected exact prompt ID.
|
||||
PromptID string
|
||||
// PromptVersion is the selected exact prompt version.
|
||||
PromptVersion string
|
||||
// ProfileID is the selected profile identity; it does not indicate
|
||||
// execution readiness.
|
||||
ProfileID string
|
||||
// Inputs contains declared inputs and recognizable template references,
|
||||
// including entries known to be optional.
|
||||
Inputs []InputRequirement
|
||||
// Variables contains recognizable caller-map keys, including keys used
|
||||
// only on known skipped paths.
|
||||
Variables []VariableRequirement
|
||||
// Complete reports discovery coverage, not requirement satisfaction or
|
||||
// execution readiness.
|
||||
Complete bool
|
||||
// Issues contains deduplicated, structured discovery limitations.
|
||||
Issues []RequirementIssue
|
||||
}
|
||||
|
||||
// InspectRequirements discovers inputs and caller variables for one selected
|
||||
// prompt before its input artifacts exist. It uses exact, case-sensitive
|
||||
// prompt selection and requires a profile ID either in the request or the
|
||||
// selected prompt's default. The selected profile establishes identity only;
|
||||
// it need not be execution-ready. A normalized nonblank SessionID bypasses the
|
||||
// captured session template. Vars are copied before analysis, and returned
|
||||
// slices and metadata pointers belong to the caller.
|
||||
//
|
||||
// Discovery recognizes literal input names, direct caller-map keys, literal-
|
||||
// key root-map lookups, simple caller-derived input names, and if/with
|
||||
// conditions based on direct values, literals, negation, or two-string
|
||||
// equality/inequality. It classifies recognizable references on known skipped
|
||||
// paths as optional. Other dependency-relevant expressions or scopes may leave
|
||||
// a useful partial result with Complete false. The returned inventories do
|
||||
// not reject otherwise valid templates or prohibit their use.
|
||||
//
|
||||
// Results exclude artifact payloads, prompt contents, and unrelated supplied
|
||||
// values. Resolved input identifiers may equal the caller values that supply
|
||||
// them; those identifiers are intentionally included in the inventory.
|
||||
//
|
||||
// This method does not perform artifact, schema, credential, rendering,
|
||||
// validation, capacity, observation, provider, or execution work. Complete
|
||||
// describes dependency discovery, not whether every dependency is supplied or
|
||||
// the prompt is ready to execute. Discovery limitations return a partial
|
||||
// inspection and no error; selection and operational failures return no
|
||||
// inspection. Blank prompt IDs, invalid session IDs, and absent profile
|
||||
// selection match ErrInvalidRequest; an absent profile selection also matches
|
||||
// ErrProfileRequired. Missing prompt/profile IDs match ErrPromptNotFound or
|
||||
// ErrProfileNotFound. Other selection/snapshot failures match ErrPromptLoad or
|
||||
// ErrProfileLoad. When cancellation prevents lookup or analysis, no inspection
|
||||
// is returned and the context error remains discoverable with errors.Is. A nil
|
||||
// Engine matches ErrInvalidConfig.
|
||||
func (e *Engine) InspectRequirements(
|
||||
ctx context.Context,
|
||||
req RequirementInspectionRequest,
|
||||
) (*RequirementInspection, error) {
|
||||
if e == nil || e.internal == nil {
|
||||
return nil, ErrInvalidConfig
|
||||
}
|
||||
inspection, err := e.internal.InspectRequirements(ctx, usecase.RequirementInspectionRequest{
|
||||
PromptID: req.PromptID, PromptVersion: req.PromptVersion, ProfileID: req.ProfileID,
|
||||
Vars: copyStringMap(req.Vars), SessionID: req.SessionID,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, mapPublicError(err)
|
||||
}
|
||||
return fromDomainRequirementInspection(inspection), nil
|
||||
}
|
||||
|
||||
func fromDomainRequirementInspection(inspection *domain.RequirementInspection) *RequirementInspection {
|
||||
if inspection == nil {
|
||||
return nil
|
||||
}
|
||||
result := &RequirementInspection{
|
||||
PromptID: inspection.PromptID, PromptVersion: inspection.PromptVersion,
|
||||
ProfileID: inspection.ProfileID, Complete: inspection.Complete,
|
||||
Inputs: make([]InputRequirement, len(inspection.Inputs)),
|
||||
Variables: make([]VariableRequirement, len(inspection.Variables)),
|
||||
Issues: make([]RequirementIssue, len(inspection.Issues)),
|
||||
}
|
||||
for index, input := range inspection.Inputs {
|
||||
result.Inputs[index] = InputRequirement{
|
||||
Name: input.Name, Kind: RequirementKind(input.Kind), Declared: input.Declared,
|
||||
ContentType: copyRequirementString(input.ContentType), Description: copyRequirementString(input.Description),
|
||||
Explanation: input.Explanation,
|
||||
}
|
||||
}
|
||||
for index, variable := range inspection.Variables {
|
||||
result.Variables[index] = VariableRequirement{
|
||||
Name: variable.Name, Kind: RequirementKind(variable.Kind), Supplied: variable.Supplied,
|
||||
Explanation: variable.Explanation,
|
||||
}
|
||||
}
|
||||
for index, issue := range inspection.Issues {
|
||||
result.Issues[index] = RequirementIssue{
|
||||
Code: RequirementIssueCode(issue.Code), InputName: issue.InputName,
|
||||
VariableName: issue.VariableName, Description: issue.Description,
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func copyRequirementString(value *string) *string {
|
||||
if value == nil {
|
||||
return nil
|
||||
}
|
||||
copy := *value
|
||||
return ©
|
||||
}
|
||||
72
requirements_inspection_benchmark_test.go
Normal file
72
requirements_inspection_benchmark_test.go
Normal file
@@ -0,0 +1,72 @@
|
||||
package promptkit_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
var inspectedRequirements *promptkit.RequirementInspection
|
||||
|
||||
func requirementInspectionExtras(entries int) map[string]any {
|
||||
providers := make(map[string]any, entries)
|
||||
for index := range entries {
|
||||
providers[fmt.Sprintf("provider-%03d", index)] = map[string]any{
|
||||
"enabled": true,
|
||||
"labels": []any{fmt.Sprintf("tier-%d", index%4), fmt.Sprintf("region-%d", index%3)},
|
||||
"limits": map[string]any{"burst": index + 1, "period": "minute"},
|
||||
"routes": []any{map[string]any{"weight": index + 1, "fallback": true}},
|
||||
}
|
||||
}
|
||||
return map[string]any{"provider_options": providers}
|
||||
}
|
||||
|
||||
func BenchmarkRequirementInspection(b *testing.B) {
|
||||
profiles := map[string]any{
|
||||
"small": map[string]any{
|
||||
"model": "inspection-model", "extra_params": requirementInspectionExtras(2),
|
||||
},
|
||||
"large": map[string]any{
|
||||
"model": "inspection-model", "extra_params": requirementInspectionExtras(64),
|
||||
},
|
||||
}
|
||||
definition := contractPrompt("small", "Hello {{.name}} {{input \"transcript\"}}", []any{
|
||||
map[string]any{"name": "transcript", "required": true},
|
||||
}, "", noValidationOutput)
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
|
||||
Source: configkit.Mapping("requirement inspection benchmark", map[string]any{
|
||||
"prompts": oneVersionPrompt("inspection", "1", definition),
|
||||
"profiles": profiles,
|
||||
}),
|
||||
}}})
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
ctx := context.Background()
|
||||
for _, profileID := range []string{"small", "large"} {
|
||||
request := promptkit.RequirementInspectionRequest{
|
||||
PromptID: "inspection", ProfileID: profileID, Vars: map[string]string{"name": "Ada"},
|
||||
}
|
||||
inspection, err := engine.InspectRequirements(ctx, request)
|
||||
if err != nil || inspection.PromptID != "inspection" || inspection.PromptVersion != "1" ||
|
||||
inspection.ProfileID != profileID || !inspection.Complete || len(inspection.Inputs) != 1 ||
|
||||
inspection.Inputs[0].Name != "transcript" || inspection.Inputs[0].Kind != promptkit.RequirementRequired ||
|
||||
len(inspection.Variables) != 1 || inspection.Variables[0].Name != "name" || !inspection.Variables[0].Supplied {
|
||||
b.Fatalf("requirement inspection fixture for %q is invalid: inspection=%+v err=%v", profileID, inspection, err)
|
||||
}
|
||||
selectedRequest := request
|
||||
b.Run(profileID+"_extra", func(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
for b.Loop() {
|
||||
inspection, err := engine.InspectRequirements(ctx, selectedRequest)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
inspectedRequirements = inspection
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
370
requirements_inspection_contract_test.go
Normal file
370
requirements_inspection_contract_test.go
Normal file
@@ -0,0 +1,370 @@
|
||||
package promptkit_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"testing/fstest"
|
||||
|
||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
func requirementsEngine(t *testing.T, prompts, profiles map[string]any, options ...promptkit.Option) *promptkit.Engine {
|
||||
t.Helper()
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
|
||||
Source: configkit.Mapping("requirements contract", map[string]any{
|
||||
"prompts": prompts, "profiles": profiles,
|
||||
}),
|
||||
}}}, options...)
|
||||
if err != nil {
|
||||
t.Fatalf("construct requirements engine: %v", err)
|
||||
}
|
||||
return engine
|
||||
}
|
||||
|
||||
type requirementCountingArtifactReader struct{ calls atomic.Int64 }
|
||||
|
||||
func (reader *requirementCountingArtifactReader) Read(context.Context, promptkit.ArtifactRef) (*promptkit.Artifact, error) {
|
||||
reader.calls.Add(1)
|
||||
return nil, errors.New("artifact reader must not be called")
|
||||
}
|
||||
|
||||
func contractPrompt(defaultProfile string, content string, inputs []any, session string, output map[string]any) map[string]any {
|
||||
definition := map[string]any{
|
||||
"messages": []any{map[string]any{"role": "user", "content": content}},
|
||||
"output": output,
|
||||
}
|
||||
if defaultProfile != "" {
|
||||
definition["default_profile"] = defaultProfile
|
||||
}
|
||||
if inputs != nil {
|
||||
definition["inputs"] = inputs
|
||||
}
|
||||
if session != "" {
|
||||
definition["session_id"] = session
|
||||
}
|
||||
return definition
|
||||
}
|
||||
|
||||
func oneVersionPrompt(id, version string, definition map[string]any) map[string]any {
|
||||
return map[string]any{id: map[string]any{version: definition}}
|
||||
}
|
||||
|
||||
var noValidationOutput = map[string]any{"format": "text", "validation_mode": "none"}
|
||||
|
||||
func TestInspectRequirementsEnumeratesCapturedDependenciesAndInactiveNames(t *testing.T) {
|
||||
var modelCalls atomic.Int64
|
||||
reader := &requirementCountingArtifactReader{}
|
||||
definition := contractPrompt("partial", `TEMPLATE_PRIVATE_MARKER {{if eq .mode "brief"}}{{input "brief"}}{{else}}{{input "long"}}{{end}} {{input "undeclared"}} {{.required}} {{if false}}{{input "inactive"}}{{.inactive_var}}{{end}}`, []any{
|
||||
map[string]any{"name": "transcript", "required": true, "content_type": "audio/wav", "description": "Transcript source."},
|
||||
map[string]any{"name": "optional", "content_type": "text/plain", "description": ""},
|
||||
}, `{{input "session-only"}} {{.session_variable}}`, map[string]any{
|
||||
"format": "json", "validation_mode": "json_schema", "schema_path": "not-loaded.json",
|
||||
})
|
||||
engine := requirementsEngine(t, oneVersionPrompt("requirements", "v1", definition), map[string]any{
|
||||
"partial": map[string]any{"model": "partial-model", "api_key_required": true},
|
||||
}, promptkit.WithArtifactReader(reader), promptkit.WithLLMClient(countingLLMClient{calls: &modelCalls}))
|
||||
inspection, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{
|
||||
PromptID: "requirements", Vars: map[string]string{"mode": "brief", "unrelated": "VARIABLE_PRIVATE_MARKER"},
|
||||
SessionID: "SESSION_PRIVATE_MARKER",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("inspect before artifacts, credentials, and schemas exist: %v", err)
|
||||
}
|
||||
if inspection.PromptID != "requirements" || inspection.PromptVersion != "v1" || inspection.ProfileID != "partial" || !inspection.Complete {
|
||||
t.Fatalf("selected context or completeness is wrong: %+v", inspection)
|
||||
}
|
||||
inputs := make(map[string]promptkit.InputRequirement, len(inspection.Inputs))
|
||||
for _, input := range inspection.Inputs {
|
||||
inputs[input.Name] = input
|
||||
}
|
||||
if len(inputs) != 6 || inputs["transcript"].Kind != promptkit.RequirementRequired || !inputs["transcript"].Declared ||
|
||||
inputs["transcript"].ContentType == nil || *inputs["transcript"].ContentType != "audio/wav" ||
|
||||
inputs["transcript"].Description == nil || *inputs["transcript"].Description != "Transcript source." {
|
||||
t.Fatalf("required declaration metadata or discovered inputs are wrong: %+v", inspection.Inputs)
|
||||
}
|
||||
if inputs["brief"].Kind != promptkit.RequirementRequired || inputs["long"].Kind != promptkit.RequirementOptional || inputs["long"].Explanation == "" ||
|
||||
inputs["inactive"].Kind != promptkit.RequirementOptional || inputs["inactive"].Explanation == "" ||
|
||||
inputs["undeclared"].Kind != promptkit.RequirementRequired || inputs["undeclared"].Declared ||
|
||||
inputs["optional"].Kind != promptkit.RequirementOptional || !inputs["optional"].Declared ||
|
||||
inputs["optional"].Description == nil || *inputs["optional"].Description != "" {
|
||||
t.Fatalf("selected, inactive, or helper-only input classifications are wrong: %+v", inspection.Inputs)
|
||||
}
|
||||
if _, found := inputs["session-only"]; found {
|
||||
t.Fatalf("bypassed session input appeared in inventory: %+v", inspection.Inputs)
|
||||
}
|
||||
variables := make(map[string]promptkit.VariableRequirement, len(inspection.Variables))
|
||||
for _, variable := range inspection.Variables {
|
||||
variables[variable.Name] = variable
|
||||
}
|
||||
if len(variables) != 3 || variables["mode"].Kind != promptkit.RequirementRequired || !variables["mode"].Supplied ||
|
||||
variables["required"].Kind != promptkit.RequirementRequired || variables["required"].Supplied ||
|
||||
variables["inactive_var"].Kind != promptkit.RequirementOptional || variables["inactive_var"].Explanation == "" {
|
||||
t.Fatalf("caller-variable inventory conflated need and presence: %+v", inspection.Variables)
|
||||
}
|
||||
if _, found := variables["unrelated"]; found {
|
||||
t.Fatalf("unreferenced caller key entered the inventory: %+v", inspection.Variables)
|
||||
}
|
||||
formatted := fmt.Sprintf("%+v", inspection)
|
||||
for _, private := range []string{
|
||||
"TEMPLATE_PRIVATE_MARKER", "VARIABLE_PRIVATE_MARKER", "SESSION_PRIVATE_MARKER",
|
||||
} {
|
||||
if strings.Contains(formatted, private) {
|
||||
t.Fatalf("inspection disclosed %q: %s", private, formatted)
|
||||
}
|
||||
}
|
||||
if reader.calls.Load() != 0 || modelCalls.Load() != 0 {
|
||||
t.Fatalf("inspection invoked execution collaborators: artifact reads=%d provider calls=%d", reader.calls.Load(), modelCalls.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestInspectRequirementsSelectionUsesExactVersionsAndProfileIdentity(t *testing.T) {
|
||||
prompts := map[string]any{
|
||||
"multi": map[string]any{
|
||||
"v1": contractPrompt("first", `{{input "one"}}`, nil, "", noValidationOutput),
|
||||
"v2": contractPrompt("second", `{{input "two"}}`, nil, "", noValidationOutput),
|
||||
},
|
||||
"without-default": map[string]any{
|
||||
"v1": contractPrompt("", "plain", nil, "", noValidationOutput),
|
||||
},
|
||||
"direct-required": map[string]any{"v1": contractPrompt("direct", "plain", nil, "", noValidationOutput)},
|
||||
}
|
||||
profiles := map[string]any{
|
||||
"first": map[string]any{"model": "first-model"},
|
||||
"second": map[string]any{"model": "second-model"},
|
||||
"direct": map[string]any{"endpoint": "https://example.test/v1", "model": "direct-model", "api_key_required": true},
|
||||
}
|
||||
engine := requirementsEngine(t, prompts, profiles)
|
||||
if _, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{PromptID: "multi"}); !errors.Is(err, promptkit.ErrPromptLoad) {
|
||||
t.Fatalf("ambiguous omitted version error=%v", err)
|
||||
}
|
||||
first, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{PromptID: "multi", PromptVersion: "v1"})
|
||||
if err != nil || first.PromptVersion != "v1" || first.ProfileID != "first" || len(first.Inputs) != 1 || first.Inputs[0].Name != "one" {
|
||||
t.Fatalf("exact first version/default profile=(%+v, %v)", first, err)
|
||||
}
|
||||
explicit, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{
|
||||
PromptID: "multi", PromptVersion: "v2", ProfileID: "first",
|
||||
})
|
||||
if err != nil || explicit.PromptVersion != "v2" || explicit.ProfileID != "first" || explicit.Inputs[0].Name != "two" {
|
||||
t.Fatalf("exact version/request profile=(%+v, %v)", explicit, err)
|
||||
}
|
||||
if _, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{PromptID: "Multi", PromptVersion: "v1"}); !errors.Is(err, promptkit.ErrPromptNotFound) {
|
||||
t.Fatalf("case-folded prompt ID unexpectedly selected: %v", err)
|
||||
}
|
||||
if _, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{PromptID: "multi", PromptVersion: "V1"}); !errors.Is(err, promptkit.ErrPromptNotFound) {
|
||||
t.Fatalf("case-folded version unexpectedly selected: %v", err)
|
||||
}
|
||||
if _, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{PromptID: "multi", PromptVersion: "v1", ProfileID: "missing"}); !errors.Is(err, promptkit.ErrProfileNotFound) {
|
||||
t.Fatalf("missing profile error=%v", err)
|
||||
}
|
||||
if _, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{PromptID: "without-default"}); !errors.Is(err, promptkit.ErrProfileRequired) || !errors.Is(err, promptkit.ErrInvalidRequest) {
|
||||
t.Fatalf("missing profile selection error=%v", err)
|
||||
}
|
||||
if direct, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{PromptID: "direct-required"}); err != nil || direct.ProfileID != "direct" {
|
||||
t.Fatalf("direct-key-required profile should be inspectable without a key: (%+v, %v)", direct, err)
|
||||
}
|
||||
if _, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{}); !errors.Is(err, promptkit.ErrInvalidRequest) {
|
||||
t.Fatalf("blank prompt error=%v", err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
if _, err := engine.InspectRequirements(ctx, promptkit.RequirementInspectionRequest{PromptID: "multi", PromptVersion: "v1"}); !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("canceled inspection error=%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInspectPromptHashMatchesPreparationForCapturedConfiguration(t *testing.T) {
|
||||
prompts := oneVersionPrompt("hash-equality", "v1", contractPrompt(
|
||||
"profile", "Hello {{.name}}", nil, "", noValidationOutput,
|
||||
))
|
||||
engine := requirementsEngine(t, prompts, map[string]any{
|
||||
"profile": map[string]any{"endpoint": "https://example.test/v1", "model": "inspection-model"},
|
||||
})
|
||||
inspection, err := engine.InspectPrompt(context.Background(), "hash-equality", "v1")
|
||||
if err != nil {
|
||||
t.Fatalf("inspect captured prompt: %v", err)
|
||||
}
|
||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||
PromptID: "hash-equality", PromptVersion: "v1", Vars: map[string]string{"name": "Ada"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("prepare captured prompt: %v", err)
|
||||
}
|
||||
if inspection.PromptHash != prepared.PromptHash {
|
||||
t.Fatalf("inspection hash=%q, preparation hash=%q", inspection.PromptHash, prepared.PromptHash)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInspectRequirementsReportsConditionalAndCompleteOptionalInventories(t *testing.T) {
|
||||
prompts := map[string]any{
|
||||
"conditional": map[string]any{"v1": contractPrompt("profile",
|
||||
`{{if .mode}}{{input "left"}}{{else}}{{input "right"}}{{end}}`, nil, "", noValidationOutput)},
|
||||
"optional": map[string]any{"v1": contractPrompt("profile", `{{index . "lookup"}}`,
|
||||
[]any{map[string]any{"name": "declared-optional"}}, "", noValidationOutput)},
|
||||
"empty": map[string]any{"v1": contractPrompt("profile", "plain", nil, "", noValidationOutput)},
|
||||
}
|
||||
engine := requirementsEngine(t, prompts, map[string]any{"profile": map[string]any{"model": "partial"}})
|
||||
conditional, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{PromptID: "conditional"})
|
||||
if err != nil || conditional.Complete || len(conditional.Issues) != 1 ||
|
||||
conditional.Issues[0].Code != promptkit.RequirementIssueMissingVariable || len(conditional.Inputs) != 2 ||
|
||||
conditional.Inputs[0].Kind != promptkit.RequirementConditional || conditional.Inputs[1].Kind != promptkit.RequirementConditional {
|
||||
t.Fatalf("conditional discovery=(%+v, %v)", conditional, err)
|
||||
}
|
||||
optional, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{PromptID: "optional"})
|
||||
if err != nil || !optional.Complete || len(optional.Issues) != 0 || len(optional.Inputs) != 1 ||
|
||||
optional.Inputs[0].Kind != promptkit.RequirementOptional || len(optional.Variables) != 1 ||
|
||||
optional.Variables[0].Name != "lookup" || optional.Variables[0].Kind != promptkit.RequirementOptional {
|
||||
t.Fatalf("all-optional result should be complete without artifact bindings: (%+v, %v)", optional, err)
|
||||
}
|
||||
empty, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{PromptID: "empty"})
|
||||
if err != nil || !empty.Complete || len(empty.Inputs) != 0 || len(empty.Variables) != 0 {
|
||||
t.Fatalf("empty complete result=(%+v, %v)", empty, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInspectRequirementsBeforePreparationAndWithDirectSessionBypass(t *testing.T) {
|
||||
definition := contractPrompt("profile", `Name={{.name}} Transcript={{input "transcript"}}{{if false}}ignored {{input "skipped"}} {{.inactive}}{{end}}`, []any{
|
||||
map[string]any{"name": "transcript", "required": true},
|
||||
}, `{{.session_var}} {{input "session-only"}}`, noValidationOutput)
|
||||
engine := requirementsEngine(t, oneVersionPrompt("ready", "v1", definition), map[string]any{
|
||||
"profile": map[string]any{"endpoint": "https://example.test/v1", "model": "offline"},
|
||||
})
|
||||
inspection, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{
|
||||
PromptID: "ready", Vars: map[string]string{"name": "Ada"}, SessionID: "direct-session",
|
||||
})
|
||||
if err != nil || inspection == nil || !inspection.Complete || len(inspection.Inputs) != 2 ||
|
||||
inspection.Inputs[0].Name != "skipped" || inspection.Inputs[0].Kind != promptkit.RequirementOptional ||
|
||||
inspection.Inputs[1].Name != "transcript" || inspection.Inputs[1].Kind != promptkit.RequirementRequired ||
|
||||
len(inspection.Variables) != 2 || inspection.Variables[0].Name != "inactive" || inspection.Variables[0].Kind != promptkit.RequirementOptional ||
|
||||
inspection.Variables[1].Name != "name" || inspection.Variables[1].Kind != promptkit.RequirementRequired {
|
||||
t.Fatalf("artifact-free inspection=(%+v, %v)", inspection, err)
|
||||
}
|
||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||
PromptID: "ready", Vars: map[string]string{"name": "Ada"}, SessionID: "direct-session",
|
||||
Inputs: map[string]promptkit.ArtifactRef{"transcript": promptkit.Inline("source transcript")},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("prepare with matching synthetic artifact: %v", err)
|
||||
}
|
||||
if prepared.Messages[0].Content != "Name=Ada Transcript=source transcript" || prepared.SessionID != "direct-session" {
|
||||
t.Fatalf("prepared run did not use the inspected dependencies: %+v", prepared)
|
||||
}
|
||||
if _, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{
|
||||
PromptID: "ready", Vars: map[string]string{"name": "Ada"},
|
||||
}); err != nil {
|
||||
t.Fatalf("inspect without session bypass: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInspectRequirementsUsesCapturedFileBackedTemplates(t *testing.T) {
|
||||
files := fstest.MapFS{
|
||||
"config/prompt.yaml": &fstest.MapFile{Data: []byte(capturedPromptDocument)},
|
||||
"config/body.tmpl": &fstest.MapFile{Data: []byte(`{{input "captured-input"}}`)},
|
||||
}
|
||||
source := &unreadableAfterConstructionFS{FS: files}
|
||||
engine := capturedPromptEngine(t, source)
|
||||
source.closed.Store(true)
|
||||
delete(files, "config/body.tmpl")
|
||||
inspection, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{PromptID: "captured"})
|
||||
if err != nil || inspection == nil || !inspection.Complete || len(inspection.Inputs) != 1 ||
|
||||
inspection.Inputs[0].Name != "captured-input" || source.reads.Load() != 0 {
|
||||
t.Fatalf("captured template inspection=(%+v, %v), post-construction reads=%d", inspection, err, source.reads.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestInspectRequirementsReturnsIndependentValuesAndCanOverlapPreparation(t *testing.T) {
|
||||
definition := contractPrompt("profile", `{{if .decision}}{{input "conditional"}}{{end}} {{input "declared"}}`, []any{
|
||||
map[string]any{"name": "declared", "required": true, "description": "captured description"},
|
||||
}, `{{.session}}`, noValidationOutput)
|
||||
engine := requirementsEngine(t, oneVersionPrompt("owned", "v1", definition), map[string]any{
|
||||
"profile": map[string]any{"endpoint": "https://example.test/v1", "model": "offline"},
|
||||
})
|
||||
request := promptkit.RequirementInspectionRequest{PromptID: "owned"}
|
||||
first, err := engine.InspectRequirements(context.Background(), request)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for index := range first.Inputs {
|
||||
if first.Inputs[index].Name == "declared" {
|
||||
first.Inputs[index].Name = "mutated"
|
||||
if first.Inputs[index].Description != nil {
|
||||
*first.Inputs[index].Description = "mutated description"
|
||||
}
|
||||
}
|
||||
}
|
||||
first.Variables[0].Name = "mutated-variable"
|
||||
first.Issues[0].Description = "mutated issue"
|
||||
second, err := engine.InspectRequirements(context.Background(), request)
|
||||
if err != nil {
|
||||
t.Fatalf("inspect after mutating returned values: %v", err)
|
||||
}
|
||||
var description *string
|
||||
for _, input := range second.Inputs {
|
||||
if input.Name == "declared" {
|
||||
description = input.Description
|
||||
}
|
||||
}
|
||||
if second.Inputs[0].Name == "mutated" || second.Variables[0].Name != "decision" ||
|
||||
description == nil || *description != "captured description" ||
|
||||
len(second.Issues) != 1 || second.Issues[0].Description == "mutated issue" {
|
||||
t.Fatalf("result mutation reached captured state or later result: (%+v, %v)", second, err)
|
||||
}
|
||||
|
||||
const count = 12
|
||||
var wait sync.WaitGroup
|
||||
errs := make(chan error, count*2)
|
||||
for index := 0; index < count; index++ {
|
||||
wait.Add(2)
|
||||
go func() {
|
||||
defer wait.Done()
|
||||
_, err := engine.InspectRequirements(context.Background(), request)
|
||||
errs <- err
|
||||
}()
|
||||
go func(index int) {
|
||||
defer wait.Done()
|
||||
_, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||
PromptID: "owned", SessionID: fmt.Sprintf("session-%d", index), Vars: map[string]string{"decision": "yes"},
|
||||
Inputs: map[string]promptkit.ArtifactRef{
|
||||
"conditional": promptkit.Inline("condition"), "declared": promptkit.Inline("body"),
|
||||
},
|
||||
})
|
||||
errs <- err
|
||||
}(index)
|
||||
}
|
||||
wait.Wait()
|
||||
close(errs)
|
||||
for err := range errs {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestInspectRequirementsDoesNotEchoUnrelatedPrivateValues(t *testing.T) {
|
||||
t.Setenv("PROMPTKIT_REQUIREMENTS_SECRET", "credential-value-marker")
|
||||
definition := contractPrompt("profile", `template-body-marker {{input "required-input"}} {{.known}}`, nil, `{{.session}}`, noValidationOutput)
|
||||
engine := requirementsEngine(t, oneVersionPrompt("privacy", "v1", definition), map[string]any{
|
||||
"profile": map[string]any{"endpoint": "https://example.test/v1", "model": "offline", "api_key_env": "PROMPTKIT_REQUIREMENTS_SECRET"},
|
||||
})
|
||||
inspection, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{
|
||||
PromptID: "privacy", Vars: map[string]string{"known": "known-value-marker", "unrelated": "unrelated-value-marker"},
|
||||
SessionID: "direct-session-marker",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
formatted := fmt.Sprintf("%+v", inspection)
|
||||
for _, marker := range []string{
|
||||
"template-body-marker", "known-value-marker", "unrelated-value-marker",
|
||||
"direct-session-marker", "credential-value-marker", "PROMPTKIT_REQUIREMENTS_SECRET",
|
||||
} {
|
||||
if strings.Contains(formatted, marker) {
|
||||
t.Fatalf("inspection disclosed %q: %s", marker, formatted)
|
||||
}
|
||||
}
|
||||
}
|
||||
15
requirements_test.go
Normal file
15
requirements_test.go
Normal file
@@ -0,0 +1,15 @@
|
||||
package promptkit
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestInspectRequirementsRejectsNilEngine(t *testing.T) {
|
||||
var engine *Engine
|
||||
inspection, err := engine.InspectRequirements(context.Background(), RequirementInspectionRequest{PromptID: "prompt"})
|
||||
if inspection != nil || !errors.Is(err, ErrInvalidConfig) {
|
||||
t.Fatalf("nil engine returned (%+v, %v), want ErrInvalidConfig", inspection, err)
|
||||
}
|
||||
}
|
||||
161
scripts/check-examples.py
Normal file
161
scripts/check-examples.py
Normal file
@@ -0,0 +1,161 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run the maintained consumer examples and check their semantic output."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
REPOSITORY_ROOT = Path(__file__).resolve().parent.parent
|
||||
|
||||
|
||||
def example_environment():
|
||||
environment = os.environ.copy()
|
||||
environment["GOTOOLCHAIN"] = "local"
|
||||
try:
|
||||
host = subprocess.run(
|
||||
["go", "env", "GOHOSTOS", "GOHOSTARCH"],
|
||||
cwd=REPOSITORY_ROOT,
|
||||
env=environment,
|
||||
text=True,
|
||||
capture_output=True,
|
||||
check=False,
|
||||
)
|
||||
except OSError as error:
|
||||
raise RuntimeError(f"could not run go env: {error}") from error
|
||||
|
||||
if host.returncode != 0:
|
||||
raise RuntimeError(
|
||||
"go env GOHOSTOS GOHOSTARCH failed "
|
||||
f"with exit status {host.returncode}: {host.stderr.strip()}"
|
||||
)
|
||||
|
||||
values = host.stdout.splitlines()
|
||||
if len(values) != 2 or not all(values):
|
||||
raise RuntimeError(
|
||||
"go env GOHOSTOS GOHOSTARCH returned unexpected output: "
|
||||
f"{host.stdout!r}"
|
||||
)
|
||||
|
||||
environment["GOOS"], environment["GOARCH"] = values
|
||||
return environment
|
||||
|
||||
|
||||
def run_example(name, environment, failures):
|
||||
command = ["go", "run", f"./examples/go-library/{name}"]
|
||||
try:
|
||||
result = subprocess.run(
|
||||
command,
|
||||
cwd=REPOSITORY_ROOT,
|
||||
env=environment,
|
||||
text=True,
|
||||
capture_output=True,
|
||||
check=False,
|
||||
)
|
||||
except OSError as error:
|
||||
failures.append(f"{name}: could not start {command[0]}: {error}")
|
||||
return None
|
||||
|
||||
if result.returncode != 0:
|
||||
failures.append(f"{name}: command exited with status {result.returncode}")
|
||||
if result.stderr:
|
||||
failures.append(f"{name}: stderr:\n{result.stderr.rstrip()}")
|
||||
if result.stdout:
|
||||
failures.append(f"{name}: stdout:\n{result.stdout.rstrip()}")
|
||||
return None
|
||||
|
||||
try:
|
||||
value = json.loads(result.stdout)
|
||||
except json.JSONDecodeError as error:
|
||||
failures.append(f"{name}: output is not valid JSON: {error}")
|
||||
if result.stdout:
|
||||
failures.append(f"{name}: stdout:\n{result.stdout.rstrip()}")
|
||||
return None
|
||||
|
||||
if not isinstance(value, dict):
|
||||
failures.append(
|
||||
f"{name}: expected a JSON object, got {type(value).__name__}"
|
||||
)
|
||||
return None
|
||||
|
||||
return value
|
||||
|
||||
|
||||
def expect_equal(example_name, value, key, expected, failures):
|
||||
actual = value.get(key)
|
||||
matches = actual is expected if isinstance(expected, bool) else actual == expected
|
||||
if not matches:
|
||||
failures.append(
|
||||
f"{example_name}: expected {key}={expected!r}, got {actual!r}"
|
||||
)
|
||||
|
||||
|
||||
def expect_positive(example_name, value, key, failures):
|
||||
actual = value.get(key)
|
||||
try:
|
||||
matches = actual > 0
|
||||
except TypeError:
|
||||
matches = False
|
||||
if not matches:
|
||||
failures.append(f"{example_name}: expected {key} to be positive, got {actual!r}")
|
||||
|
||||
|
||||
def main():
|
||||
try:
|
||||
environment = example_environment()
|
||||
except RuntimeError as error:
|
||||
print(error, file=sys.stderr)
|
||||
return 1
|
||||
|
||||
failures = []
|
||||
prepared = run_example("prepare", environment, failures)
|
||||
result = run_example("run", environment, failures)
|
||||
observed = run_example("observe", environment, failures)
|
||||
|
||||
if prepared is not None:
|
||||
expect_equal("prepare", prepared, "prompt_id", "example.prepare", failures)
|
||||
expect_equal(
|
||||
"prepare", prepared, "selected_profile", "offline-example", failures
|
||||
)
|
||||
expect_equal("prepare", prepared, "model", "offline-model", failures)
|
||||
expect_positive("prepare", prepared, "message_count", failures)
|
||||
|
||||
if result is not None:
|
||||
expect_equal(
|
||||
"run", result, "output", "Ada finished the migration review.", failures
|
||||
)
|
||||
expect_equal("run", result, "is_valid", True, failures)
|
||||
expect_equal("run", result, "validation_status", "passed", failures)
|
||||
expect_equal("run", result, "model", "offline-model", failures)
|
||||
expect_equal("run", result, "total_tokens", 18, failures)
|
||||
|
||||
if observed is not None:
|
||||
expect_equal("observe", observed, "proposal_attempts", 2, failures)
|
||||
expect_equal(
|
||||
"observe", observed, "proposal_transient_retries", 1, failures
|
||||
)
|
||||
expect_equal(
|
||||
"observe", observed, "proposal_usage_complete", False, failures
|
||||
)
|
||||
expect_equal("observe", observed, "assessment_attempts", 1, failures)
|
||||
expect_equal(
|
||||
"observe", observed, "assessment_transient_retries", 0, failures
|
||||
)
|
||||
expect_equal(
|
||||
"observe", observed, "assessment_outcome", "accepted", failures
|
||||
)
|
||||
|
||||
if failures:
|
||||
print("Consumer example checks failed:", file=sys.stderr)
|
||||
for failure in failures:
|
||||
print(f"- {failure}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
print("Consumer example checks passed: prepare, run, observe")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,152 +0,0 @@
|
||||
#!/bin/sh
|
||||
# Run local checks without publishing or changing files.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
go test -race -vet=off ./...
|
||||
go vet ./...
|
||||
python3 - <<'PY'
|
||||
import json
|
||||
import subprocess
|
||||
|
||||
def example(name):
|
||||
return json.loads(subprocess.check_output(
|
||||
["go", "run", "./examples/go-library/" + name], text=True))
|
||||
|
||||
prepared = example("prepare")
|
||||
assert prepared["prompt_id"] == "example.prepare", prepared
|
||||
assert prepared["selected_profile"] == "offline-example", prepared
|
||||
assert prepared["model"] == "offline-model", prepared
|
||||
assert prepared["message_count"] > 0, prepared
|
||||
result = example("run")
|
||||
assert result["output"] == "Ada finished the migration review.", result
|
||||
assert result["is_valid"] is True, result
|
||||
assert result["validation_status"] == "passed", result
|
||||
assert result["model"] == "offline-model", result
|
||||
assert result["total_tokens"] == 18, result
|
||||
observed = example("observe")
|
||||
assert observed["proposal_attempts"] == 2, observed
|
||||
assert observed["proposal_transient_retries"] == 1, observed
|
||||
assert observed["proposal_usage_complete"] is False, observed
|
||||
assert observed["assessment_attempts"] == 1, observed
|
||||
assert observed["assessment_transient_retries"] == 0, observed
|
||||
assert observed["assessment_outcome"] == "accepted", observed
|
||||
PY
|
||||
|
||||
unformatted=$(
|
||||
git ls-files '*.go' |
|
||||
while IFS= read -r go_file
|
||||
do
|
||||
gofmt -l "$go_file"
|
||||
done
|
||||
)
|
||||
test -z "$unformatted"
|
||||
|
||||
python3 - <<'PY'
|
||||
from pathlib import Path
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from urllib.parse import unquote
|
||||
|
||||
root = Path.cwd().resolve()
|
||||
markdown_files = [
|
||||
root / name
|
||||
for name in subprocess.check_output(
|
||||
["git", "ls-files", "*.md"], text=True
|
||||
).splitlines()
|
||||
]
|
||||
link_pattern = re.compile(r"!?\[[^]]*\]\(([^)]+)\)")
|
||||
heading_pattern = re.compile(r"^#{1,6}\s+(.+?)\s*#*\s*$")
|
||||
scheme_pattern = re.compile(r"^[a-z][a-z0-9+.-]*:", re.IGNORECASE)
|
||||
|
||||
|
||||
def markdown_lines(path):
|
||||
in_fence = False
|
||||
fence = ""
|
||||
for line in path.read_text(encoding="utf-8").splitlines():
|
||||
stripped = line.lstrip()
|
||||
marker = stripped[:3]
|
||||
if marker in {"```", "~~~"}:
|
||||
if not in_fence:
|
||||
in_fence = True
|
||||
fence = marker
|
||||
elif marker == fence:
|
||||
in_fence = False
|
||||
fence = ""
|
||||
continue
|
||||
if not in_fence:
|
||||
yield line
|
||||
|
||||
|
||||
anchor_cache = {}
|
||||
|
||||
|
||||
def anchors(path):
|
||||
if path in anchor_cache:
|
||||
return anchor_cache[path]
|
||||
found = set()
|
||||
counts = {}
|
||||
for line in markdown_lines(path):
|
||||
match = heading_pattern.match(line)
|
||||
if not match:
|
||||
continue
|
||||
heading = re.sub(r"<[^>]+>", "", match.group(1)).replace("`", "")
|
||||
base = re.sub(r"[^\w\- ]", "", heading.lower()).replace(" ", "-")
|
||||
count = counts.get(base, 0)
|
||||
counts[base] = count + 1
|
||||
found.add(base if count == 0 else f"{base}-{count}")
|
||||
anchor_cache[path] = found
|
||||
return found
|
||||
|
||||
|
||||
failures = []
|
||||
for source in markdown_files:
|
||||
text = "\n".join(markdown_lines(source))
|
||||
for match in link_pattern.finditer(text):
|
||||
target = match.group(1).strip()
|
||||
if target.startswith("<") and target.endswith(">"):
|
||||
target = target[1:-1]
|
||||
if scheme_pattern.match(target) or target.startswith("//"):
|
||||
continue
|
||||
path_text, separator, fragment = target.partition("#")
|
||||
destination = source if not path_text else source.parent / unquote(path_text)
|
||||
try:
|
||||
destination = destination.resolve()
|
||||
destination.relative_to(root)
|
||||
except ValueError:
|
||||
failures.append(f"{source.relative_to(root)}: escapes repository: {target}")
|
||||
continue
|
||||
if not destination.exists():
|
||||
failures.append(f"{source.relative_to(root)}: missing target: {target}")
|
||||
continue
|
||||
if separator and destination.suffix.lower() == ".md":
|
||||
fragment = unquote(fragment).lower()
|
||||
if fragment not in anchors(destination):
|
||||
failures.append(f"{source.relative_to(root)}: missing anchor: {target}")
|
||||
|
||||
if failures:
|
||||
print("\n".join(failures), file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
PY
|
||||
|
||||
case "$(go env GOWORK)" in
|
||||
''|off) ;;
|
||||
*) printf '%s\n' 'an active Go workspace is not allowed' >&2; exit 1 ;;
|
||||
esac
|
||||
test -z "$(git ls-files go.work go.work.sum)"
|
||||
test ! -e vendor
|
||||
if grep -Eq '^[[:space:]]*replace([[:space:]]|\()' go.mod
|
||||
then
|
||||
printf '%s\n' 'go.mod contains a replacement' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
git diff --check
|
||||
git diff --cached --check
|
||||
credential_pattern='-----BEGIN ([A-Z0-9]+ )?PRIV''ATE KEY-----|AKI''A[0-9A-Z]{16}|gh[pousr]_[A-Za-z0-9]{36,}|sk-[A-Za-z0-9]{32,}'
|
||||
if git grep -nEI -e "$credential_pattern" -- .
|
||||
then
|
||||
printf '%s\n' 'possible credential found' >&2
|
||||
exit 1
|
||||
fi
|
||||
@@ -1,57 +1,5 @@
|
||||
#!/bin/sh
|
||||
# Validate one committed candidate and publish only its selected annotated tag.
|
||||
set -eu
|
||||
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
release_version=${1:?usage: scripts/release.sh vMAJOR.MINOR.PATCH}
|
||||
printf '%s\n' "$release_version" |
|
||||
grep -Eq '^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$'
|
||||
release_commit=$(git rev-parse --verify 'HEAD^{commit}')
|
||||
|
||||
check_candidate() {
|
||||
test "$(git branch --show-current)" = main
|
||||
test -z "$(git status --porcelain)"
|
||||
test "$(git rev-parse HEAD)" = "$release_commit"
|
||||
git fetch origin main
|
||||
test "$(git rev-parse refs/remotes/origin/main)" = "$release_commit"
|
||||
if git show-ref --verify --quiet "refs/tags/$release_version"; then
|
||||
printf '%s\n' "local tag already exists: $release_version" >&2
|
||||
exit 1
|
||||
fi
|
||||
remote_refs=$(git ls-remote --tags origin "refs/tags/$release_version" "refs/tags/$release_version^{}")
|
||||
test -z "$remote_refs"
|
||||
}
|
||||
|
||||
check_candidate
|
||||
python3 - "$release_version" <<'PY'
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
version = sys.argv[1]
|
||||
note = Path("docs/releases") / (version + ".md")
|
||||
lines = note.read_text(encoding="utf-8").splitlines()
|
||||
assert lines and lines[0] == "# Promptkit " + version, f"wrong release title: {note}"
|
||||
headings = set()
|
||||
fence = None
|
||||
for line in lines:
|
||||
marker = line.lstrip()[:3]
|
||||
if marker in {"```", "~~~"}:
|
||||
if fence is None:
|
||||
fence = marker
|
||||
elif fence == marker:
|
||||
fence = None
|
||||
elif fence is None:
|
||||
headings.add(line)
|
||||
for section in ("Summary", "Compatibility", "Upgrade", "Changes"):
|
||||
assert "## " + section in headings, f"missing {section}: {note}"
|
||||
PY
|
||||
|
||||
./scripts/release-check.sh
|
||||
check_candidate
|
||||
release_note=$(git show "$release_commit:docs/releases/$release_version.md")
|
||||
printf '%s\n\nValidated commit: %s\n' "$release_note" "$release_commit" |
|
||||
git tag --annotate --cleanup=verbatim "$release_version" --file - "$release_commit"
|
||||
test "$(git cat-file -t "refs/tags/$release_version")" = tag
|
||||
test "$(git rev-parse "refs/tags/$release_version^{commit}")" = "$release_commit"
|
||||
git push origin "refs/tags/$release_version:refs/tags/$release_version"
|
||||
printf '%s\n' "Released $release_version. No CI or module-availability wait is required."
|
||||
exec ./scripts/releasekit.sh publish "$@"
|
||||
|
||||
18
scripts/releasekit.sh
Executable file
18
scripts/releasekit.sh
Executable file
@@ -0,0 +1,18 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
export GOTOOLCHAIN=local
|
||||
export GOWORK=off
|
||||
export GOFLAGS=-mod=readonly
|
||||
|
||||
GOOS=$(go env GOHOSTOS)
|
||||
GOARCH=$(go env GOHOSTARCH)
|
||||
export GOOS GOARCH
|
||||
|
||||
mkdir -p .releasekit/bin
|
||||
go build -mod=readonly -o .releasekit/bin/releasekit \
|
||||
"gitea.maximumdirect.net/eric/ReleaseKit/cmd/releasekit"
|
||||
|
||||
exec ./.releasekit/bin/releasekit "$@"
|
||||
476
scripts/test-release.py
Normal file
476
scripts/test-release.py
Normal file
@@ -0,0 +1,476 @@
|
||||
"""Exercise ReleaseKit publication wiring in disposable repositories."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import textwrap
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
REPOSITORY_ROOT = Path(__file__).resolve().parent.parent
|
||||
RELEASE_VERSION = "v0.0.1"
|
||||
FAILED_RELEASE_VERSION = "v0.0.2"
|
||||
RELEASEKIT_MODULE = "gitea.maximumdirect.net/eric/ReleaseKit"
|
||||
|
||||
|
||||
def execute(command, cwd, environment):
|
||||
try:
|
||||
return subprocess.run(
|
||||
command,
|
||||
cwd=cwd,
|
||||
env=environment,
|
||||
text=True,
|
||||
capture_output=True,
|
||||
check=False,
|
||||
)
|
||||
except OSError as error:
|
||||
raise RuntimeError(f"could not run {command[0]}: {error}") from error
|
||||
|
||||
|
||||
def run(command, cwd, environment):
|
||||
result = execute(command, cwd, environment)
|
||||
if result.returncode != 0:
|
||||
details = [
|
||||
f"command failed with status {result.returncode}: {' '.join(command)}"
|
||||
]
|
||||
if result.stdout:
|
||||
details.append(f"stdout:\n{result.stdout.rstrip()}")
|
||||
if result.stderr:
|
||||
details.append(f"stderr:\n{result.stderr.rstrip()}")
|
||||
raise RuntimeError("\n".join(details))
|
||||
return result.stdout
|
||||
|
||||
|
||||
def write_file(path, contents):
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(textwrap.dedent(contents).lstrip("\n"), encoding="utf-8")
|
||||
|
||||
|
||||
def repo_git(repository, environment, *arguments):
|
||||
return run(
|
||||
["git", "-C", str(repository), *arguments],
|
||||
cwd=repository.parent,
|
||||
environment=environment,
|
||||
)
|
||||
|
||||
|
||||
def remote_git(remote, temporary_root, environment, *arguments):
|
||||
return run(
|
||||
["git", "--git-dir", str(remote), *arguments],
|
||||
cwd=temporary_root,
|
||||
environment=environment,
|
||||
)
|
||||
|
||||
|
||||
def fixture_sources(repository, environment):
|
||||
write_file(
|
||||
repository / "fixture.go",
|
||||
"""
|
||||
package fixture
|
||||
|
||||
func candidateValue() string { return "ready" }
|
||||
""",
|
||||
)
|
||||
write_file(
|
||||
repository / "fixture_test.go",
|
||||
f"""
|
||||
package fixture
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"runtime"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestCandidateSuite(t *testing.T) {{
|
||||
if candidateValue() != "ready" {{
|
||||
t.Fatal("candidate package is not ready")
|
||||
}}
|
||||
marker := os.Getenv("FIXTURE_SUITE_MARKER")
|
||||
if marker == "" {{
|
||||
t.Fatal("FIXTURE_SUITE_MARKER is not set")
|
||||
}}
|
||||
record := struct {{
|
||||
GoVersion string `json:"go_version"`
|
||||
Race bool `json:"race"`
|
||||
}}{{GoVersion: runtime.Version(), Race: raceDetectorEnabled}}
|
||||
data, err := json.Marshal(record)
|
||||
if err != nil {{
|
||||
t.Fatal(err)
|
||||
}}
|
||||
file, err := os.OpenFile(marker, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600)
|
||||
if err != nil {{
|
||||
t.Fatal(err)
|
||||
}}
|
||||
if _, err := file.Write(append(data, '\\n')); err != nil {{
|
||||
file.Close()
|
||||
t.Fatal(err)
|
||||
}}
|
||||
if err := file.Close(); err != nil {{
|
||||
t.Fatal(err)
|
||||
}}
|
||||
}}
|
||||
""",
|
||||
)
|
||||
write_file(
|
||||
repository / "race_enabled_test.go",
|
||||
"""
|
||||
//go:build race
|
||||
|
||||
package fixture
|
||||
|
||||
const raceDetectorEnabled = true
|
||||
""",
|
||||
)
|
||||
write_file(
|
||||
repository / "race_disabled_test.go",
|
||||
"""
|
||||
//go:build !race
|
||||
|
||||
package fixture
|
||||
|
||||
const raceDetectorEnabled = false
|
||||
""",
|
||||
)
|
||||
go_files = [
|
||||
str(repository / filename)
|
||||
for filename in (
|
||||
"fixture.go",
|
||||
"fixture_test.go",
|
||||
"race_enabled_test.go",
|
||||
"race_disabled_test.go",
|
||||
)
|
||||
]
|
||||
run(["gofmt", "-w", *go_files], cwd=repository, environment=environment)
|
||||
|
||||
|
||||
def prepare_repository(repository, remote, environment, module_bytes, version):
|
||||
run(
|
||||
["git", "init", "--bare", "--initial-branch=main", str(remote)],
|
||||
cwd=repository.parent,
|
||||
environment=environment,
|
||||
)
|
||||
run(
|
||||
["git", "init", "--initial-branch=main", str(repository)],
|
||||
cwd=repository.parent,
|
||||
environment=environment,
|
||||
)
|
||||
repo_git(repository, environment, "config", "user.name", "ReleaseKit Fixture")
|
||||
repo_git(repository, environment, "config", "user.email", "fixture@example.invalid")
|
||||
repo_git(repository, environment, "remote", "add", "origin", str(remote))
|
||||
|
||||
for name, contents in module_bytes.items():
|
||||
(repository / name).write_bytes(contents)
|
||||
|
||||
write_file(
|
||||
repository / ".gitignore",
|
||||
"""
|
||||
.releasekit/
|
||||
""",
|
||||
)
|
||||
write_file(
|
||||
repository / "releasekit.json",
|
||||
(REPOSITORY_ROOT / "releasekit.json").read_text(encoding="utf-8"),
|
||||
)
|
||||
|
||||
scripts = repository / "scripts"
|
||||
scripts.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(REPOSITORY_ROOT / "scripts/release.sh", scripts / "release.sh")
|
||||
shutil.copy2(REPOSITORY_ROOT / "scripts/releasekit.sh", scripts / "releasekit.sh")
|
||||
write_file(
|
||||
scripts / "check-examples.py",
|
||||
"""
|
||||
import os
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
marker = Path(os.environ["FIXTURE_EXAMPLE_MARKER"])
|
||||
with marker.open("a", encoding="utf-8") as stream:
|
||||
stream.write("called\\n")
|
||||
if os.environ.get("FIXTURE_EXAMPLE_FAILURE") == "1":
|
||||
print("fixture example check rejected candidate", file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
""",
|
||||
)
|
||||
write_file(
|
||||
repository / f"docs/releases/{version}.md",
|
||||
f"""
|
||||
# Promptkit {version}
|
||||
|
||||
Disposable publication fixture.
|
||||
""",
|
||||
)
|
||||
fixture_sources(repository, environment)
|
||||
|
||||
repo_git(repository, environment, "add", "-A")
|
||||
run(
|
||||
["git", "-C", str(repository), "commit", "-m", "Add disposable release fixture"],
|
||||
cwd=repository.parent,
|
||||
environment=environment,
|
||||
)
|
||||
repo_git(repository, environment, "push", "origin", "main")
|
||||
|
||||
|
||||
def verify_publication(repository, remote, temporary_root, environment, module_bytes, selected_go):
|
||||
selected_releasekit = run(
|
||||
["go", "list", "-mod=readonly", "-m", "-f", "{{.Version}}", RELEASEKIT_MODULE],
|
||||
cwd=repository,
|
||||
environment=environment,
|
||||
).strip()
|
||||
publication = repository / "scripts/release.sh"
|
||||
run(
|
||||
[str(publication), RELEASE_VERSION],
|
||||
cwd=temporary_root,
|
||||
environment=environment,
|
||||
)
|
||||
|
||||
suite_lines = Path(environment["FIXTURE_SUITE_MARKER"]).read_text(
|
||||
encoding="utf-8"
|
||||
).splitlines()
|
||||
if len(suite_lines) != 1:
|
||||
raise RuntimeError(f"expected one suite marker, found {len(suite_lines)}")
|
||||
suite_record = json.loads(suite_lines[0])
|
||||
if suite_record != {"go_version": selected_go, "race": True}:
|
||||
raise RuntimeError(f"unexpected suite marker: {suite_record!r}")
|
||||
|
||||
example_lines = Path(environment["FIXTURE_EXAMPLE_MARKER"]).read_text(
|
||||
encoding="utf-8"
|
||||
).splitlines()
|
||||
if example_lines != ["called"]:
|
||||
raise RuntimeError(f"expected one additional-check marker, got {example_lines!r}")
|
||||
|
||||
binary = repository / ".releasekit/bin/releasekit"
|
||||
if not binary.is_file():
|
||||
raise RuntimeError("ReleaseKit bootstrap did not leave its ignored executable")
|
||||
build_info = run(
|
||||
["go", "version", "-m", str(binary)],
|
||||
cwd=temporary_root,
|
||||
environment=environment,
|
||||
)
|
||||
first_line = build_info.splitlines()[0]
|
||||
if selected_go not in first_line:
|
||||
raise RuntimeError(f"tool was built with {first_line!r}, expected {selected_go}")
|
||||
expected_module = f"\tmod\t{RELEASEKIT_MODULE}\t{selected_releasekit}\t"
|
||||
if expected_module not in build_info:
|
||||
raise RuntimeError(
|
||||
f"bootstrap executable does not record selected ReleaseKit {selected_releasekit}"
|
||||
)
|
||||
|
||||
candidate = repo_git(repository, environment, "rev-parse", "HEAD").strip()
|
||||
if repo_git(repository, environment, "rev-parse", "main").strip() != candidate:
|
||||
raise RuntimeError("fixture main moved away from the published candidate")
|
||||
if remote_git(remote, temporary_root, environment, "rev-parse", "refs/heads/main").strip() != candidate:
|
||||
raise RuntimeError("local origin main does not identify the fixture candidate")
|
||||
|
||||
tag_ref = f"refs/tags/{RELEASE_VERSION}"
|
||||
if repo_git(repository, environment, "cat-file", "-t", tag_ref).strip() != "tag":
|
||||
raise RuntimeError("local release ref is not an annotated tag")
|
||||
local_tag = repo_git(repository, environment, "rev-parse", tag_ref).strip()
|
||||
local_tag_commit = repo_git(
|
||||
repository, environment, "rev-parse", f"{tag_ref}^{{}}"
|
||||
).strip()
|
||||
if local_tag_commit != candidate:
|
||||
raise RuntimeError("local annotated tag does not identify the fixture candidate")
|
||||
|
||||
remote_tag = remote_git(remote, temporary_root, environment, "rev-parse", tag_ref).strip()
|
||||
remote_tag_commit = remote_git(
|
||||
remote, temporary_root, environment, "rev-parse", f"{tag_ref}^{{}}"
|
||||
).strip()
|
||||
if remote_tag != local_tag or remote_tag_commit != candidate:
|
||||
raise RuntimeError("remote annotated tag does not identify the fixture candidate")
|
||||
if remote_git(
|
||||
remote, temporary_root, environment, "cat-file", "-t", tag_ref
|
||||
).strip() != "tag":
|
||||
raise RuntimeError("remote release ref is not an annotated tag")
|
||||
|
||||
tag_object = repo_git(repository, environment, "cat-file", "tag", tag_ref)
|
||||
separator = tag_object.find("\n\n")
|
||||
if separator < 0:
|
||||
raise RuntimeError("annotated tag has no message")
|
||||
tag_message = tag_object[separator + 2 :]
|
||||
note = (repository / f"docs/releases/{RELEASE_VERSION}.md").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
if tag_message != note:
|
||||
raise RuntimeError("annotated tag message does not match the committed release note")
|
||||
|
||||
for name, contents in module_bytes.items():
|
||||
if (repository / name).read_bytes() != contents:
|
||||
raise RuntimeError(f"publication changed {name}")
|
||||
|
||||
status = repo_git(
|
||||
repository, environment, "status", "--porcelain", "--untracked-files=all"
|
||||
)
|
||||
if status:
|
||||
raise RuntimeError(f"fixture checkout is not clean after publication:\n{status}")
|
||||
repo_git(repository, environment, "check-ignore", "-q", ".releasekit/bin/releasekit")
|
||||
|
||||
|
||||
def verify_validation_failure(
|
||||
repository, remote, temporary_root, environment, module_bytes, selected_go
|
||||
):
|
||||
scenario_environment = environment.copy()
|
||||
suite_marker = temporary_root / "failure-suite.jsonl"
|
||||
example_marker = temporary_root / "failure-example.log"
|
||||
scenario_environment.update(
|
||||
{
|
||||
"FIXTURE_SUITE_MARKER": str(suite_marker),
|
||||
"FIXTURE_EXAMPLE_MARKER": str(example_marker),
|
||||
"FIXTURE_EXAMPLE_FAILURE": "1",
|
||||
}
|
||||
)
|
||||
prepare_repository(
|
||||
repository,
|
||||
remote,
|
||||
scenario_environment,
|
||||
module_bytes,
|
||||
FAILED_RELEASE_VERSION,
|
||||
)
|
||||
|
||||
result = execute(
|
||||
[str(repository / "scripts/release.sh"), FAILED_RELEASE_VERSION],
|
||||
cwd=temporary_root,
|
||||
environment=scenario_environment,
|
||||
)
|
||||
if result.returncode != 1:
|
||||
raise RuntimeError(
|
||||
"validation failure returned status "
|
||||
f"{result.returncode}, want 1; stdout:\n{result.stdout}\nstderr:\n{result.stderr}"
|
||||
)
|
||||
if "fixture example check rejected candidate" not in result.stderr:
|
||||
raise RuntimeError(f"failure diagnostic omitted useful context:\n{result.stderr}")
|
||||
|
||||
suite_lines = suite_marker.read_text(encoding="utf-8").splitlines()
|
||||
if len(suite_lines) != 1:
|
||||
raise RuntimeError(f"expected one failure-case suite marker, found {len(suite_lines)}")
|
||||
suite_record = json.loads(suite_lines[0])
|
||||
if suite_record != {"go_version": selected_go, "race": True}:
|
||||
raise RuntimeError(f"unexpected failure-case suite marker: {suite_record!r}")
|
||||
if example_marker.read_text(encoding="utf-8").splitlines() != ["called"]:
|
||||
raise RuntimeError("expected one additional-check invocation before failure")
|
||||
|
||||
tag_ref = f"refs/tags/{FAILED_RELEASE_VERSION}"
|
||||
for description, command in (
|
||||
(
|
||||
"local",
|
||||
["git", "-C", str(repository), "show-ref", "--verify", "--quiet", tag_ref],
|
||||
),
|
||||
(
|
||||
"remote",
|
||||
["git", "--git-dir", str(remote), "show-ref", "--verify", "--quiet", tag_ref],
|
||||
),
|
||||
):
|
||||
result = execute(command, cwd=temporary_root, environment=scenario_environment)
|
||||
if result.returncode == 0:
|
||||
raise RuntimeError(f"validation failure created the {description} tag {tag_ref}")
|
||||
if result.returncode != 1:
|
||||
raise RuntimeError(
|
||||
f"could not verify {description} tag absence: "
|
||||
f"status {result.returncode}: {result.stderr.strip()}"
|
||||
)
|
||||
|
||||
for name, contents in module_bytes.items():
|
||||
if (repository / name).read_bytes() != contents:
|
||||
raise RuntimeError(f"failed publication changed {name}")
|
||||
status = repo_git(
|
||||
repository,
|
||||
scenario_environment,
|
||||
"status",
|
||||
"--porcelain",
|
||||
"--untracked-files=all",
|
||||
)
|
||||
if status:
|
||||
raise RuntimeError(f"failure fixture checkout is not clean:\n{status}")
|
||||
|
||||
|
||||
def verify_usage_classification(binary, temporary_root, environment):
|
||||
result = execute(
|
||||
[str(binary), "unknown"], cwd=temporary_root, environment=environment
|
||||
)
|
||||
if result.returncode != 2:
|
||||
raise RuntimeError(
|
||||
f"unsupported command returned status {result.returncode}, want 2; "
|
||||
f"stdout:\n{result.stdout}\nstderr:\n{result.stderr}"
|
||||
)
|
||||
if result.stdout:
|
||||
raise RuntimeError(f"unsupported command wrote to stdout: {result.stdout!r}")
|
||||
if 'unknown command "unknown"' not in result.stderr or "usage: releasekit" not in result.stderr:
|
||||
raise RuntimeError(f"unsupported command diagnostic was not useful:\n{result.stderr}")
|
||||
|
||||
|
||||
def main():
|
||||
environment = os.environ.copy()
|
||||
environment.update(
|
||||
{
|
||||
"GOTOOLCHAIN": "local",
|
||||
"GOWORK": "off",
|
||||
"GOPROXY": "off",
|
||||
"GONOPROXY": "none",
|
||||
"GOSUMDB": "off",
|
||||
}
|
||||
)
|
||||
selected_go = run(
|
||||
["go", "env", "GOVERSION"],
|
||||
cwd=REPOSITORY_ROOT,
|
||||
environment=environment,
|
||||
).strip()
|
||||
if not selected_go:
|
||||
print("go env GOVERSION returned no version", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
ignored_rule = ".releasekit/"
|
||||
ignore_lines = (REPOSITORY_ROOT / ".gitignore").read_text(
|
||||
encoding="utf-8"
|
||||
).splitlines()
|
||||
if ignored_rule not in ignore_lines:
|
||||
print(".gitignore does not contain the ReleaseKit output rule", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
module_bytes = {
|
||||
name: (REPOSITORY_ROOT / name).read_bytes() for name in ("go.mod", "go.sum")
|
||||
}
|
||||
with tempfile.TemporaryDirectory(prefix="promptkit-release-fixture-") as directory:
|
||||
temporary_root = Path(directory)
|
||||
repository = temporary_root / "checkout"
|
||||
remote = temporary_root / "origin.git"
|
||||
suite_marker = temporary_root / "suite.jsonl"
|
||||
example_marker = temporary_root / "example.log"
|
||||
environment["FIXTURE_SUITE_MARKER"] = str(suite_marker)
|
||||
environment["FIXTURE_EXAMPLE_MARKER"] = str(example_marker)
|
||||
try:
|
||||
prepare_repository(
|
||||
repository, remote, environment, module_bytes, RELEASE_VERSION
|
||||
)
|
||||
verify_publication(
|
||||
repository,
|
||||
remote,
|
||||
temporary_root,
|
||||
environment,
|
||||
module_bytes,
|
||||
selected_go,
|
||||
)
|
||||
verify_usage_classification(
|
||||
repository / ".releasekit/bin/releasekit", temporary_root, environment
|
||||
)
|
||||
verify_validation_failure(
|
||||
temporary_root / "failure-checkout",
|
||||
temporary_root / "failure-origin.git",
|
||||
temporary_root,
|
||||
environment,
|
||||
module_bytes,
|
||||
selected_go,
|
||||
)
|
||||
except (RuntimeError, OSError, json.JSONDecodeError) as error:
|
||||
print(error, file=sys.stderr)
|
||||
return 1
|
||||
|
||||
print("ReleaseKit publication success and failure fixtures passed")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user