Compare commits
84 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 143c6e9b3a | |||
| 48e4784fcd | |||
| 78af5069c1 | |||
| 18963266d7 | |||
| d039a019cb | |||
| d1c0281053 | |||
| 870e4ca4e6 | |||
| fd82524977 | |||
| 3f76c75919 | |||
| fe764f72d8 | |||
| 26ab566f38 | |||
| 18ca21fa45 | |||
| e32fdc22f4 | |||
| b02b802427 | |||
| 301192b974 | |||
| f2aeb5eb30 | |||
| a27ee8f7da | |||
| 4d1895c223 | |||
| d7fe70520a | |||
| 7325a34c07 | |||
| 03c44923c8 | |||
| dc0adf5751 | |||
| eae21797f7 | |||
| ea6c7d5f0f | |||
| 389ef26db3 | |||
| 8a5c0de162 | |||
| 80aecb646d | |||
| b7b4000aab | |||
| 11ecb923ba | |||
| f46d5a4591 | |||
| 743e78d514 | |||
| 3743221aa5 | |||
| a8863d49d0 | |||
| c5f733af45 | |||
| c09d2c6e20 | |||
| 6c5dec761c | |||
| 74b91fae99 | |||
| b7ff37118c | |||
| ef012ea1e5 | |||
| 04d5d695e0 | |||
| 78bff91395 | |||
| c63ca5352f | |||
| 3f157150e4 | |||
| f1e1a45a05 | |||
| 996295d5c1 | |||
| 58a5bbe790 | |||
| dd89be3b03 | |||
| ed7c27324d | |||
| 05fe13d80b | |||
| 0eb009ad5a | |||
| 3fde1f6cbb | |||
| 481e2d494c | |||
| eb830bb613 | |||
| dd6d4591ae | |||
| de6443f3ae | |||
| 5a63b649f9 | |||
| c7e22c0dbe | |||
| e69a847cf2 | |||
| 21815ad7c0 | |||
| a051edd82d | |||
| ee0ad99e17 | |||
| bc5a2da0f2 | |||
| 089e824a92 | |||
| 93e8874813 | |||
| cc0b6183bf | |||
| bf472dad8a | |||
| a6e0ffea8f | |||
| e459c7d54b | |||
| 71f6683fda | |||
| 7827ca043b | |||
| b595dca8c7 | |||
| d5f34642d7 | |||
| b1866946d1 | |||
| 05082c6fdf | |||
| 51b072965c | |||
| d5940856be | |||
| 0d163f8165 | |||
| 1889915a81 | |||
| 3ca01ae0b0 | |||
| 6e283d16c3 | |||
| 41b2021d8c | |||
| 3add7024db | |||
| a148a2b1f4 | |||
| 7310ef34a4 |
4
.gitignore
vendored
4
.gitignore
vendored
@@ -22,6 +22,9 @@
|
||||
go.work
|
||||
go.work.sum
|
||||
|
||||
# Local ReleaseKit executable
|
||||
.releasekit/
|
||||
|
||||
# env file
|
||||
.env
|
||||
|
||||
@@ -67,4 +70,3 @@ Icon
|
||||
.AppleDesktop
|
||||
Network Trash Folder
|
||||
Temporary Items
|
||||
.apdisk
|
||||
|
||||
237
assembled_workflow_benchmark_test.go
Normal file
237
assembled_workflow_benchmark_test.go
Normal file
@@ -0,0 +1,237 @@
|
||||
package promptkit_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"testing/fstest"
|
||||
|
||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
var (
|
||||
assembledPreparedSink *promptkit.PreparedRun
|
||||
assembledResultSink *promptkit.RunResult
|
||||
assembledDetailsSink promptkit.PreparedRun
|
||||
assembledHandleSink *promptkit.PreparedExecution
|
||||
)
|
||||
|
||||
type assembledWorkflowClient struct{ content string }
|
||||
|
||||
func (client assembledWorkflowClient) Generate(
|
||||
context.Context,
|
||||
promptkit.GenerateRequest,
|
||||
) (*promptkit.GenerateResponse, error) {
|
||||
return &promptkit.GenerateResponse{Content: client.content}, nil
|
||||
}
|
||||
|
||||
type assembledWorkflowFixture struct {
|
||||
engine *promptkit.Engine
|
||||
request promptkit.RunRequest
|
||||
expectedMessage string
|
||||
response string
|
||||
}
|
||||
|
||||
func assembledWorkflowSchema(b *testing.B) []byte {
|
||||
b.Helper()
|
||||
properties := make(map[string]any, 256)
|
||||
for index := range 256 {
|
||||
properties[fmt.Sprintf("property-%03d", index)] = map[string]any{"type": "string"}
|
||||
}
|
||||
data, err := json.Marshal(map[string]any{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"type": "object",
|
||||
"properties": properties,
|
||||
"additionalProperties": false,
|
||||
})
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
return data
|
||||
}
|
||||
|
||||
func assembledWorkflowExtras(entries int) map[string]any {
|
||||
providers := make(map[string]any, entries)
|
||||
for index := range entries {
|
||||
providers[fmt.Sprintf("provider-%03d", index)] = map[string]any{
|
||||
"enabled": true,
|
||||
"labels": []any{fmt.Sprintf("tier-%d", index%4), fmt.Sprintf("region-%d", index%3)},
|
||||
"limits": map[string]any{"burst": index + 1, "period": "minute"},
|
||||
"routes": []any{map[string]any{"weight": index + 1, "fallback": true}},
|
||||
}
|
||||
}
|
||||
return map[string]any{"provider_options": providers}
|
||||
}
|
||||
|
||||
func newAssembledWorkflowFixture(b *testing.B, name string) assembledWorkflowFixture {
|
||||
b.Helper()
|
||||
message := "Hello {{.name}}"
|
||||
response := "ok"
|
||||
output := map[string]any{"format": "text", "validation_mode": "none"}
|
||||
profile := map[string]any{"endpoint": "http://example.test/v1", "model": "benchmark-model"}
|
||||
options := []promptkit.Option{promptkit.WithLLMClient(assembledWorkflowClient{content: response})}
|
||||
var expectedMessage string
|
||||
|
||||
switch name {
|
||||
case "small":
|
||||
expectedMessage = "Hello Ada"
|
||||
case "large_message":
|
||||
prefix := strings.Repeat("x", 256*1024)
|
||||
message = prefix + "{{.name}}"
|
||||
expectedMessage = prefix + "Ada"
|
||||
case "mutable_payload":
|
||||
message = `{"subject":"{{.name}}"}`
|
||||
expectedMessage = `{"subject":"Ada"}`
|
||||
response = "{}"
|
||||
profile["extra_params"] = assembledWorkflowExtras(64)
|
||||
output = map[string]any{
|
||||
"format": "json", "validation_mode": "json_schema", "schema_path": "workflow-schema.json",
|
||||
}
|
||||
options[0] = promptkit.WithLLMClient(assembledWorkflowClient{content: response})
|
||||
options = append(options, promptkit.WithSchemaFS(fstest.MapFS{
|
||||
"workflow-schema.json": &fstest.MapFile{Data: assembledWorkflowSchema(b)},
|
||||
}, "."))
|
||||
default:
|
||||
b.Fatalf("unknown assembled workflow fixture %q", name)
|
||||
}
|
||||
|
||||
config := map[string]any{
|
||||
"profiles": map[string]any{"profile": profile},
|
||||
"prompts": map[string]any{"workflow": map[string]any{"v1": map[string]any{
|
||||
"default_profile": "profile",
|
||||
"messages": []any{map[string]any{"role": "user", "content": message}},
|
||||
"output": output,
|
||||
}}},
|
||||
}
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
|
||||
Source: configkit.Mapping("assembled workflow benchmark", config),
|
||||
}}}, options...)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
request := promptkit.RunRequest{
|
||||
PromptID: "workflow", PromptVersion: "v1", Vars: map[string]string{"name": "Ada"},
|
||||
}
|
||||
return assembledWorkflowFixture{
|
||||
engine: engine, request: request, expectedMessage: expectedMessage, response: response,
|
||||
}
|
||||
}
|
||||
|
||||
func verifyAssembledWorkflowFixture(b *testing.B, fixture assembledWorkflowFixture) {
|
||||
b.Helper()
|
||||
ctx := context.Background()
|
||||
prepared, err := fixture.engine.Prepare(ctx, fixture.request)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
if prepared.PromptID != "workflow" || prepared.PromptVersion != "v1" || len(prepared.Messages) != 1 ||
|
||||
prepared.Messages[0].Content != fixture.expectedMessage {
|
||||
b.Fatalf("prepared fixture output is invalid: %+v", prepared)
|
||||
}
|
||||
handle, err := fixture.engine.PrepareExecution(ctx, fixture.request)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
details := handle.Details()
|
||||
if details.PromptID != "workflow" || len(details.Messages) != 1 || details.Messages[0].Content != fixture.expectedMessage {
|
||||
handle.Discard()
|
||||
b.Fatalf("prepared execution details are invalid: %+v", details)
|
||||
}
|
||||
handle.Discard()
|
||||
result, err := fixture.engine.Run(ctx, fixture.request)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
if result.RawOutput != fixture.response {
|
||||
b.Fatalf("run output=%q, want %q", result.RawOutput, fixture.response)
|
||||
}
|
||||
if fixture.response == "{}" && result.Validation.Status != promptkit.ValidationPassed {
|
||||
b.Fatalf("structured response validation=%+v, want passed", result.Validation)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkEnginePreparation(b *testing.B) {
|
||||
for _, name := range []string{"small", "large_message", "mutable_payload"} {
|
||||
b.Run(name, func(b *testing.B) {
|
||||
fixture := newAssembledWorkflowFixture(b, name)
|
||||
verifyAssembledWorkflowFixture(b, fixture)
|
||||
ctx := context.Background()
|
||||
b.Run("prepare", func(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for b.Loop() {
|
||||
prepared, err := fixture.engine.Prepare(ctx, fixture.request)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
assembledPreparedSink = prepared
|
||||
}
|
||||
b.StopTimer()
|
||||
assembledPreparedSink = nil
|
||||
})
|
||||
b.Run("prepared_execution_creation_and_discard", func(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for b.Loop() {
|
||||
handle, err := fixture.engine.PrepareExecution(ctx, fixture.request)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
handle.Discard()
|
||||
assembledHandleSink = handle
|
||||
}
|
||||
b.StopTimer()
|
||||
assembledHandleSink = nil
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkEngineRun(b *testing.B) {
|
||||
for _, name := range []string{"small", "large_message", "mutable_payload"} {
|
||||
b.Run(name, func(b *testing.B) {
|
||||
fixture := newAssembledWorkflowFixture(b, name)
|
||||
verifyAssembledWorkflowFixture(b, fixture)
|
||||
ctx := context.Background()
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for b.Loop() {
|
||||
result, err := fixture.engine.Run(ctx, fixture.request)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
assembledResultSink = result
|
||||
}
|
||||
b.StopTimer()
|
||||
assembledResultSink = nil
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkPreparedExecutionDetails(b *testing.B) {
|
||||
for _, name := range []string{"small", "large_message", "mutable_payload"} {
|
||||
b.Run(name, func(b *testing.B) {
|
||||
fixture := newAssembledWorkflowFixture(b, name)
|
||||
verifyAssembledWorkflowFixture(b, fixture)
|
||||
handle, err := fixture.engine.PrepareExecution(context.Background(), fixture.request)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
defer handle.Discard()
|
||||
initial := handle.Details()
|
||||
if len(initial.Messages) != 1 || initial.Messages[0].Content != fixture.expectedMessage {
|
||||
b.Fatalf("prepared details fixture is invalid: %+v", initial)
|
||||
}
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for b.Loop() {
|
||||
assembledDetailsSink = handle.Details()
|
||||
}
|
||||
b.StopTimer()
|
||||
assembledDetailsSink = promptkit.PreparedRun{}
|
||||
})
|
||||
}
|
||||
}
|
||||
243
configuration_diff_contract_test.go
Normal file
243
configuration_diff_contract_test.go
Normal file
@@ -0,0 +1,243 @@
|
||||
package promptkit_test
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
func TestConfigurationInspectionDiffsCapturedValues(t *testing.T) {
|
||||
left := comparisonInspection(t, "earlier-model")
|
||||
right := comparisonInspection(t, "later-model")
|
||||
|
||||
differences, err := left.Diff(right)
|
||||
if err != nil {
|
||||
t.Fatalf("compare captured configurations: %v", err)
|
||||
}
|
||||
if len(differences) != 1 {
|
||||
t.Fatalf("difference count = %d, want 1: %+v", len(differences), differences)
|
||||
}
|
||||
difference := differences[0]
|
||||
if got, want := difference.Path.String(), "/profiles/offline/model"; got != want {
|
||||
t.Fatalf("difference path = %q, want %q", got, want)
|
||||
}
|
||||
if !strings.Contains(difference.Left.YAML, "earlier-model") || !strings.Contains(difference.Right.YAML, "later-model") {
|
||||
t.Fatalf("difference snapshots = (%q, %q)", difference.Left.YAML, difference.Right.YAML)
|
||||
}
|
||||
|
||||
reversed, err := right.Diff(left)
|
||||
if err != nil {
|
||||
t.Fatalf("compare reversed configurations: %v", err)
|
||||
}
|
||||
if len(reversed) != 1 || reversed[0].Path.String() != difference.Path.String() ||
|
||||
!strings.Contains(reversed[0].Left.YAML, "later-model") ||
|
||||
!strings.Contains(reversed[0].Right.YAML, "earlier-model") {
|
||||
t.Fatalf("reversed difference = %+v", reversed)
|
||||
}
|
||||
|
||||
equal := comparisonInspection(t, "earlier-model")
|
||||
if equal, err := left.Diff(equal); err != nil || len(equal) != 0 {
|
||||
t.Fatalf("compare equal configurations = (%+v, %v), want no differences", equal, err)
|
||||
}
|
||||
if equal, err := left.Diff(left); err != nil || len(equal) != 0 {
|
||||
t.Fatalf("compare view with itself = (%+v, %v), want no differences", equal, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigurationInspectionDiffRejectsZeroViews(t *testing.T) {
|
||||
valid := comparisonInspection(t, "current-model")
|
||||
var zero promptkit.ConfigurationInspection
|
||||
|
||||
for _, comparison := range []struct {
|
||||
name string
|
||||
left, right promptkit.ConfigurationInspection
|
||||
}{
|
||||
{name: "zero left", left: zero, right: valid},
|
||||
{name: "zero right", left: valid, right: zero},
|
||||
{name: "both zero", left: zero, right: zero},
|
||||
} {
|
||||
t.Run(comparison.name, func(t *testing.T) {
|
||||
differences, err := comparison.left.Diff(comparison.right)
|
||||
if !errors.Is(err, promptkit.ErrInvalidConfig) || len(differences) != 0 {
|
||||
t.Fatalf("Diff() = (%+v, %v), want ErrInvalidConfig and no records", differences, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigurationInspectionDiffIgnoresSourceHistory(t *testing.T) {
|
||||
left := comparisonInspectionFrom(t, "earlier source", "same-model", nil)
|
||||
right := comparisonInspectionFrom(t, "later source", "same-model", nil)
|
||||
if !hasSourceLabel(left, "earlier source") || !hasSourceLabel(right, "later source") {
|
||||
t.Fatal("comparison fixtures did not retain their distinct source labels")
|
||||
}
|
||||
|
||||
differences, err := left.Diff(right)
|
||||
if err != nil || len(differences) != 0 {
|
||||
t.Fatalf("compare equal effective values with different sources = (%+v, %v), want no differences", differences, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigurationInspectionDiffReportsPresenceAndSequences(t *testing.T) {
|
||||
left := comparisonInspectionFrom(t, "before", "same-model", map[string]any{
|
||||
"sequence": []any{"earlier", "unchanged"},
|
||||
})
|
||||
right := comparisonInspectionFrom(t, "after", "same-model", map[string]any{
|
||||
"nullable": nil,
|
||||
"sequence": []any{"later", "unchanged"},
|
||||
})
|
||||
|
||||
differences, err := left.Diff(right)
|
||||
if err != nil {
|
||||
t.Fatalf("compare presence and sequence values: %v", err)
|
||||
}
|
||||
if len(differences) != 2 {
|
||||
t.Fatalf("difference count = %d, want 2: %+v", len(differences), differences)
|
||||
}
|
||||
byPath := differencesByPath(differences)
|
||||
nullable, ok := byPath["/backends/local/extra_params/nullable"]
|
||||
if !ok || nullable.Left.Present || !nullable.Right.Present || strings.TrimSpace(nullable.Right.YAML) != "null" {
|
||||
t.Fatalf("absent-to-null difference = (%+v, %t)", nullable, ok)
|
||||
}
|
||||
sequence, ok := byPath["/backends/local/extra_params/sequence"]
|
||||
if !ok || !sequence.Left.Present || !sequence.Right.Present ||
|
||||
!strings.Contains(sequence.Left.YAML, "earlier") || !strings.Contains(sequence.Right.YAML, "later") {
|
||||
t.Fatalf("atomic sequence difference = (%+v, %t)", sequence, ok)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigurationInspectionDiffProtectsSensitiveValuesAndReturnedRecords(t *testing.T) {
|
||||
left := comparisonInspectionFrom(t, "before", "same-model", map[string]any{
|
||||
"token": "left-secret-sentinel",
|
||||
"visible": "before-value",
|
||||
})
|
||||
right := comparisonInspectionFrom(t, "after", "same-model", map[string]any{
|
||||
"token": "right-secret-sentinel",
|
||||
"visible": "after-value",
|
||||
})
|
||||
|
||||
differences, err := left.Diff(right)
|
||||
if err != nil {
|
||||
t.Fatalf("compare sensitive values: %v", err)
|
||||
}
|
||||
if len(differences) != 2 {
|
||||
t.Fatalf("difference count = %d, want 2: %+v", len(differences), differences)
|
||||
}
|
||||
byPath := differencesByPath(differences)
|
||||
secret, ok := byPath["/backends/local/extra_params/token"]
|
||||
if !ok || !secret.Left.Redacted || !secret.Right.Redacted || !secret.HiddenChanged {
|
||||
t.Fatalf("sensitive difference = (%+v, %t)", secret, ok)
|
||||
}
|
||||
if strings.Contains(secret.Left.YAML, "left-secret-sentinel") || strings.Contains(secret.Left.YAML, "right-secret-sentinel") ||
|
||||
strings.Contains(secret.Right.YAML, "left-secret-sentinel") || strings.Contains(secret.Right.YAML, "right-secret-sentinel") {
|
||||
t.Fatalf("sensitive snapshots disclosed a value: %+v", secret)
|
||||
}
|
||||
visible, ok := byPath["/backends/local/extra_params/visible"]
|
||||
if !ok || visible.Left.Redacted || visible.Right.Redacted || visible.HiddenChanged ||
|
||||
!strings.Contains(visible.Left.YAML, "before-value") || !strings.Contains(visible.Right.YAML, "after-value") {
|
||||
t.Fatalf("public sibling difference = (%+v, %t)", visible, ok)
|
||||
}
|
||||
|
||||
differences[0].Path = configkit.Keys("caller", "mutation")
|
||||
differences[0].Left.YAML = "caller mutation"
|
||||
differences[0].Right.YAML = "caller mutation"
|
||||
again, err := left.Diff(right)
|
||||
if err != nil {
|
||||
t.Fatalf("repeat comparison after mutating returned records: %v", err)
|
||||
}
|
||||
if len(again) != 2 {
|
||||
t.Fatalf("repeat difference count = %d, want 2: %+v", len(again), again)
|
||||
}
|
||||
againByPath := differencesByPath(again)
|
||||
if _, exists := againByPath["/caller/mutation"]; exists {
|
||||
t.Fatal("mutating returned path reached retained inspection")
|
||||
}
|
||||
if _, exists := againByPath["/backends/local/extra_params/token"]; !exists {
|
||||
t.Fatalf("original sensitive path was lost after mutation: %+v", again)
|
||||
}
|
||||
if _, exists := againByPath["/backends/local/extra_params/visible"]; !exists {
|
||||
t.Fatalf("original public path was lost after mutation: %+v", again)
|
||||
}
|
||||
for _, difference := range again {
|
||||
if difference.Left.YAML == "caller mutation" || difference.Right.YAML == "caller mutation" {
|
||||
t.Fatalf("mutating returned snapshots reached retained inspection: %+v", difference)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigurationInspectionDiffMasksSensitiveShapeReplacement(t *testing.T) {
|
||||
left := comparisonInspectionFrom(t, "before", "same-model", map[string]any{
|
||||
"details": map[string]any{
|
||||
"token": "nested-secret-sentinel",
|
||||
"label": "public detail",
|
||||
},
|
||||
})
|
||||
right := comparisonInspectionFrom(t, "after", "same-model", map[string]any{
|
||||
"details": "replacement",
|
||||
})
|
||||
|
||||
differences, err := left.Diff(right)
|
||||
if err != nil {
|
||||
t.Fatalf("compare sensitive shape replacement: %v", err)
|
||||
}
|
||||
if len(differences) != 1 || differences[0].Path.String() != "/backends/local/extra_params/details" {
|
||||
t.Fatalf("shape replacement differences = %+v", differences)
|
||||
}
|
||||
difference := differences[0]
|
||||
if !difference.Left.Redacted || !difference.Right.Redacted || !difference.HiddenChanged {
|
||||
t.Fatalf("shape replacement masking = %+v", difference)
|
||||
}
|
||||
if strings.Contains(difference.Left.YAML, "nested-secret-sentinel") || strings.Contains(difference.Right.YAML, "nested-secret-sentinel") {
|
||||
t.Fatalf("shape replacement disclosed a sensitive value: %+v", difference)
|
||||
}
|
||||
}
|
||||
|
||||
func comparisonInspection(t *testing.T, model string) promptkit.ConfigurationInspection {
|
||||
t.Helper()
|
||||
return comparisonInspectionFrom(t, "comparison fixture", model, nil)
|
||||
}
|
||||
|
||||
func comparisonInspectionFrom(t *testing.T, source, model string, extraParams map[string]any) promptkit.ConfigurationInspection {
|
||||
t.Helper()
|
||||
backend := map[string]any{"endpoint": "https://example.invalid/v1"}
|
||||
if extraParams != nil {
|
||||
backend["extra_params"] = extraParams
|
||||
}
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: configkit.Mapping(source, map[string]any{
|
||||
"backends": map[string]any{"local": backend},
|
||||
"profiles": map[string]any{"offline": map[string]any{
|
||||
"backend": "local",
|
||||
"model": model,
|
||||
}},
|
||||
})}}}},
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatalf("construct comparison engine: %v", err)
|
||||
}
|
||||
inspection, err := engine.InspectConfiguration()
|
||||
if err != nil {
|
||||
t.Fatalf("inspect comparison engine: %v", err)
|
||||
}
|
||||
return inspection
|
||||
}
|
||||
|
||||
func differencesByPath(differences []configkit.Difference) map[string]configkit.Difference {
|
||||
byPath := make(map[string]configkit.Difference, len(differences))
|
||||
for _, difference := range differences {
|
||||
byPath[difference.Path.String()] = difference
|
||||
}
|
||||
return byPath
|
||||
}
|
||||
|
||||
func hasSourceLabel(view promptkit.ConfigurationInspection, label string) bool {
|
||||
for _, source := range view.Sources() {
|
||||
if source.Label == label {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -27,6 +27,23 @@ func (e *Engine) InspectConfiguration() (ConfigurationInspection, error) {
|
||||
return ConfigurationInspection{source: e.internal.InspectConfiguration()}, nil
|
||||
}
|
||||
|
||||
// Diff compares this view's captured effective configuration with other. This
|
||||
// view is the earlier (left) value and other is the later (right) value. The
|
||||
// comparison delegates to ConfigKit's frozen effective-value semantics, so
|
||||
// source or resolution history alone does not create differences. It does not
|
||||
// reread external sources or compare referenced file contents, resolved
|
||||
// credential contents, later execution preparation, or request-specific
|
||||
// overrides. Either zero view returns ErrInvalidConfig and no records; other
|
||||
// ConfigKit errors are returned unchanged. An empty diff is not evidence of
|
||||
// equivalent engine behavior or execution. Records can contain ordinary
|
||||
// application data, so callers choose where and to whom to disclose them.
|
||||
func (v ConfigurationInspection) Diff(other ConfigurationInspection) ([]configkit.Difference, error) {
|
||||
if v.source == nil || other.source == nil {
|
||||
return nil, ErrInvalidConfig
|
||||
}
|
||||
return configuration.Diff(v.source, other.source)
|
||||
}
|
||||
|
||||
// Metadata returns detached effective presence and history records.
|
||||
func (v ConfigurationInspection) Metadata() configkit.Metadata {
|
||||
if v.source == nil {
|
||||
|
||||
@@ -144,9 +144,17 @@ func TestConfigurationInspectionUsesCapturedSourcesWithoutProviderCalls(t *testi
|
||||
if err != nil {
|
||||
t.Fatalf("select inputs: %v", err)
|
||||
}
|
||||
otherSource := &unreadableAfterConstructionFS{FS: fstest.MapFS{
|
||||
"config/prompt.yaml": &fstest.MapFile{Data: []byte(capturedPromptDocument)},
|
||||
"config/body.tmpl": &fstest.MapFile{Data: []byte("hello")},
|
||||
}}
|
||||
otherInputs, err := promptkit.FilesystemInputs("comparison source", otherSource, "config")
|
||||
if err != nil {
|
||||
t.Fatalf("select comparison inputs: %v", err)
|
||||
}
|
||||
var modelCalls atomic.Int64
|
||||
client := countingLLMClient{calls: &modelCalls}
|
||||
endpoint, model := "https://example.invalid/v1", "offline-model"
|
||||
endpoint, model, otherModel := "https://example.invalid/v1", "offline-model", "comparison-model"
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||
Underlays: []configkit.Underlay{{Inputs: inputs}, {Inputs: []configkit.Input{{Source: promptkit.ProfileSource("offline profile", map[string]promptkit.ProfileConfig{
|
||||
"offline": {Endpoint: &endpoint, Model: &model},
|
||||
@@ -155,11 +163,42 @@ func TestConfigurationInspectionUsesCapturedSourcesWithoutProviderCalls(t *testi
|
||||
if err != nil {
|
||||
t.Fatalf("construct engine: %v", err)
|
||||
}
|
||||
otherEngine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||
Underlays: []configkit.Underlay{{Inputs: otherInputs}, {Inputs: []configkit.Input{{Source: promptkit.ProfileSource("comparison profile", map[string]promptkit.ProfileConfig{
|
||||
"offline": {Endpoint: &endpoint, Model: &otherModel},
|
||||
})}}}},
|
||||
}}, promptkit.WithLLMClient(client))
|
||||
if err != nil {
|
||||
t.Fatalf("construct comparison engine: %v", err)
|
||||
}
|
||||
source.closed.Store(true)
|
||||
otherSource.closed.Store(true)
|
||||
view, err := engine.InspectConfiguration()
|
||||
if err != nil {
|
||||
t.Fatalf("inspect configuration: %v", err)
|
||||
}
|
||||
otherView, err := otherEngine.InspectConfiguration()
|
||||
if err != nil {
|
||||
t.Fatalf("inspect comparison configuration: %v", err)
|
||||
}
|
||||
var expected configkit.Difference
|
||||
for index := 0; index < 3; index++ {
|
||||
differences, err := view.Diff(otherView)
|
||||
if err != nil || len(differences) != 1 {
|
||||
t.Fatalf("compare captured configurations = (%+v, %v), want one difference", differences, err)
|
||||
}
|
||||
if differences[0].Path.String() != "/profiles/offline/model" ||
|
||||
!strings.Contains(differences[0].Left.YAML, model) ||
|
||||
!strings.Contains(differences[0].Right.YAML, otherModel) {
|
||||
t.Fatalf("captured difference = %+v", differences[0])
|
||||
}
|
||||
if index == 0 {
|
||||
expected = differences[0]
|
||||
} else if differences[0].Path.String() != expected.Path.String() ||
|
||||
differences[0].Left.YAML != expected.Left.YAML || differences[0].Right.YAML != expected.Right.YAML {
|
||||
t.Fatalf("captured difference changed between reads: first=%+v next=%+v", expected, differences[0])
|
||||
}
|
||||
}
|
||||
if _, found := view.Explain(configkit.Keys("prompts", "captured", "1", "messages").Index(0).Key("content_file")); !found {
|
||||
t.Fatal("captured prompt content path was not inspectable")
|
||||
}
|
||||
@@ -167,24 +206,32 @@ func TestConfigurationInspectionUsesCapturedSourcesWithoutProviderCalls(t *testi
|
||||
if err := view.WriteEffective(&report); err != nil || !strings.Contains(report.String(), "captured") {
|
||||
t.Fatalf("write captured report = (%q, %v)", report.String(), err)
|
||||
}
|
||||
if source.reads.Load() != 0 || modelCalls.Load() != 0 {
|
||||
t.Fatalf("inspection touched source or provider: reads=%d requests=%d", source.reads.Load(), modelCalls.Load())
|
||||
if source.reads.Load() != 0 || otherSource.reads.Load() != 0 || modelCalls.Load() != 0 {
|
||||
t.Fatalf("inspection or comparison touched source or provider: reads=(%d,%d) requests=%d", source.reads.Load(), otherSource.reads.Load(), modelCalls.Load())
|
||||
}
|
||||
|
||||
const calls = 16
|
||||
const calls = 24
|
||||
var wait sync.WaitGroup
|
||||
errs := make(chan error, calls)
|
||||
for index := 0; index < calls; index++ {
|
||||
wait.Add(1)
|
||||
go func(index int) {
|
||||
defer wait.Done()
|
||||
if index%2 == 0 {
|
||||
switch index % 3 {
|
||||
case 0:
|
||||
_, err := engine.InspectConfiguration()
|
||||
errs <- err
|
||||
return
|
||||
case 1:
|
||||
_, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "captured", SessionID: "direct"})
|
||||
errs <- err
|
||||
case 2:
|
||||
differences, err := view.Diff(otherView)
|
||||
if err == nil && (len(differences) != 1 || differences[0].Path.String() != expected.Path.String() ||
|
||||
differences[0].Left.YAML != expected.Left.YAML || differences[0].Right.YAML != expected.Right.YAML) {
|
||||
err = errors.New("concurrent comparison returned an unexpected difference")
|
||||
}
|
||||
errs <- err
|
||||
}
|
||||
_, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "captured", SessionID: "direct"})
|
||||
errs <- err
|
||||
}(index)
|
||||
}
|
||||
wait.Wait()
|
||||
@@ -194,7 +241,7 @@ func TestConfigurationInspectionUsesCapturedSourcesWithoutProviderCalls(t *testi
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if source.reads.Load() != 0 || modelCalls.Load() != calls/2 {
|
||||
t.Fatalf("concurrent inspection/execution = source reads %d, model calls %d", source.reads.Load(), modelCalls.Load())
|
||||
if source.reads.Load() != 0 || otherSource.reads.Load() != 0 || modelCalls.Load() != calls/3 {
|
||||
t.Fatalf("concurrent inspection/comparison/execution = source reads (%d,%d), model calls %d", source.reads.Load(), otherSource.reads.Load(), modelCalls.Load())
|
||||
}
|
||||
}
|
||||
|
||||
25
doc.go
25
doc.go
@@ -4,15 +4,16 @@
|
||||
// Applications construct an [Engine] with [NewEngine], select ConfigKit
|
||||
// sources for prompts, profiles, and backends, and
|
||||
// call [Engine.InspectConfiguration], [Engine.InspectPrompt],
|
||||
// [Engine.InspectProfile], [Engine.Prepare],
|
||||
// [Engine.PrepareExecution], [Engine.Run], or [Engine.RunPrepared]. Concrete
|
||||
// registries, repositories, validators, and the built-in OpenAI-compatible
|
||||
// client remain internal implementation details.
|
||||
// [Engine.InspectRequirements], [Engine.InspectProfile], [Engine.ListProfiles],
|
||||
// [Engine.ListPrompts], [Engine.Prepare], [Engine.PrepareExecution],
|
||||
// [Engine.Run], or [Engine.RunPrepared]. Concrete registries, repositories,
|
||||
// validators, and the built-in OpenAI-compatible client remain internal
|
||||
// implementation details.
|
||||
//
|
||||
// # Concurrency and ownership
|
||||
//
|
||||
// An Engine supports concurrent InspectConfiguration, InspectPrompt,
|
||||
// InspectProfile, Prepare,
|
||||
// InspectRequirements, InspectProfile, ListProfiles, ListPrompts, Prepare,
|
||||
// PrepareExecution, Run, and RunPrepared calls. Engine-local backend policies
|
||||
// bound admitted Run and RunPrepared calls and model generations where
|
||||
// configured, while different backend pools and unlimited backends continue
|
||||
@@ -20,11 +21,12 @@
|
||||
// still receive concurrent calls and must be safe for that use.
|
||||
//
|
||||
// ConfigKit captures the selected configuration when NewEngine constructs an
|
||||
// Engine. Prepare,
|
||||
// PrepareExecution, and Run copy request maps, slices, pointer values, and
|
||||
// JSON-compatible extra parameters before using them. InspectPrompt and
|
||||
// InspectProfile return copied inspection values. Returned values and values
|
||||
// passed to extension interfaces are likewise isolated from engine state.
|
||||
// Engine. Prepare, PrepareExecution, and Run copy request maps, slices, pointer
|
||||
// values, and JSON-compatible extra parameters before using them. InspectPrompt,
|
||||
// InspectRequirements, and InspectProfile return copied inspection values.
|
||||
// ListProfiles and ListPrompts return caller-owned identity slices.
|
||||
// Returned values and values passed to extension interfaces are likewise
|
||||
// isolated from engine state.
|
||||
// Callers own those copies and may mutate them after the call that supplied or
|
||||
// returned them. [CapacityError] values are caller-owned and may be mutated
|
||||
// without affecting engine state or another error. Immutable [GenerationError]
|
||||
@@ -58,7 +60,8 @@
|
||||
// [ConfigurationSources], [ConfigurationInspection], [BackendConfig],
|
||||
// [ProfileConfig], [RunRequest],
|
||||
// [ArtifactRef], [ExecutionTargetOverride], [ProfileInspection],
|
||||
// [PromptInputDefinition], [PromptInspection], [PreparedExecution],
|
||||
// [PromptIdentity], [PromptInputDefinition], [PromptInspection],
|
||||
// [RequirementInspectionRequest], [RequirementInspection], [PreparedExecution],
|
||||
// [CapacityError], [GenerationError], [RetryExhaustedError], and
|
||||
// [ExecutionEvent], do not have stable JSON
|
||||
// representations. Direct API keys are nevertheless excluded from JSON for
|
||||
|
||||
@@ -19,6 +19,20 @@ The following Go fragments are illustrative and omit surrounding package,
|
||||
import, and error-handling code. Use the maintained examples for complete
|
||||
programs.
|
||||
|
||||
## Go Compatibility
|
||||
|
||||
The `go` directive in the selected release's [module metadata](../../go.mod)
|
||||
declares its minimum supported consumer Go version. Maintainers build and
|
||||
validate releases with the locally selected Go patch release; the minimum
|
||||
does not simply track that patch version.
|
||||
|
||||
Successful release validation on the selected patch is sufficient to presume
|
||||
support across its Go series, subject to the declared minimum and concrete
|
||||
evidence of incompatibility. Specific compatibility concerns are investigated
|
||||
as they arise. Minor releases may raise the minimum; patch releases preserve
|
||||
it. Release notes explain increases and any required consumer action. See the
|
||||
[release policy](../policy/release.md#consumer-minimum-and-release-toolchain).
|
||||
|
||||
## Construct An Engine
|
||||
|
||||
Create an engine with [`NewEngine`](../../engine.go). Select each directory
|
||||
@@ -104,6 +118,43 @@ engines and prepared handles retain their captured definitions; schema and
|
||||
request artifact resources still follow their runtime lifecycle. Inspection
|
||||
does not watch or reload sources automatically.
|
||||
|
||||
## Discover Available Profiles And Prompts
|
||||
|
||||
Use [`Engine.ListProfiles`](../../listing.go) and
|
||||
[`Engine.ListPrompts`](../../listing.go) to discover exact identities from the
|
||||
engine's final captured configuration, then inspect a selected prompt:
|
||||
|
||||
```go
|
||||
profiles, err := engine.ListProfiles(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(profiles) > 0 {
|
||||
_, err := engine.InspectProfile(ctx, profiles[0])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
prompts, err := engine.ListPrompts(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(prompts) > 0 {
|
||||
selected := prompts[0]
|
||||
inspection, err := engine.InspectPrompt(ctx, selected.ID, selected.Version)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_ = inspection
|
||||
}
|
||||
```
|
||||
|
||||
These lists describe effective membership, not ConfigKit contribution history,
|
||||
and do not promise that a profile is ready to execute. Use inspection for
|
||||
details and preparation to resolve request resources. The linked method GoDoc
|
||||
owns ordering, ownership, concurrency, and error behavior; identifiers may be
|
||||
sensitive application data.
|
||||
|
||||
## Inspect The Captured Configuration
|
||||
|
||||
Use [`Engine.InspectConfiguration`](../../configuration_inspection.go) for an
|
||||
@@ -130,6 +181,41 @@ shows backend-completed execution settings for one profile, and `Prepare` or
|
||||
`PreparedExecution.Details` shows request-specific rendered messages, inputs,
|
||||
and validation resources. None of those views is a substitute for the others.
|
||||
|
||||
To review changes between two independently captured configurations, compare
|
||||
their inspection views:
|
||||
|
||||
```go
|
||||
earlier, err := earlierEngine.InspectConfiguration()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
later, err := candidateEngine.InspectConfiguration()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
differences, err := earlier.Diff(later)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, difference := range differences {
|
||||
if difference.HiddenChanged {
|
||||
// Report a protected change without inferring its value.
|
||||
}
|
||||
if difference.Left.Redacted || difference.Right.Redacted {
|
||||
// Apply the application's disclosure policy before presenting snapshots.
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
See the [`ConfigurationInspection.Diff` GoDoc](../../configuration_inspection.go)
|
||||
and ConfigKit's [v0.7.0 effective-configuration diff guidance](https://gitea.maximumdirect.net/eric/ConfigKit/src/tag/v0.7.0/docs/config.md#effective-configuration-diffs)
|
||||
for the exact comparison contract. The operation compares only frozen effective
|
||||
configuration values: it does not reread external files or compare their
|
||||
contents, resolved credential contents, later execution preparation, or
|
||||
request-specific overrides. An empty diff is not proof of equivalent execution.
|
||||
Records can contain ordinary application data, so use the application's
|
||||
disclosure policy when presenting them.
|
||||
|
||||
## Inspect A Prompt Before Preparation
|
||||
|
||||
Use [`Engine.InspectPrompt`](../../engine.go) to check one configured prompt's
|
||||
@@ -154,6 +240,50 @@ rendered messages, and use prepared execution when that work must remain tied
|
||||
to later execution. The method's [GoDoc](../../engine.go) owns exact fields,
|
||||
hash, ownership, and error semantics.
|
||||
|
||||
## Inspect Requirements Before Preparation
|
||||
|
||||
Use [`Engine.InspectRequirements`](../../requirements.go) when the application
|
||||
needs the prompt's recognizable input and caller-variable dependencies for a
|
||||
particular context before it has input artifacts. Unlike `InspectPrompt`, it
|
||||
also walks captured template syntax and selects a profile identity; unlike
|
||||
`Prepare`, it does not load artifacts or render messages.
|
||||
|
||||
```go
|
||||
requirements, err := engine.InspectRequirements(ctx, promptkit.RequirementInspectionRequest{
|
||||
PromptID: "meeting.summary",
|
||||
Vars: map[string]string{"audience": "project leads"},
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, input := range requirements.Inputs {
|
||||
if input.Kind == promptkit.RequirementRequired {
|
||||
// Plan to bind this input before preparation.
|
||||
}
|
||||
if input.Kind == promptkit.RequirementOptional && input.Explanation != "" {
|
||||
// A recognizable reference may occur only on a known skipped path.
|
||||
}
|
||||
}
|
||||
for _, variable := range requirements.Variables {
|
||||
if variable.Kind == promptkit.RequirementRequired && !variable.Supplied {
|
||||
// The key is required even though this inspection did not receive it.
|
||||
}
|
||||
}
|
||||
if !requirements.Complete {
|
||||
// Preserve the partial inventory and account for its discovery issues.
|
||||
}
|
||||
```
|
||||
|
||||
Optional entries can include recognizable inputs and variables found only in
|
||||
inactive branches. A missing required variable is reported separately from
|
||||
discovery completeness: missing a value used only for direct text substitution
|
||||
does not by itself make the inventory incomplete. Conversely, an unresolved
|
||||
dependency-bearing branch can make discovery incomplete while retaining known
|
||||
conditional entries. Do not treat an empty partial inventory as proof that no
|
||||
input bindings are needed. See the method's [GoDoc](../../requirements.go) for
|
||||
the exact classifications, supported syntax, ownership, and error contract.
|
||||
|
||||
## Prepare Without Model Execution
|
||||
|
||||
[`Engine.Prepare`](../../engine.go) resolves the selected prompt and profile,
|
||||
|
||||
@@ -59,37 +59,43 @@ exist.
|
||||
## Maintainer Validation
|
||||
|
||||
For ordinary work, select validation proportionate to the change. Do not repeat
|
||||
successful checks for unchanged content merely to create a commit. The full
|
||||
release-candidate checks have one executable entry point from the repository root:
|
||||
successful checks for unchanged content merely to create a commit. During
|
||||
development, run relevant Go tests and vet, format changed Go files, and execute
|
||||
the maintained examples when they are affected:
|
||||
|
||||
```sh
|
||||
./scripts/release-check.sh
|
||||
go test ./...
|
||||
go vet ./...
|
||||
gofmt -w path/to/changed.go
|
||||
python3 scripts/check-examples.py
|
||||
git diff --check
|
||||
```
|
||||
|
||||
The [script](../scripts/release-check.sh) owns the executable checks:
|
||||
race-enabled tests, vet, the three offline consumer examples with semantic output
|
||||
assertions, tracked Go formatting, local Markdown links, module/workspace
|
||||
hygiene, whitespace, and the credential-pattern scan. It does not publish,
|
||||
change files, validate release-note structure, or retire roadmaps. It requires Go,
|
||||
Git, a POSIX shell, and Python 3. Tests and examples require no provider
|
||||
credentials or live inference calls.
|
||||
Release candidates use `./scripts/release.sh VERSION`, which runs the selected
|
||||
profile checks and publishes the tag. The [release procedure](release.md)
|
||||
describes its requirements and recovery. For an optional diagnostic of a
|
||||
committed candidate without publication, run `./scripts/releasekit.sh check
|
||||
VERSION`; this is not a required pre-publication step.
|
||||
|
||||
The script can run on intended uncommitted changes during development.
|
||||
Formatting and documentation checks use tracked files; stage intended new
|
||||
files before final validation so those checks include them. The release
|
||||
procedure separately enforces a clean, committed candidate.
|
||||
The standalone `python3 scripts/check-examples.py` command checks semantic
|
||||
expectations for the three offline consumer examples. The focused
|
||||
`python3 scripts/test-release.py` command exercises the release tooling against
|
||||
disposable local repositories; use it when changing that tooling, not as a
|
||||
recurring release gate. Tests and examples require no provider credentials or
|
||||
live inference calls.
|
||||
|
||||
Ordinary validation can run on intended uncommitted work. Release validation
|
||||
uses committed candidate contents and the selected local Go installation.
|
||||
|
||||
### Change Review
|
||||
|
||||
Inspect `git status --short --untracked-files=all` and the complete diff before
|
||||
accepting a change. Reject unintended files, credentials, private data,
|
||||
generated binaries, test output, downloaded assets, and template residue.
|
||||
The automated credential scan does not replace review of the actual change.
|
||||
|
||||
For releases, follow the [release procedure](release.md). Its publication
|
||||
command runs the full local checks once on the final committed candidate and
|
||||
automates release-note structure and Git checks. Do not repeat ordinary change
|
||||
review as a release sign-off. Roadmap retirement remains preparation housekeeping.
|
||||
For releases, follow the [release procedure](release.md). Do not repeat its
|
||||
candidate checks as a separate sign-off. Roadmap retirement remains preparation
|
||||
housekeeping.
|
||||
|
||||
Follow the [agent commit policy](policy/agents.md#git-commit-policy) for local
|
||||
commits. Ordinary commits may leave unrelated work in the workspace; preserve
|
||||
|
||||
@@ -51,7 +51,8 @@ message files on ordinary runs; construct a new engine to observe edits.
|
||||
Schema and request artifact reads still happen during preparation. Use
|
||||
`InspectConfiguration` for an explicit effective-configuration or provenance
|
||||
report, `InspectProfile` for backend-completed profile settings, and preparation
|
||||
for request-specific rendered details.
|
||||
for request-specific rendered details. For identity discovery before inspection,
|
||||
see the consumer guide's [profile and prompt listing](consumers/pkg-promptkit.md#discover-available-profiles-and-prompts).
|
||||
|
||||
## Prompt Definitions
|
||||
|
||||
@@ -124,6 +125,12 @@ Optional `session_id` uses the same template variables and input helper. Its
|
||||
rendered value is trimmed, omitted if empty, and limited to 256 Unicode code
|
||||
points. A nonblank direct request session ID bypasses the template.
|
||||
|
||||
`InspectRequirements` can enumerate recognizable dependencies from captured
|
||||
templates before artifact content is available. Its conservative syntax
|
||||
coverage does not change template validity or rendering; see the [consumer
|
||||
guide](consumers/pkg-promptkit.md#inspect-requirements-before-preparation) and
|
||||
the method's [GoDoc](../requirements.go) for that inspection boundary.
|
||||
|
||||
### Cache Control
|
||||
|
||||
Optional `cache_control` has required `type: ephemeral` and optional `ttl`,
|
||||
|
||||
@@ -78,9 +78,8 @@ The client conditionally includes:
|
||||
|
||||
The engine resolves backend, profile, and request extra-parameter maps by
|
||||
whole-map replacement rather than key merging. The resulting effective map is
|
||||
then merged directly into the top-level body after JSON serialization is
|
||||
verified. Empty keys and collisions with these reserved fields are rejected
|
||||
before any provider call:
|
||||
merged into the top-level body. Empty keys, values that cannot be encoded, and
|
||||
collisions with these reserved fields are rejected before any provider call:
|
||||
|
||||
- `model`
|
||||
- `session_id`
|
||||
|
||||
@@ -34,9 +34,13 @@ resolved request target may supply the endpoint. Generation preparation then:
|
||||
1. validates shared execution-setting invariants and the final selected base
|
||||
endpoint;
|
||||
2. maps the internal request into the OpenAI-compatible chat payload;
|
||||
3. validates and merges extra parameters;
|
||||
3. checks extra-parameter keys against empty and reserved names, then merges
|
||||
their values into the payload;
|
||||
4. composes `/chat/completions` through parsed URL path operations;
|
||||
5. resolves authentication and freezes the URL, body, and optional credential.
|
||||
5. encodes the complete body once on success. If encoding fails, extra values
|
||||
are checked individually to retain invalid-request classification; otherwise
|
||||
the complete-request failure remains a request failure;
|
||||
6. resolves authentication and freezes the URL, body, and optional credential.
|
||||
|
||||
Each execution of that prepared request creates a fresh HTTP request, body
|
||||
reader, and attempt timeout context. It closes the response body and returns
|
||||
|
||||
@@ -11,28 +11,28 @@ contributor workflow and validation.
|
||||
|
||||
| Component | Implemented responsibility | References |
|
||||
| --- | --- | --- |
|
||||
| Root `promptkit` package | Provides the supported engine facade, ConfigKit source helpers, profile and backend mapping conveniences, injection options, public requests and results, explicit inspection, observation values, opaque prepared-execution handles, value conversion, redacted formatting, and typed capacity and generation error mapping. | [Package GoDoc](../../doc.go), [observations](../../observations.go), [prepared execution](../../prepared_execution.go), [backend API](../../backends.go), [engine facade](../../engine.go) |
|
||||
| Root `promptkit` package | Provides the supported engine facade, ConfigKit source helpers, profile and backend mapping conveniences, injection options, public requests and results, prompt/profile/requirement inspection and identity listing, observation values, opaque prepared-execution handles, value conversion, redacted formatting, and typed capacity and generation error mapping. | [Package GoDoc](../../doc.go), [observations](../../observations.go), [prepared execution](../../prepared_execution.go), [backend API](../../backends.go), [engine facade](../../engine.go), [identity listing](../../listing.go), [requirement facade](../../requirements.go) |
|
||||
| `examples/go-library/prepare` | Demonstrates offline preparation and explicit configuration inspection with a prompt file, programmatic profile, and inline input. It is not a public library package. | [Example program](../../examples/go-library/prepare/main.go) |
|
||||
| `examples/go-library/run` | Demonstrates an offline downstream consumer using a prompt file, programmatic profile, inline input, an injected deterministic model client, and `Run`. It is not a public library package. | [Example program](../../examples/go-library/run/main.go) |
|
||||
| `examples/go-library/observe` | Demonstrates offline built-in HTTP retry and synchronous metadata observation across structured proposal and assessment runs using embedded sources, two profiles on one finite backend, and an application decision. It is not a public library package. | [Example program](../../examples/go-library/observe/main.go) |
|
||||
| `internal/backend` | Holds a defensive, immutable lookup and capacity-policy index for final backend definitions validated by the configuration adapter. | [Backend registry](../../internal/backend/registry.go) |
|
||||
| `internal/catalog` | Selects released keyed catalog assets and PromptKit-owned profile definitions as ConfigKit underlays; effective validation belongs to the configuration loader. | [Catalog adapter](../../internal/catalog/catalog.go), [internal sources](sources.md#profiles-and-built-ins) |
|
||||
| `internal/configuration` | Loads keyed backend, profile, and prompt configuration through ConfigKit; records effective field presence, validates final values, captures indexed prompt definitions and referenced message content, constructs backend and presence-aware profile snapshots, and retains an explicit read-only inspection view. | [Typed loader](../../internal/configuration/load.go), [internal sources](sources.md#typed-configuration-loading) |
|
||||
| `internal/configuration` | Loads keyed backend, profile, and prompt configuration through ConfigKit; records effective field presence, validates final values, captures indexed prompt definitions, referenced message content, compiled templates, input metadata snapshots, sorted identity-only profile and prompt inventories, backend and presence-aware profile snapshots, and an explicit read-only inspection view. | [Typed loader](../../internal/configuration/load.go), [identity listing](../../internal/configuration/listing.go), [internal sources](sources.md#typed-configuration-loading) |
|
||||
| `internal/capacity` | Owns engine-local bounded execution admission and FIFO model-generation permits for limited backend IDs, including cancellation-safe waiter removal and client wrapping. | [Internal capacity management](capacity.md) |
|
||||
| `internal/domain` | Defines internal framework values for requests, artifacts, prompt definitions, profiles, execution targets, rendering, generation, and validation, and owns source-neutral invariants for shared execution settings, OpenAI-compatible base endpoints, session identifiers, and output contracts. Source parsing, required fields, other source-specific normalization, defaulting, and boundary-specific error classification remain with their callers. | [Domain declarations](../../internal/domain/domain.go), [endpoint invariant](../../internal/domain/endpoint.go) |
|
||||
| `internal/engine` | Assembles selected catalog and caller configuration into one snapshot, then constructs capacity, rendering, validation, generation, repair, and the use-case runner for each engine. | [Engine assembly](../../internal/engine/engine.go), [internal sources](sources.md#profiles-and-built-ins), [internal runner](runner.md) |
|
||||
| `internal/domain` | Defines internal framework values for requests, artifacts, prompt definitions, requirement inspection, profiles, execution targets, rendering, generation, and validation, and owns source-neutral invariants for shared execution settings, OpenAI-compatible base endpoints, session identifiers, and output contracts. Source parsing, required fields, other source-specific normalization, defaulting, and boundary-specific error classification remain with their callers. | [Domain declarations](../../internal/domain/domain.go), [requirement values](../../internal/domain/requirements.go), [endpoint invariant](../../internal/domain/endpoint.go) |
|
||||
| `internal/engine` | Assembles selected catalog and caller configuration into one snapshot, then constructs capacity, rendering, validation, generation, repair, and the use-case runner for each engine. It delegates identity listing directly to the captured configuration inventory rather than the runner. | [Engine assembly](../../internal/engine/engine.go), [internal sources](sources.md#profiles-and-built-ins), [internal runner](runner.md) |
|
||||
| `internal/generation` | Coordinates frozen built-in HTTP attempts and one-shot injected calls through active backend permits, including bounded status retries, cancellable backoff, and attempt observations. | [Internal generation coordination](generation.md) |
|
||||
| `internal/defaults` | Defines application-neutral framework constants and constructs the default execution target. It contains no CLI, server, or inbound HTTP limits. | [Framework defaults](../../internal/defaults/defaults.go) |
|
||||
| `internal/filecatalog` | Provides deterministic YAML discovery and path helpers for operating-system filesystems and `fs.FS` sources. | [File catalog](../../internal/filecatalog/catalog.go) |
|
||||
| `internal/jsonvalue` | Validates and deeply copies bounded JSON-compatible extra-parameter and prepared-schema trees while preserving supported concrete value types and rejecting cycles or excessive depth and work. | [JSON values](../../internal/jsonvalue/jsonvalue.go) |
|
||||
| `internal/promptdef` | Provides a narrow captured-prompt lookup contract, not-found identity, and execution-ready definition hash. | [Prompt lookup](../../internal/promptdef/repository.go), [framework formats](../formats.md) |
|
||||
| `internal/profile` | Provides a narrow captured-profile lookup contract and not-found identity. | [Profile lookup](../../internal/profile/repository.go), [framework formats](../formats.md) |
|
||||
| `internal/prompt` | Renders prompt messages from Go templates with artifact, variable, session, and cache-control data. | [Go-template renderer](../../internal/prompt/go_renderer.go) |
|
||||
| `internal/prompt` | Renders prompt messages from Go templates with artifact, variable, session, and cache-control data, and conservatively enumerates dependencies from captured template syntax. | [Go-template renderer](../../internal/prompt/go_renderer.go), [requirement discovery](../../internal/prompt/requirements.go) |
|
||||
| `internal/artifact` | Resolves ordinary inline and unrestricted caller-selected file references into copied artifacts with metadata and hashes. | [Internal sources and validation](sources.md) |
|
||||
| `internal/validate` | Validates basic, JSON, and JSON Schema output using operating-system filesystem or `fs.FS` schema sources and creates operation-local validation plans with canonical contained schema resources. | [Framework formats](../formats.md#schemas), [internal sources and validation](sources.md) |
|
||||
| `internal/llm` | Defines the internal generation boundary and implements outbound OpenAI-compatible chat requests from resolved execution targets, including bounded structured non-success response decoding, successful-response decoding, authentication, deadline handling, and ownership of the OpenAI-compatible reserved request-field policy. | [Internal model client](llm.md) |
|
||||
| `internal/observation` | Owns invocation-local run IDs, presence-aware usage and phase accounting, and synchronous metadata event delivery for ordinary and claimed prepared runs and generation attempts. | [Internal execution observation](observation.md) |
|
||||
| `internal/usecase` | Selects captured prompts and profiles, applies request overrides and completion checks, and coordinates ordinary and single-claim prepared execution across rendering, artifact loading, operation-local validation plans, generation, capacity, and bounded repair. | [Internal runner](runner.md), [prepared-execution implementation](../../internal/usecase/prepared_execution.go) |
|
||||
| `internal/usecase` | Selects captured prompts and profiles, enumerates requirements from captured prompt syntax, applies request overrides and completion checks, and coordinates ordinary and single-claim prepared execution across rendering, artifact loading, operation-local validation plans, generation, capacity, and bounded repair. | [Internal runner](runner.md), [requirement inspection](../../internal/usecase/requirements_inspection.go), [prepared-execution implementation](../../internal/usecase/prepared_execution.go) |
|
||||
|
||||
The internal engine assembles these internal components without exposing their
|
||||
representations. Consumers depend only on the root facade.
|
||||
|
||||
@@ -32,16 +32,29 @@ The runner has no durable run or session store.
|
||||
|
||||
## Shared Prompt Selection
|
||||
|
||||
The runner uses one prompt-selection and hashing boundary for ordinary
|
||||
preparation and exact prompt inspection. Preparation retains its early
|
||||
request-ID check before direct-session normalization; both operations select
|
||||
one captured definition and its construction-time hash. Referenced message
|
||||
content and static template syntax were captured when the engine was built.
|
||||
Ordinary preparation selects the full prompt definition and construction-time
|
||||
hash. It retains its early request-ID check before direct-session
|
||||
normalization. Referenced message content and static template syntax were
|
||||
captured when the engine was built.
|
||||
|
||||
Inspection stops after that structural lookup. It does not render templates or
|
||||
touch profile, artifact, schema, renderer, validator, admission, or model
|
||||
collaborators. The root [`Engine.InspectPrompt`](../../engine.go) GoDoc owns
|
||||
the public operation's exact contract.
|
||||
`InspectPrompt` selects captured metadata through an optional repository
|
||||
capability, falling back to the full definition and existing hash resolution
|
||||
for internal repositories without it. It returns owned metadata without
|
||||
rendering templates or touching profile, artifact, schema, renderer, validator,
|
||||
admission, or model collaborators. The root
|
||||
[`Engine.InspectPrompt`](../../engine.go) GoDoc owns the public operation's
|
||||
exact contract.
|
||||
|
||||
`InspectRequirements` uses the same metadata selection to resolve the
|
||||
request-profile/default identity, then obtains the exact captured requirement
|
||||
snapshot by the selected prompt ID and version. The analyzer receives only
|
||||
those identity fields rather than an execution definition. It analyzes the
|
||||
snapshot with the supplied caller variables and normalized direct-session
|
||||
bypass; it does not resolve target readiness or enter the preparation
|
||||
pipeline. Missing snapshot capability is a prompt-load failure rather than a
|
||||
source fallback. The root
|
||||
[`Engine.InspectRequirements`](../../requirements.go) GoDoc owns the public
|
||||
operation's exact contract.
|
||||
|
||||
## Shared Profile Selection
|
||||
|
||||
@@ -92,8 +105,11 @@ one execution-precedence and error-ordering implementation while allowing a
|
||||
full backend pool to reject work before expensive schema, artifact, and
|
||||
rendering operations. `Prepare` discards the plan after returning its public
|
||||
metadata. `Run` retains the plan through initial and repaired-output validation
|
||||
and discards it when the operation ends. Prepared execution stores the same
|
||||
kind of plan only in its private payload.
|
||||
and discards it when the operation ends. Prepared execution transfers the
|
||||
completed preparation's owned message, input-hash, and copied-schema values
|
||||
into its private payload, while deep-copying mutable provider extras and
|
||||
creating an independent details snapshot. Claim completion or discard clears
|
||||
the execution payload without removing the details snapshot.
|
||||
|
||||
Pointer-based numeric overrides preserve an explicit zero. Invalid negative or
|
||||
out-of-range values fail as invalid requests. Endpoint overrides do not change
|
||||
|
||||
@@ -21,6 +21,21 @@ profile inheritance. It retains ConfigKit's result privately for explicit
|
||||
inspection, not for per-run resolution. The runtime/domain packages do not
|
||||
depend on ConfigKit.
|
||||
|
||||
The `ConfigurationInspection.Diff` facade method compares two retained results
|
||||
through a private wrapper and a narrow `internal/configuration.Diff` bridge.
|
||||
The wrapper keeps `Result[document]` inside the adapter; the bridge delegates
|
||||
once to ConfigKit v0.7.0 and returns its public difference records. Comparison
|
||||
uses only frozen effective values and does not reopen sources, resolve runtime
|
||||
credentials, or enter the execution path. ConfigKit applies sensitivity from
|
||||
both results; reports may still contain ordinary application data. See the
|
||||
[consumer comparison guide](../consumers/pkg-promptkit.md#inspect-the-captured-configuration)
|
||||
and ConfigKit's [v0.7.0 comparison contract](https://gitea.maximumdirect.net/eric/ConfigKit/src/tag/v0.7.0/docs/config.md#effective-configuration-diffs).
|
||||
|
||||
After prompt materialization and effective-profile snapshot construction, the
|
||||
adapter captures lexically ordered, identity-only inventories from the final
|
||||
prompt and indexed-profile maps. Its listing methods return caller-owned copies
|
||||
of those inventories without consulting ConfigKit or copying definitions.
|
||||
|
||||
The root `DirectoryInputs` and `FilesystemInputs` helpers discover sorted YAML
|
||||
files under an explicit root, provide resource context, and leave imports
|
||||
disabled. Each returned slice belongs to one strict-composition group; file
|
||||
@@ -35,21 +50,68 @@ The configuration adapter indexes each prompt by exact ID and version,
|
||||
materializes file-backed message content using the exact effective field's
|
||||
origin and resource context, and checks static message/session template syntax
|
||||
at construction. It stores captured content, compiled templates, declared
|
||||
input metadata, and a definition hash. It closes content-file handles before
|
||||
input metadata, and a definition hash. Alongside the unchanged hashed
|
||||
definition, it retains content-type and description presence for each declared
|
||||
input and provides an internal requirement snapshot with those declarations
|
||||
and the captured parses. Metadata values come from the final normalized
|
||||
declaration; absent and null fields remain distinguishable from explicit
|
||||
strings, including empty strings. It closes content-file handles before
|
||||
returning and retains no source capability for public inspection.
|
||||
|
||||
Role and cache-control normalization is applied to execution-owned message
|
||||
copies. The retained ConfigKit configuration keeps effective input message
|
||||
metadata and its resource origins: recording a sequence-wide normalization in
|
||||
ConfigKit v0.5.0 would replace origins of unchanged content-file fields.
|
||||
copies. ConfigKit v0.7.0 retains effective input message metadata and its
|
||||
resource origins; recording a sequence-wide normalization in the retained
|
||||
configuration would replace origins of unchanged content-file fields.
|
||||
|
||||
The runner selects from this index for preparation and `InspectPrompt`.
|
||||
Neither path rescans configuration or reopens message files. `InspectPrompt`
|
||||
The runner selects full definitions from this index for preparation and uses
|
||||
captured prompt metadata for `InspectPrompt` and `InspectRequirements`; the
|
||||
internal requirement snapshot lookup selects by exact prompt ID and version.
|
||||
These paths do not rescan configuration or reopen message files. `InspectPrompt`
|
||||
returns declared metadata without rendering or resolving profile, input, or
|
||||
schema resources. Runtime rendering still checks request variables and input
|
||||
helpers. The [prompt adapter tests](../../internal/configuration/prompts_test.go)
|
||||
schema resources, and its public shape and prompt hashes do not include the
|
||||
captured presence state. Runtime rendering still checks request variables and
|
||||
input helpers. The [prompt adapter tests](../../internal/configuration/prompts_test.go)
|
||||
cover resource origins, context containment, capture, and selection.
|
||||
|
||||
The configuration adapter shares exact ID/version selection across its
|
||||
internal prompt lookups. The execution snapshot lookup returns an owned prompt
|
||||
definition and cached hash; metadata inspection returns owned input metadata
|
||||
without copying execution templates. Profile selection likewise has an
|
||||
identity-only lookup for consumers that do not need the execution target,
|
||||
while the full profile lookup continues to return an owned target and deep copy
|
||||
of provider extras. These lookups use captured indexes and do not query
|
||||
ConfigKit again.
|
||||
|
||||
The `internal/prompt` requirement walker uses the captured parse trees and
|
||||
declaration metadata without executing templates or reading artifacts. It
|
||||
collects literal `input` names, direct root caller keys, and literal-key
|
||||
`index` lookups. For `if` and `with`, it classifies branches when their
|
||||
conditions are simple supported values: direct values, literals, negation, or
|
||||
two-string equality/inequality. It uses Go template truthiness, so a supplied
|
||||
nonempty string such as `false` is true. Missing decision values and
|
||||
content-dependent conditions leave dependency-bearing branches conditional
|
||||
and report incomplete discovery; unresolved conditions with no branch
|
||||
dependencies do not by themselves make discovery incomplete. Recognizable
|
||||
references on known skipped paths remain optional inventory entries with an
|
||||
inactive-use explanation, unless another applicable use establishes a
|
||||
stronger classification. Skipped expressions are scanned only for
|
||||
syntactically recognizable names and are not evaluated. A direct caller value
|
||||
or literal-key lookup can also supply an input name, including through simple
|
||||
parentheses or a direct pipeline to `input`; names are used exactly as
|
||||
supplied. Unresolved names and unsupported dependency-bearing expressions
|
||||
produce structured limitations while retaining recognizable references.
|
||||
Where an unresolved name or scope could affect existing optional entries, those
|
||||
entries are reported as conditional rather than inactive-only or definitively
|
||||
optional. Direct root-map `with` forms retain caller-map identity. Known scalar
|
||||
values remain separate from unknown scopes; accesses through untracked local
|
||||
maps report incomplete discovery rather than silently losing caller keys.
|
||||
Unsupported name expressions reuse value analysis's conservative syntax scan
|
||||
instead of recursively rescanning the same subtree.
|
||||
Invoked named templates are scanned conservatively, and
|
||||
dependency-bearing range scopes make discovery incomplete. Uncalled definitions
|
||||
and bypassed session templates are not traversed. This internal walker does not
|
||||
change rendering behavior.
|
||||
|
||||
## Profiles And Built-Ins
|
||||
|
||||
`internal/catalog` selects the released keyed OpenRouter and Rakestrawhome
|
||||
@@ -66,10 +128,16 @@ snapshots with backend defaults and presence/suppression state. A profile can
|
||||
remain partial until runtime overrides complete it. `InspectProfile` selects
|
||||
that captured target without prompt, artifact, schema, credential lookup, or
|
||||
provider work. Preparation applies runtime overrides and then checks target
|
||||
completeness. The [target tests](../../internal/configuration/targets_test.go)
|
||||
cover effective target interpretation, while the [catalog tests](../../internal/catalog/catalog_test.go)
|
||||
completeness. Identity-only profile selection can identify a partial captured
|
||||
profile without copying its target or requiring readiness. The
|
||||
[target tests](../../internal/configuration/targets_test.go) cover effective
|
||||
target interpretation, while the [catalog tests](../../internal/catalog/catalog_test.go)
|
||||
cover selected released assets.
|
||||
|
||||
The internal engine delegates listing through a narrow identity-inventory
|
||||
interface directly to the configuration snapshot. The execution runner has no
|
||||
listing responsibility.
|
||||
|
||||
## Ordinary Artifacts
|
||||
|
||||
`internal/artifact` accepts explicitly typed inline references even when their
|
||||
@@ -79,10 +147,15 @@ and after opening the file. It copies content into an artifact, records
|
||||
metadata and an opaque content-equality value, and applies a content-type
|
||||
fallback.
|
||||
|
||||
Regular files are read synchronously in bounded chunks. Cancellation is
|
||||
checked before opening, before and after every read, and before publishing the
|
||||
artifact, so a canceled read never publishes partial content. The ordinary
|
||||
reader does not detach file reads into background goroutines.
|
||||
Regular files are read synchronously in bounded chunks. The opened file's size
|
||||
is only an initial-capacity hint, capped at 1 MiB; reads continue to EOF, and
|
||||
the returned body determines artifact size and hash. If an overreported hint
|
||||
leaves unused capacity greater than the actual body length, the completed body
|
||||
is copied to release the excess backing storage, including when the body is
|
||||
empty. Cancellation is checked before opening, before and after every read,
|
||||
and before publishing the artifact, so a
|
||||
canceled read never publishes partial content. The ordinary reader does not
|
||||
detach file reads into background goroutines.
|
||||
|
||||
This ordinary reader does not implement an inbound HTTP security boundary. In
|
||||
particular, it does not constrain files to an application root or impose an
|
||||
@@ -106,7 +179,9 @@ between artifact conversion chunks, around each message, and before publishing
|
||||
the complete prompt. It cannot interrupt template work already in progress and
|
||||
never publishes a partial prompt after observing cancellation. It validates and
|
||||
canonicalizes message roles before carrying roles, session IDs, and cache
|
||||
control into the rendered prompt. The
|
||||
control into the rendered prompt. Message and session templates execute into
|
||||
per-call string builders, so their rendered strings do not require a final
|
||||
buffer-to-string copy. The
|
||||
[renderer tests](../../internal/prompt/renderer_test.go) own rendering behavior.
|
||||
|
||||
## Schemas And Output Validation
|
||||
@@ -114,7 +189,8 @@ control into the rendered prompt. The
|
||||
`internal/validate` provides validators backed by an operating-system
|
||||
filesystem or an `fs.FS`. Invalid generated content is returned as a validation
|
||||
result; inability to load, register, or compile a schema is an operational
|
||||
error.
|
||||
error. Basic validation checks body emptiness on the byte slice directly
|
||||
instead of first making a response-sized string copy.
|
||||
|
||||
Every preparation operation creates one operation-local validation plan. None,
|
||||
basic, and JSON modes retain the effective output contract without source
|
||||
|
||||
@@ -6,29 +6,37 @@ Material: policy documents under `docs/policy/` and the Policy Reading Guide in
|
||||
`docs/development.md`, adapted for this repository; root `AGENTS.md`, copied
|
||||
without modification from upstream `agents/AGENTS.md`.
|
||||
|
||||
The release policy uses upstream commit
|
||||
`e833faddf120c789a9b45348edaf75720de16348`; the remaining material uses
|
||||
`d84a38ebab26f048cafbe59a2246700c35d29843`.
|
||||
|
||||
Copyright (C) 2026 Eric Rakestraw
|
||||
|
||||
SPDX-License-Identifier: CC-BY-SA-4.0
|
||||
|
||||
License and notices: [CC BY-SA 4.0](licenses/CC-BY-SA-4.0.txt) |
|
||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/COPYRIGHT) |
|
||||
[LICENSE](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/LICENSE).
|
||||
[LICENSE](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/LICENSE) |
|
||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/policy/src/commit/e833faddf120c789a9b45348edaf75720de16348/COPYRIGHT) |
|
||||
[LICENSE](https://gitea.maximumdirect.net/eric/policy/src/commit/e833faddf120c789a9b45348edaf75720de16348/LICENSE).
|
||||
|
||||
[Source](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843).
|
||||
|
||||
[Source](https://gitea.maximumdirect.net/eric/policy/src/commit/e833faddf120c789a9b45348edaf75720de16348).
|
||||
|
||||
## ConfigKit
|
||||
|
||||
Selected module: `gitea.maximumdirect.net/eric/ConfigKit` at `v0.5.0`.
|
||||
Selected module: `gitea.maximumdirect.net/eric/ConfigKit` at `v0.7.1`.
|
||||
|
||||
Copyright (C) 2026 Eric Rakestraw
|
||||
|
||||
SPDX-License-Identifier: LGPL-3.0-or-later
|
||||
|
||||
License and notices: [LGPL 3.0](../../LICENSE) |
|
||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/ConfigKit/src/commit/dc09959cca4e105f49ec316c6d4ba585bd2340bd/COPYRIGHT) |
|
||||
[LICENSE](https://gitea.maximumdirect.net/eric/ConfigKit/src/commit/dc09959cca4e105f49ec316c6d4ba585bd2340bd/LICENSE).
|
||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/ConfigKit/src/commit/6fffde4af8f31f99d3ee295c426eb3234cb0fb77/COPYRIGHT) |
|
||||
[LICENSE](https://gitea.maximumdirect.net/eric/ConfigKit/src/commit/6fffde4af8f31f99d3ee295c426eb3234cb0fb77/LICENSE).
|
||||
|
||||
[Source](https://gitea.maximumdirect.net/eric/ConfigKit/src/commit/dc09959cca4e105f49ec316c6d4ba585bd2340bd).
|
||||
[Source](https://gitea.maximumdirect.net/eric/ConfigKit/src/commit/6fffde4af8f31f99d3ee295c426eb3234cb0fb77).
|
||||
|
||||
## Promptkit OpenRouter Catalog v3
|
||||
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
# Release Policy
|
||||
|
||||
**Base Policy Version:** 1.4
|
||||
**Base Policy Version:** 1.6
|
||||
|
||||
**Last Updated**: 2026-09-27
|
||||
**Last Updated**: 2026-10-08
|
||||
|
||||
**Upstream Source:** [release.md](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/docs/policy/release.md)
|
||||
**Upstream Source:** [release.md](https://gitea.maximumdirect.net/eric/policy/src/commit/e833faddf120c789a9b45348edaf75720de16348/docs/policy/release.md)
|
||||
|
||||
## Purpose
|
||||
|
||||
@@ -12,6 +12,9 @@ This policy defines a proportionate workflow for validating and publishing an
|
||||
intended source revision under a fresh version. Releases should require little
|
||||
routine intervention, avoid repeated work, and preserve published contents.
|
||||
|
||||
Adopt stronger verification or reproducibility requirements through an explicit
|
||||
project-specific decision justified by the intended audience and support needs.
|
||||
|
||||
The repository-specific commands belong in `docs/release.md`. Historical release
|
||||
summaries belong in `docs/releases/`. The procedure must agree with the adopted
|
||||
policy and link to it rather than introduce additional release requirements.
|
||||
@@ -59,7 +62,8 @@ requires a deliberate update to the policy and procedure where needed.
|
||||
For releases at or after `v1.0.0`:
|
||||
|
||||
- a major release may make intentional incompatible changes;
|
||||
- a minor release may add backward-compatible behavior; and
|
||||
- a minor release may add backward-compatible behavior or raise the Go consumer
|
||||
minimum as described below; and
|
||||
- a patch release may make backward-compatible fixes.
|
||||
|
||||
Before `v1.0.0`, a minor release may change a documented public, operational,
|
||||
@@ -71,14 +75,44 @@ Additional compatibility promises and deliberate exceptions belong in the
|
||||
adopted policy or canonical contract documentation. Release notes explain
|
||||
changes; they do not redefine current compatibility contracts.
|
||||
|
||||
## Release Documentation
|
||||
## Consumer Minimum And Release Toolchain
|
||||
|
||||
Distinguish the minimum toolchain supported for consumers from the toolchain
|
||||
maintainers use to build and validate a release. Record the consumer minimum in
|
||||
the language's standard metadata and explain its support meaning in canonical
|
||||
documentation. Define the selected release toolchain and its setup commands in
|
||||
`docs/release.md`. Toolchain selection and upgrade cadence are repository-specific
|
||||
choices.
|
||||
|
||||
For Go repositories, normally declare the initial stable release of the selected
|
||||
series as the consumer minimum, such as `go 1.26.0`. Use a higher patch minimum
|
||||
when a dependency or an identified correctness fix requires it, and document
|
||||
the reason.
|
||||
|
||||
For release-policy purposes, successful build and release validation on the
|
||||
selected Go patch release are sufficient to presume support across that Go
|
||||
series, subject to the declared consumer minimum and concrete evidence of
|
||||
incompatibility. Investigate specific compatibility concerns as they arise.
|
||||
|
||||
Use the selected release toolchain for candidate validation and release builds,
|
||||
including CI jobs performing those tasks. A newer validation patch version alone
|
||||
leaves the consumer minimum unchanged.
|
||||
|
||||
A minor release may raise the Go consumer minimum, including from one Go series
|
||||
to the next, both before and after `v1.0.0`. This is an explicit exception to the
|
||||
backward-compatibility requirement for minor releases. Patch releases preserve
|
||||
the consumer minimum. Explain an increase's impact and required consumer action
|
||||
in the release note. Other incompatible changes remain subject to the versioning
|
||||
rules above.
|
||||
|
||||
## Release Documentation And Preparation
|
||||
|
||||
Every new release must include `docs/releases/<version>.md` in the tagged
|
||||
source tree. Use the complete version tag as the filename and begin with a
|
||||
title containing the product name and that exact version.
|
||||
|
||||
Include a short summary and any relevant compatibility impact or upgrade
|
||||
instructions. Use the Promptkit section structure below and keep the note
|
||||
instructions. Choose sections suited to the release and keep the note
|
||||
proportionate to the change.
|
||||
|
||||
Keep exact current contracts in their canonical documents and update affected
|
||||
@@ -186,7 +220,8 @@ release without replacing its contents or reusing its version.
|
||||
Maintain `docs/release.md` as a short, executable maintainer runbook. It must
|
||||
identify:
|
||||
|
||||
1. prerequisites, primary branch, and authoritative remote;
|
||||
1. prerequisites, selected release toolchain, primary branch, and authoritative
|
||||
remote;
|
||||
2. version and release-note preparation;
|
||||
3. one publication command with its selected local checks and automated guards;
|
||||
4. declared deliverables, required build targets, and automation responsibilities;
|
||||
@@ -210,20 +245,39 @@ annotated stable SemVer tags. It has no binaries, hosted download pages, or
|
||||
asynchronous deliverables. The source commit is pushed through main before the
|
||||
tag. The [release procedure](../release.md) owns commands and recovery.
|
||||
|
||||
The existing publication script requires Summary, Compatibility, Upgrade, and
|
||||
Changes headings beneath a title of `# Promptkit vMAJOR.MINOR.PATCH`. State
|
||||
“No action required” or “None” where appropriate. This retained local structure
|
||||
matches the automated release-note guard; changing it is separate work.
|
||||
Release preparation uses the latest Go installation available in the development
|
||||
environment to build the maintainer release executable and validate the
|
||||
candidate, including the consumer examples. Keep that same local toolchain
|
||||
active throughout those operations.
|
||||
|
||||
The [publication script](../../scripts/release.sh) runs the
|
||||
[local checks](../../scripts/release-check.sh) once for the final committed
|
||||
candidate: race-enabled tests, vet, three offline consumer examples, tracked
|
||||
Go formatting, Markdown links, module/workspace hygiene, whitespace, and a
|
||||
credential-pattern scan. These protect concurrency, assembled library use,
|
||||
source distribution, and documentation integrity. A separate ordinary test run
|
||||
or standalone build is not an additional release gate. Candidate, tag, and
|
||||
synchronized-main checks remain automated. No CI or module-availability wait
|
||||
is required. Licensing footer URLs are preparation work, not a network gate.
|
||||
For a minor release, advance Promptkit's consumer minimum to the initial stable
|
||||
release of that Go series, retaining a higher patch minimum when required by a
|
||||
dependency or an identified correctness fix. Patch releases preserve the
|
||||
existing consumer minimum. Commit any module update before validating the final
|
||||
candidate. The [release procedure](../release.md#prepare-the-candidate) owns
|
||||
setup commands, and the consumer guide explains
|
||||
[Go compatibility](../consumers/pkg-promptkit.md#go-compatibility).
|
||||
|
||||
Use a title containing `Promptkit` and the exact version, followed by a short
|
||||
summary and any relevant compatibility impact or upgrade instructions. The
|
||||
note is committed with the candidate and supplies the annotated tag message.
|
||||
No fixed set of section headings is required.
|
||||
|
||||
The [publication command](../../scripts/release.sh) delegates to the released
|
||||
ReleaseKit tool selected in `go.mod`. The committed
|
||||
[profile](../../releasekit.json) selects the race-enabled Go suite, full vet,
|
||||
committed Go formatting, local Markdown links, workspace and filesystem
|
||||
replacement inspection, and the offline semantic checks for the three
|
||||
maintained consumer examples. These are the selected checks for the final
|
||||
candidate; a separate ordinary test run or standalone build is not an
|
||||
additional release gate. The upstream tool owns candidate, remote, and tag
|
||||
publication checks. It returns native status 0 for confirmed publication, 1
|
||||
for a known validation or publication failure, 2 for invalid input or profile,
|
||||
and 3 when the push outcome is unknown. For status 3, resolve the exact
|
||||
captured publication endpoint before retrying; the repository-specific
|
||||
[procedure](../release.md#failure-recovery) links to the upstream recovery
|
||||
instructions. Do not wait for CI or Go module availability. Licensing footer
|
||||
URLs are preparation work, not a network gate.
|
||||
|
||||
## Catalog Profile Compatibility
|
||||
|
||||
|
||||
@@ -58,17 +58,24 @@ and repository hygiene.
|
||||
Introducing hosted CI later would supplement, not silently redefine, this
|
||||
documented validation model.
|
||||
|
||||
Full release-candidate validation includes race-enabled package tests, explicit
|
||||
static analysis, and execution with automated semantic checks of the three maintained
|
||||
offline consumer examples. The test run builds the packages; the examples
|
||||
also exercise ordinary compilation. Separate ordinary tests and a standalone
|
||||
build are not additional release gates. Ordinary tests remain useful during
|
||||
development. The preparation example protects assembled preparation and
|
||||
The selected ReleaseKit profile runs the race-enabled package suite and full
|
||||
vet, checks committed Go formatting, local Markdown links, and module
|
||||
distribution hygiene, and executes semantic assertions for the three
|
||||
maintained offline consumer examples. ReleaseKit owns this shared validation
|
||||
machinery and publication workflow. The test run builds the packages; the
|
||||
examples also exercise ordinary compilation. Separate ordinary tests and a
|
||||
standalone build are not additional release gates. Ordinary tests remain useful
|
||||
during development. The preparation example protects assembled preparation and
|
||||
inspection behavior. The execution example separately protects assembled
|
||||
`Run`, injected-client, validation, usage, and result behavior. The observation
|
||||
example protects assembled built-in retries, structured proposal and assessment
|
||||
runs, and application-owned interpretation of metadata.
|
||||
|
||||
The standalone `python3 scripts/test-release.py` command exercises Promptkit's
|
||||
ReleaseKit wiring against disposable local repositories. It is focused
|
||||
development validation when release tooling changes, not part of the default
|
||||
Go test suite or a recurring release check.
|
||||
|
||||
Tests in the default suite must be deterministic, offline, and independent of
|
||||
real credentials. They must not invoke paid APIs, use live network
|
||||
dependencies, or depend on mutable external services. Tests that require live
|
||||
|
||||
157
docs/release.md
157
docs/release.md
@@ -1,64 +1,101 @@
|
||||
# Release Procedure
|
||||
|
||||
This runbook implements the [release policy](policy/release.md). A successful
|
||||
push of the selected annotated tag to origin completes the release. Never wait
|
||||
for CI to finish or succeed.
|
||||
This runbook implements the [release policy](policy/release.md). Promptkit
|
||||
publishes source-only Go module tags from `main` to `origin`. A successful push
|
||||
of the selected annotated tag, after its source commit is available on `main`,
|
||||
completes the release. Do not wait for CI or Go module availability.
|
||||
|
||||
## Prepare The Candidate
|
||||
|
||||
Required tools are Git, Go with race-detector support, POSIX shell, and Python
|
||||
3. Publication requires push access to origin. Work from the repository root.
|
||||
3. Work from the repository root with push access to `origin`.
|
||||
|
||||
Select an unused version and create `docs/releases/<version>.md` using the
|
||||
[required structure](policy/release.md#release-documentation). Describe
|
||||
compatibility impact and consumer actions and update affected canonical docs.
|
||||
For catalog updates, record selected catalog versions and any consumer action,
|
||||
linking to the catalog policy rather than repeating its contracts.
|
||||
Use the latest Go installation available locally for the release tool, the
|
||||
candidate validation, and the maintained examples. Make it the active `go`
|
||||
command and record its version:
|
||||
|
||||
Follow the [OSS policy](policy/oss.md) when preparing licensing and attribution
|
||||
changes. Each new note must end with the following navigation, replacing
|
||||
`vMAJOR.MINOR.PATCH` with the selected release tag:
|
||||
|
||||
```markdown
|
||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit/src/tag/vMAJOR.MINOR.PATCH/COPYRIGHT) | [LICENSE](https://gitea.maximumdirect.net/eric/promptkit/src/tag/vMAJOR.MINOR.PATCH/LICENSE) | [ATTRIBUTION](https://gitea.maximumdirect.net/eric/promptkit/src/tag/vMAJOR.MINOR.PATCH/docs/oss/attribution.md)
|
||||
```sh
|
||||
export GOTOOLCHAIN=local
|
||||
go version
|
||||
```
|
||||
|
||||
Check these file paths against the candidate; a future tag need not resolve
|
||||
remotely during preparation. The annotated tag derives from this same note,
|
||||
including its footer. Promptkit publishes no separate hosted release body or
|
||||
download page. If such a channel is deliberately introduced, reuse the note
|
||||
and preserve its footer. No extra network checks, artifacts, or approval gates
|
||||
are introduced, and published notes are not retrofitted.
|
||||
For a minor release, advance Promptkit's consumer minimum to the initial stable
|
||||
release of that Go series before committing the candidate:
|
||||
|
||||
Retire completed roadmaps, preserving planned or deferred work and repairing
|
||||
links. This is housekeeping, not a separate approval gate. Review changes
|
||||
through the ordinary development workflow, commit them, and push main to
|
||||
origin. No repeated release-specific diff review is required.
|
||||
```sh
|
||||
release_go_version=$(go env GOVERSION)
|
||||
release_go_series=${release_go_version#go}
|
||||
release_go_series=${release_go_series%.*}
|
||||
go mod edit "-go=${release_go_series}.0"
|
||||
```
|
||||
|
||||
For example, Go 1.26.8 normally results in `go 1.26.0`. Keep a higher patch
|
||||
minimum if a dependency or identified correctness fix requires it. For a patch
|
||||
release, leave the consumer minimum unchanged; using a newer validation
|
||||
toolchain does not itself raise that minimum. If the candidate needs a higher
|
||||
minimum, prepare a minor release instead. See the
|
||||
[consumer-minimum policy](policy/release.md#consumer-minimum-and-release-toolchain).
|
||||
|
||||
Prepare module caches before validation; this may update `go.sum` and belongs
|
||||
before the candidate commit:
|
||||
|
||||
```sh
|
||||
GOTOOLCHAIN=local go mod download
|
||||
```
|
||||
|
||||
Choose an unused stable version and create `docs/releases/<version>.md` with a
|
||||
title containing `Promptkit` and that exact version, a short summary, and any
|
||||
relevant compatibility impact or upgrade instructions. No fixed section
|
||||
headings are required. Describe a raised Go minimum and its compatibility
|
||||
impact under the [versioning policy](policy/release.md#versioning-and-compatibility).
|
||||
End each new note with the licensing footer below, using the selected version:
|
||||
|
||||
```markdown
|
||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit/src/tag/vX.Y.Z/COPYRIGHT) | [LICENSE](https://gitea.maximumdirect.net/eric/promptkit/src/tag/vX.Y.Z/LICENSE) | [ATTRIBUTION](https://gitea.maximumdirect.net/eric/promptkit/src/tag/vX.Y.Z/docs/oss/attribution.md)
|
||||
```
|
||||
|
||||
Check the footer paths against the candidate; the future tag need not resolve
|
||||
remotely during preparation. Follow the [OSS policy](policy/oss.md) for these
|
||||
links. Promptkit publishes no binaries, hosted download page, or other
|
||||
asynchronous deliverables.
|
||||
|
||||
Retire completed roadmaps while preserving deferred work and repairing incoming
|
||||
links. Review the intended changes through the ordinary development workflow,
|
||||
commit the candidate on `main`, and push `main` to `origin` before publication.
|
||||
Preparation may update module metadata; the release bootstrap and validation
|
||||
preserve the committed candidate.
|
||||
|
||||
## Validate And Publish
|
||||
|
||||
Run the following with the chosen unused version (replace the placeholder):
|
||||
For an optional local diagnostic of a committed candidate, run:
|
||||
|
||||
```sh
|
||||
./scripts/releasekit.sh check vMAJOR.MINOR.PATCH
|
||||
```
|
||||
|
||||
This is not a required step before publication. The upstream
|
||||
[candidate-check contract](https://gitea.maximumdirect.net/eric/ReleaseKit/src/tag/v0.4.1/docs/consumers/check.md)
|
||||
describes its requirements.
|
||||
|
||||
Run the single publication command with the chosen version:
|
||||
|
||||
```sh
|
||||
./scripts/release.sh vMAJOR.MINOR.PATCH
|
||||
```
|
||||
|
||||
The [publication script](../scripts/release.sh) checks a clean, synchronized
|
||||
main candidate, the version and unused tag, and the release note's title and
|
||||
required headings. It runs the [local checks](../scripts/release-check.sh)
|
||||
once, rechecks candidate identity and publication state, generates the
|
||||
annotation from the committed note plus the validated commit, and pushes only
|
||||
the selected annotated tag.
|
||||
The familiar wrapper delegates to the native ReleaseKit executable selected in
|
||||
`go.mod`. The committed [profile](../releasekit.json) selects the race-enabled
|
||||
Go suite, full vet, committed Go formatting, local Markdown links, workspace
|
||||
and filesystem replacement inspection, and semantic checks for the three
|
||||
maintained offline consumer examples. ReleaseKit also verifies the committed
|
||||
candidate and publication state, then publishes only the selected tag. Its
|
||||
checks run once for this candidate; do not repeat them as a separate sign-off.
|
||||
|
||||
There is no separate tag-message authoring, example-output inspection,
|
||||
metadata inspection, or human sign-off. Compatibility and release-note content
|
||||
remain ordinary editorial responsibilities. Development validation is useful
|
||||
but does not require a second full run before invoking the publication script.
|
||||
|
||||
The script stops successfully after the tag push. CI completion, CI success,
|
||||
remote read-back, and Go module availability are not release completion gates.
|
||||
Do not wait or poll for them. Provider probes and inference calls are never
|
||||
part of release validation.
|
||||
The native command returns status 0 for confirmed publication, 1 for a known
|
||||
validation or publication failure, 2 for invalid input or profile, and 3 when
|
||||
the push outcome remains unknown. Successful publication needs no remote
|
||||
read-back, CI wait, or installation check. Provider probes and inference calls
|
||||
are not part of validation.
|
||||
|
||||
## Catalog Profile Compatibility
|
||||
|
||||
@@ -67,28 +104,32 @@ catalog compatibility boundary. This heading preserves older incoming links.
|
||||
|
||||
## Failure Recovery
|
||||
|
||||
If local checks fail, fix the candidate, commit and push changes, then rerun
|
||||
the publication command. It never automatically removes a tag.
|
||||
For status 1, use the diagnostic to distinguish a validation failure from a
|
||||
known publication conflict. Correct and commit candidate defects, push `main`,
|
||||
and rerun publication with an unused version when appropriate. Status 2 means
|
||||
the invocation or profile must be corrected.
|
||||
|
||||
If a push fails or its result is ambiguous, set `RELEASE_VERSION` to the
|
||||
attempted version and inspect origin before retrying:
|
||||
For status 3, do not retry or remove the local tag until the remote outcome is
|
||||
resolved. ReleaseKit reports the attempted version, tag object, and recovery
|
||||
state, including the captured publication endpoint when available. Use that
|
||||
endpoint—not merely the current `origin` URL—to query the exact tag; follow the
|
||||
upstream
|
||||
[failure-recovery procedure](https://gitea.maximumdirect.net/eric/ReleaseKit/src/tag/v0.4.1/docs/release.md#resolve-failures):
|
||||
|
||||
```sh
|
||||
git ls-remote --tags origin "refs/tags/$RELEASE_VERSION" "refs/tags/$RELEASE_VERSION^{}"
|
||||
git rev-parse "refs/tags/$RELEASE_VERSION" "refs/tags/$RELEASE_VERSION^{}"
|
||||
releasekit_endpoint='COPY_CAPTURED_ENDPOINT'
|
||||
releasekit_version='vX.Y.Z'
|
||||
test "$(git ls-remote --get-url -- "$releasekit_endpoint")" = "$releasekit_endpoint" &&
|
||||
git ls-remote --refs -- "$releasekit_endpoint" "refs/tags/$releasekit_version"
|
||||
```
|
||||
|
||||
Matching remote tag and peeled commit values confirm publication, even if the
|
||||
push command reported an error. If the remote query fails, publication status
|
||||
is still unknown. Do not delete or recreate the local tag until absence on
|
||||
origin is confirmed. Once confirmed unpublished, an unsuccessful attempt's
|
||||
local tag can be removed and the publication command rerun. Never overwrite
|
||||
a published tag.
|
||||
|
||||
CI or module-resolution problems may be investigated separately; temporary
|
||||
service, network, or authentication failures do not require another release.
|
||||
Only an actual defect requiring changes to published contents warrants a new
|
||||
appropriate version. See the [recovery policy](policy/release.md#immutability-and-failure-recovery).
|
||||
Compare the returned object ID with the reported tag object. An exact match
|
||||
confirms publication. A missing ref confirms absence only when the endpoint
|
||||
identity check and query both succeed. If either check fails, the outcome
|
||||
remains unknown; correct the transport configuration and resolve it before
|
||||
retrying. Only after authoritative absence is confirmed may an unpublished
|
||||
local tag be removed before correcting the candidate or retrying. Never move,
|
||||
overwrite, or delete a published tag.
|
||||
|
||||
## Policy Changes
|
||||
|
||||
|
||||
69
docs/releases/v0.13.0.md
Normal file
69
docs/releases/v0.13.0.md
Normal file
@@ -0,0 +1,69 @@
|
||||
# Promptkit v0.13.0
|
||||
|
||||
This release adds prompt requirement discovery and profile/prompt listing,
|
||||
reduces avoidable configuration and execution allocations, and standardizes
|
||||
maintainer publication through ReleaseKit. The minimum supported Go version
|
||||
increases to 1.26.0.
|
||||
|
||||
## Discover Requirements And Available Definitions
|
||||
|
||||
- `Engine.InspectRequirements` reports recognizable input and caller-variable
|
||||
dependencies for a selected prompt and supplied caller context before input
|
||||
artifacts exist. It classifies dependencies as required, conditional, or
|
||||
optional and reports structured limitations when discovery is incomplete.
|
||||
Discovery coverage is not proof that requirements are satisfied or execution
|
||||
will succeed. See the [requirement inspection guide](../consumers/pkg-promptkit.md#inspect-requirements-before-preparation)
|
||||
and [GoDoc](../../requirements.go).
|
||||
- `Engine.ListProfiles` returns effective profile IDs, and `Engine.ListPrompts`
|
||||
returns exact ID/version pairs as `PromptIdentity` values. Lists come from the
|
||||
engine's captured configuration, are lexically ordered and caller-owned, and
|
||||
include consumer definitions and maintained profiles that remain effective.
|
||||
Listing performs no provider or execution work. See the
|
||||
[discovery guide](../consumers/pkg-promptkit.md#discover-available-profiles-and-prompts)
|
||||
and [GoDoc](../../listing.go).
|
||||
|
||||
## Efficiency And Tooling
|
||||
|
||||
- Reduce response-sized copies in basic validation and rendered-message output,
|
||||
and encode valid provider extra parameters only as part of the complete
|
||||
request rather than serializing them twice.
|
||||
- Use bounded initial file-size reservations without limiting accepted content,
|
||||
releasing excess backing storage for small or empty completed artifacts.
|
||||
- Reuse owned preparation values when creating prepared-execution snapshots,
|
||||
while preserving independent inspection details and caller-owned data.
|
||||
- Use lightweight captured profile identities and prompt metadata for inspection
|
||||
instead of copying execution targets and definitions unnecessarily.
|
||||
- Upgrade ConfigKit to v0.6.0 for more efficient loading, resolution, metadata,
|
||||
and inspection processing. Configuration semantics remain unchanged; these
|
||||
improvements primarily benefit construction and explicit operator inspection,
|
||||
not upstream inference latency.
|
||||
- Adopt module-selected ReleaseKit v0.3.0 for maintainer validation and tag
|
||||
publication. The [release procedure](../release.md) owns the current commands
|
||||
and recovery workflow; successful tag publication remains the completion
|
||||
point, without waiting for CI.
|
||||
|
||||
These changes preserve existing execution and ownership contracts. Benefits
|
||||
depend on workload; this release introduces no fixed performance guarantee.
|
||||
|
||||
## Compatibility And Upgrade
|
||||
|
||||
Consumers upgrading from v0.12.0 need **Go 1.26.0 or newer**, rather than the
|
||||
previous 1.25.5 minimum. Update application build environments and any CI images
|
||||
still using Go 1.25 before upgrading:
|
||||
|
||||
```sh
|
||||
go get gitea.maximumdirect.net/eric/promptkit@v0.13.0
|
||||
```
|
||||
|
||||
No application-code or configuration migration is required from v0.12.0 beyond
|
||||
the Go toolchain update. The new discovery methods are optional additions;
|
||||
existing construction, inspection, preparation, and execution workflows remain
|
||||
available. See [Go compatibility](../consumers/pkg-promptkit.md#go-compatibility)
|
||||
for the distinction between the consumer minimum and maintainer toolchain.
|
||||
|
||||
Maintainers should follow the current release runbook rather than the retired
|
||||
`scripts/release-check.sh`. The publication entry point remains
|
||||
`scripts/release.sh`; ReleaseKit now owns its shared validation and publication
|
||||
mechanics.
|
||||
|
||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.13.0/COPYRIGHT) | [LICENSE](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.13.0/LICENSE) | [ATTRIBUTION](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.13.0/docs/oss/attribution.md)
|
||||
48
docs/releases/v0.14.0.md
Normal file
48
docs/releases/v0.14.0.md
Normal file
@@ -0,0 +1,48 @@
|
||||
# Promptkit v0.14.0
|
||||
|
||||
This release adds comparison of captured configuration inspections and updates
|
||||
ConfigKit and the maintainer release tool. The minimum supported Go version
|
||||
remains 1.26.0.
|
||||
|
||||
## Compare Captured Configurations
|
||||
|
||||
`ConfigurationInspection.Diff` compares the effective configuration captured by
|
||||
two engines. Consumers can use it to review a candidate configuration, inspect
|
||||
an overlay's effect, or present configuration changes to operators.
|
||||
|
||||
Comparison delegates to ConfigKit and returns its difference records, including
|
||||
presence information, paired redaction, and protected-change indicators. It
|
||||
does not reload sources or contact providers. Source-history differences alone
|
||||
do not produce value differences.
|
||||
|
||||
An empty diff is not proof of equivalent execution: referenced file contents,
|
||||
resolved credentials, later execution preparation, and request-specific
|
||||
overrides are outside the comparison. Records can contain ordinary application
|
||||
data; callers retain responsibility for their disclosure.
|
||||
|
||||
See the [configuration comparison guide](../consumers/pkg-promptkit.md#inspect-the-captured-configuration)
|
||||
and [GoDoc](../../configuration_inspection.go) for usage and the exact contract.
|
||||
|
||||
## Dependency And Tooling Updates
|
||||
|
||||
- Upgrade ConfigKit from v0.6.0 to v0.7.1, providing the upstream comparison API.
|
||||
v0.7.1 also updates ConfigKit's maintainer tooling without changing its runtime
|
||||
behavior from v0.7.0.
|
||||
- Upgrade the module-selected ReleaseKit tool from v0.3.0 to v0.4.1. Promptkit
|
||||
retains its existing root-module profile, publication entry point, and release
|
||||
gates. See the [release procedure](../release.md) for maintainer commands.
|
||||
|
||||
## Compatibility And Upgrade
|
||||
|
||||
No application-code, configuration, or Go-toolchain migration is required from
|
||||
v0.13.0. Configuration comparison is an optional API addition; existing
|
||||
construction, inspection, preparation, and execution workflows remain available.
|
||||
|
||||
```sh
|
||||
go get gitea.maximumdirect.net/eric/promptkit@v0.14.0
|
||||
```
|
||||
|
||||
See [Go compatibility](../consumers/pkg-promptkit.md#go-compatibility) for the
|
||||
distinction between the consumer minimum and maintainer toolchain.
|
||||
|
||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.14.0/COPYRIGHT) | [LICENSE](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.14.0/LICENSE) | [ATTRIBUTION](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.14.0/docs/oss/attribution.md)
|
||||
@@ -82,12 +82,12 @@ var (
|
||||
ErrValidation = errors.New("failed to validate output")
|
||||
)
|
||||
|
||||
// Engine inspects prompts and profiles and prepares and runs Promptkit prompt
|
||||
// requests.
|
||||
// Engine inspects prompts and profiles, enumerates prompt requirements, and
|
||||
// prepares and runs Promptkit prompt requests.
|
||||
//
|
||||
// An Engine is safe for concurrent calls to [Engine.InspectConfiguration],
|
||||
// [Engine.InspectPrompt],
|
||||
// [Engine.InspectProfile], [Engine.Prepare], [Engine.PrepareExecution],
|
||||
// [Engine.InspectRequirements], [Engine.InspectProfile], [Engine.Prepare], [Engine.PrepareExecution],
|
||||
// [Engine.Run], and [Engine.RunPrepared]. Each Engine owns independent
|
||||
// backend-capacity pools that coordinate Run and RunPrepared admission and
|
||||
// model generation. Injected collaborators may still be invoked concurrently
|
||||
|
||||
@@ -36,6 +36,17 @@ func main() {
|
||||
if _, found := view.Explain(configkit.Keys("profiles", "offline-example", "model")); !found {
|
||||
exit(fmt.Errorf("configured profile model was not found"))
|
||||
}
|
||||
requirements, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{
|
||||
PromptID: "example.prepare",
|
||||
})
|
||||
if err != nil {
|
||||
exit(err)
|
||||
}
|
||||
if !requirements.Complete || len(requirements.Inputs) != 1 ||
|
||||
requirements.Inputs[0].Name != "note" ||
|
||||
requirements.Inputs[0].Kind != promptkit.RequirementRequired {
|
||||
exit(fmt.Errorf("required note input was not discovered before preparation"))
|
||||
}
|
||||
|
||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||
PromptID: "example.prepare",
|
||||
|
||||
9
go.mod
9
go.mod
@@ -1,15 +1,20 @@
|
||||
module gitea.maximumdirect.net/eric/promptkit
|
||||
|
||||
go 1.25.5
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
gitea.maximumdirect.net/eric/ConfigKit v0.5.0
|
||||
gitea.maximumdirect.net/eric/ConfigKit v0.7.1
|
||||
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v3 v3.20260930.0
|
||||
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v3 v3.20260930.0
|
||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2
|
||||
)
|
||||
|
||||
require (
|
||||
gitea.maximumdirect.net/eric/ReleaseKit v0.4.1 // indirect
|
||||
github.com/yuin/goldmark/v2 v2.1.6 // indirect
|
||||
go.yaml.in/yaml/v4 v4.0.0-rc.6 // indirect
|
||||
golang.org/x/mod v0.40.0 // indirect
|
||||
golang.org/x/text v0.14.0 // indirect
|
||||
)
|
||||
|
||||
tool gitea.maximumdirect.net/eric/ReleaseKit/cmd/releasekit
|
||||
|
||||
10
go.sum
10
go.sum
@@ -1,5 +1,7 @@
|
||||
gitea.maximumdirect.net/eric/ConfigKit v0.5.0 h1:2KwG10+kApWM3avOoYA+p768xcbHUp7tQ2rqizBNpyg=
|
||||
gitea.maximumdirect.net/eric/ConfigKit v0.5.0/go.mod h1:tYWpsrbLyu05r8+1MapWp5fN6ZrX9I8hU2xEPedYmYk=
|
||||
gitea.maximumdirect.net/eric/ConfigKit v0.7.1 h1:q3HrX4WWCPt1WfMydqFEaAmCf9s6Dq+vknJj0C/mdcI=
|
||||
gitea.maximumdirect.net/eric/ConfigKit v0.7.1/go.mod h1:C82sbyKHYMV1OwaD2GJanFrwTfcZfjlW2JsGze5b0kA=
|
||||
gitea.maximumdirect.net/eric/ReleaseKit v0.4.1 h1:Qn+mdebYOzCVCEXgCbuGSO/WHmJtItwaQ8e3XwY3qtI=
|
||||
gitea.maximumdirect.net/eric/ReleaseKit v0.4.1/go.mod h1:LVodqNQ3OIKv9NBq+fFeyMByd1lHY25YOmmpah/NaNI=
|
||||
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v3 v3.20260930.0 h1:lzfCDbzJQRGzIlRPRprnAV9hVVL5zUQy80VMQ0kbYa0=
|
||||
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v3 v3.20260930.0/go.mod h1:HjS0f+3q0IdHCTEz+VVAEgIwIWJFAZUjuKdAtvtqKdg=
|
||||
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v3 v3.20260930.0 h1:h12ZTRldAmpLAomj0oQRGteFxHLN1LPE6XDl9zJNbm8=
|
||||
@@ -8,8 +10,12 @@ github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxK
|
||||
github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
|
||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ=
|
||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU=
|
||||
github.com/yuin/goldmark/v2 v2.1.6 h1:igy0q9JcqovoFXFx/XjtU8sHVhcUHavPj2sA3EoKJUE=
|
||||
github.com/yuin/goldmark/v2 v2.1.6/go.mod h1:G6M4/qOFtfn01/o14BU1UR2Lo5N3S9Qo7xCuz/sHjGQ=
|
||||
go.yaml.in/yaml/v4 v4.0.0-rc.6 h1:1h7H1ohdUh93/FyE4YaDa1Zh64K6VVbjF4K6WUxMtH4=
|
||||
go.yaml.in/yaml/v4 v4.0.0-rc.6/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0=
|
||||
golang.org/x/mod v0.40.0 h1:hUv+3cXcdRHz08UmSiOob7sadHig73uo5bkXxQ/tvUs=
|
||||
golang.org/x/mod v0.40.0/go.mod h1:0/weTWkPWGBikyTWAX3dkjVztMmBA5hM0DH6BElSupE=
|
||||
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
|
||||
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
|
||||
@@ -21,6 +21,7 @@ var (
|
||||
)
|
||||
|
||||
const fileReadChunkSize = 64 * 1024
|
||||
const fileReadInitialCapacityLimit = 1 << 20
|
||||
|
||||
// Reader resolves artifact references into actual artifacts.
|
||||
type Reader interface {
|
||||
@@ -133,7 +134,11 @@ func readFileArtifact(ctx context.Context, path string, open func(string) (artif
|
||||
return nil, fmt.Errorf("%w: %s", ErrUnsupportedFile, path)
|
||||
}
|
||||
|
||||
data := make([]byte, 0)
|
||||
capacity := 0
|
||||
if size := openedInfo.Size(); size > 0 {
|
||||
capacity = int(min(size, int64(fileReadInitialCapacityLimit)))
|
||||
}
|
||||
data := make([]byte, 0, capacity)
|
||||
chunk := make([]byte, fileReadChunkSize)
|
||||
for {
|
||||
if err := ctx.Err(); err != nil {
|
||||
@@ -153,6 +158,13 @@ func readFileArtifact(ctx context.Context, path string, open func(string) (artif
|
||||
return nil, fmt.Errorf("failed to read file %s: %w", path, readErr)
|
||||
}
|
||||
}
|
||||
if cap(data)-len(data) > len(data) {
|
||||
// A stale size hint should not make a small completed artifact retain a
|
||||
// disproportionately large backing array.
|
||||
compact := make([]byte, len(data))
|
||||
copy(compact, data)
|
||||
data = compact
|
||||
}
|
||||
|
||||
contentType := mime.TypeByExtension(filepath.Ext(path))
|
||||
if contentType == "" {
|
||||
|
||||
180
internal/artifact/reader_benchmark_test.go
Normal file
180
internal/artifact/reader_benchmark_test.go
Normal file
@@ -0,0 +1,180 @@
|
||||
package artifact
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"fmt"
|
||||
"mime"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
)
|
||||
|
||||
var artifactFileReadBenchmarkSink *domain.Artifact
|
||||
|
||||
func BenchmarkArtifactFileRead(b *testing.B) {
|
||||
for _, fixture := range makeArtifactReadBenchmarkFiles(b) {
|
||||
b.Run(fixture.name, func(b *testing.B) {
|
||||
reader := &fileReader{open: openArtifactFile}
|
||||
ref := domain.ArtifactRef{Type: domain.ArtifactRefFile, URI: fixture.path}
|
||||
checkArtifactFileRead(b, reader, ref, fixture.body)
|
||||
|
||||
b.ReportAllocs()
|
||||
b.SetBytes(int64(len(fixture.body)))
|
||||
b.ResetTimer()
|
||||
for range b.N {
|
||||
artifact, err := reader.Read(context.Background(), ref)
|
||||
if err != nil {
|
||||
b.Fatalf("read artifact: %v", err)
|
||||
}
|
||||
artifactFileReadBenchmarkSink = artifact
|
||||
}
|
||||
b.StopTimer()
|
||||
runtime.KeepAlive(artifactFileReadBenchmarkSink)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// BenchmarkRetainedArtifactFileRead measures live artifact bodies when an
|
||||
// opened file reports a size larger than the content that reaches EOF.
|
||||
func BenchmarkRetainedArtifactFileRead(b *testing.B) {
|
||||
for _, fixture := range []struct {
|
||||
name string
|
||||
body []byte
|
||||
}{
|
||||
{name: "small", body: bytes.Repeat([]byte("x"), 256<<10)},
|
||||
{name: "empty", body: []byte{}},
|
||||
} {
|
||||
b.Run(fixture.name, func(b *testing.B) {
|
||||
benchmarkRetainedArtifactFileRead(b, fixture.body)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func benchmarkRetainedArtifactFileRead(b *testing.B, body []byte) {
|
||||
path := filepath.Join(b.TempDir(), "overreported.txt")
|
||||
if err := os.WriteFile(path, body, 0o600); err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
mime.TypeByExtension(filepath.Ext(path))
|
||||
|
||||
reader := &fileReader{open: func(path string) (artifactFile, error) {
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &sizeReportingArtifactFile{artifactFile: file, reportedSize: 16 << 20}, nil
|
||||
}}
|
||||
ref := domain.ArtifactRef{Type: domain.ArtifactRefFile, URI: path}
|
||||
checkArtifactFileRead(b, reader, ref, body)
|
||||
|
||||
const retainedBatchSize = 4
|
||||
retained := make([]*domain.Artifact, retainedBatchSize)
|
||||
artifactFileReadBenchmarkSink = nil
|
||||
runtime.GC()
|
||||
var setupMemory runtime.MemStats
|
||||
runtime.ReadMemStats(&setupMemory)
|
||||
runtime.KeepAlive(reader)
|
||||
runtime.KeepAlive(ref)
|
||||
|
||||
b.ReportAllocs()
|
||||
b.SetBytes(int64(len(body)))
|
||||
b.ResetTimer()
|
||||
for iteration := 0; iteration < b.N; iteration++ {
|
||||
artifact, err := reader.Read(context.Background(), ref)
|
||||
if err != nil {
|
||||
b.Fatalf("read retained artifact: %v", err)
|
||||
}
|
||||
retained[iteration%len(retained)] = artifact
|
||||
}
|
||||
b.StopTimer()
|
||||
for i, artifact := range retained {
|
||||
if artifact != nil {
|
||||
continue
|
||||
}
|
||||
artifact, err := reader.Read(context.Background(), ref)
|
||||
if err != nil {
|
||||
b.Fatalf("fill retained artifact batch: %v", err)
|
||||
}
|
||||
retained[i] = artifact
|
||||
}
|
||||
runtime.GC()
|
||||
var retainedMemory runtime.MemStats
|
||||
runtime.ReadMemStats(&retainedMemory)
|
||||
b.ReportMetric(float64(int64(retainedMemory.HeapAlloc)-int64(setupMemory.HeapAlloc)), "retained-B")
|
||||
runtime.KeepAlive(retained)
|
||||
runtime.KeepAlive(reader)
|
||||
runtime.KeepAlive(ref)
|
||||
}
|
||||
|
||||
type artifactReadBenchmarkFile struct {
|
||||
name string
|
||||
path string
|
||||
body []byte
|
||||
}
|
||||
|
||||
func makeArtifactReadBenchmarkFiles(b *testing.B) []artifactReadBenchmarkFile {
|
||||
b.Helper()
|
||||
fixtures := []struct {
|
||||
name string
|
||||
size int
|
||||
}{
|
||||
{name: "small_4KiB", size: 4 << 10},
|
||||
{name: "1MiB", size: 1 << 20},
|
||||
{name: "large_4MiB", size: 4 << 20},
|
||||
}
|
||||
results := make([]artifactReadBenchmarkFile, 0, len(fixtures))
|
||||
for _, fixture := range fixtures {
|
||||
body := bytes.Repeat([]byte("x"), fixture.size)
|
||||
path := filepath.Join(b.TempDir(), fixture.name+".txt")
|
||||
if err := os.WriteFile(path, body, 0o600); err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
mime.TypeByExtension(filepath.Ext(path))
|
||||
results = append(results, artifactReadBenchmarkFile{
|
||||
name: fixture.name,
|
||||
path: path,
|
||||
body: body,
|
||||
})
|
||||
}
|
||||
return results
|
||||
}
|
||||
|
||||
func checkArtifactFileRead(b *testing.B, reader Reader, ref domain.ArtifactRef, want []byte) {
|
||||
b.Helper()
|
||||
artifact, err := reader.Read(context.Background(), ref)
|
||||
if err != nil {
|
||||
b.Fatalf("check artifact read: %v", err)
|
||||
}
|
||||
if !bytes.Equal(artifact.Body, want) || artifact.Size != int64(len(want)) {
|
||||
b.Fatalf("artifact size=%d body length=%d, want %d", artifact.Size, len(artifact.Body), len(want))
|
||||
}
|
||||
wantHash := fmt.Sprintf("%x", sha256.Sum256(want))
|
||||
if artifact.Hash != wantHash {
|
||||
b.Fatalf("artifact hash=%q, want %q", artifact.Hash, wantHash)
|
||||
}
|
||||
}
|
||||
|
||||
type sizeReportingArtifactFile struct {
|
||||
artifactFile
|
||||
reportedSize int64
|
||||
}
|
||||
|
||||
func (file *sizeReportingArtifactFile) Stat() (os.FileInfo, error) {
|
||||
info, err := file.artifactFile.Stat()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return reportedArtifactFileInfo{FileInfo: info, size: file.reportedSize}, nil
|
||||
}
|
||||
|
||||
type reportedArtifactFileInfo struct {
|
||||
os.FileInfo
|
||||
size int64
|
||||
}
|
||||
|
||||
func (info reportedArtifactFileInfo) Size() int64 { return info.size }
|
||||
@@ -3,7 +3,9 @@ package artifact
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
@@ -232,6 +234,64 @@ func TestFileReaderFailuresAndMetadata(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
func TestFileReaderUsesOpenedSizeOnlyAsACapacityHint(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
body []byte
|
||||
reportedSize int64
|
||||
}{
|
||||
{name: "overreported size", body: []byte("actual file content"), reportedSize: 128},
|
||||
{name: "overreported size with empty content", body: []byte{}, reportedSize: 16 << 20},
|
||||
{name: "underreported size", body: bytes.Repeat([]byte("x"), fileReadChunkSize+17), reportedSize: 1},
|
||||
{name: "very large hint with small content", body: []byte("small actual content"), reportedSize: int64(^uint64(0) >> 1)},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
filePath := filepath.Join(t.TempDir(), "artifact.txt")
|
||||
if err := os.WriteFile(filePath, tc.body, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reader := &fileReader{open: func(path string) (artifactFile, error) {
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
info, err := file.Stat()
|
||||
if err != nil {
|
||||
file.Close()
|
||||
return nil, err
|
||||
}
|
||||
return &reportedInfoFile{
|
||||
artifactFile: file,
|
||||
info: reportedFileSizeInfo{FileInfo: info, size: tc.reportedSize},
|
||||
}, nil
|
||||
}}
|
||||
|
||||
artifact, err := reader.Read(context.Background(), domain.ArtifactRef{
|
||||
Type: domain.ArtifactRefFile,
|
||||
URI: filePath,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("read artifact: %v", err)
|
||||
}
|
||||
if !bytes.Equal(artifact.Body, tc.body) {
|
||||
t.Fatalf("artifact body length=%d, want %d", len(artifact.Body), len(tc.body))
|
||||
}
|
||||
if artifact.Size != int64(len(tc.body)) {
|
||||
t.Fatalf("artifact size=%d, want %d", artifact.Size, len(tc.body))
|
||||
}
|
||||
if artifact.Name != filepath.Base(filePath) || artifact.URI != filePath || artifact.ContentType == "" {
|
||||
t.Fatalf("unexpected artifact metadata: name=%q URI=%q content type=%q", artifact.Name, artifact.URI, artifact.ContentType)
|
||||
}
|
||||
wantHash := fmt.Sprintf("%x", sha256.Sum256(tc.body))
|
||||
if artifact.Hash != wantHash {
|
||||
t.Fatalf("artifact hash=%q, want %q", artifact.Hash, wantHash)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileReaderCancelsAfterReadProgress(t *testing.T) {
|
||||
filePath := filepath.Join(t.TempDir(), "artifact.bin")
|
||||
content := bytes.Repeat([]byte("x"), fileReadChunkSize*2)
|
||||
@@ -268,6 +328,13 @@ func (f *reportedInfoFile) Stat() (os.FileInfo, error) {
|
||||
return f.info, nil
|
||||
}
|
||||
|
||||
type reportedFileSizeInfo struct {
|
||||
os.FileInfo
|
||||
size int64
|
||||
}
|
||||
|
||||
func (info reportedFileSizeInfo) Size() int64 { return info.size }
|
||||
|
||||
type cancelAfterProgressFile struct {
|
||||
artifactFile
|
||||
cancel context.CancelFunc
|
||||
|
||||
@@ -13,9 +13,12 @@ import (
|
||||
const benchmarkDefinitionCount = 128
|
||||
|
||||
var (
|
||||
constructionResult *loaded
|
||||
selectedPrompt *domain.PromptDefinition
|
||||
selectedProfile *domain.ExecutionProfile
|
||||
constructionResult *loaded
|
||||
selectedPrompt *domain.PromptDefinition
|
||||
selectedPromptHash string
|
||||
selectedPromptInspection *domain.PromptInspection
|
||||
selectedProfile *domain.ExecutionProfile
|
||||
selectedProfileIdentity string
|
||||
)
|
||||
|
||||
type configurationShape struct {
|
||||
@@ -177,3 +180,116 @@ func BenchmarkConfigurationSelection(b *testing.B) {
|
||||
b.ReportMetric(float64(shape.effectiveEntries), "effective-entries/op")
|
||||
})
|
||||
}
|
||||
|
||||
func inspectionExtraParams(entries int) map[string]any {
|
||||
providers := make(map[string]any, entries)
|
||||
for index := range entries {
|
||||
providers[fmt.Sprintf("provider-%03d", index)] = map[string]any{
|
||||
"enabled": true,
|
||||
"labels": []any{fmt.Sprintf("tier-%d", index%4), fmt.Sprintf("region-%d", index%3)},
|
||||
"limits": map[string]any{"burst": index + 1, "period": "minute"},
|
||||
"routes": []any{map[string]any{"weight": index + 1, "fallback": true}},
|
||||
}
|
||||
}
|
||||
return map[string]any{"provider_options": providers}
|
||||
}
|
||||
|
||||
func inspectionDefinitions(extraParams map[string]any) configurationShape {
|
||||
profiles := map[string]any{
|
||||
"small": map[string]any{"model": "inspection-model", "extra_params": extraParams},
|
||||
}
|
||||
prompt := benchmarkPrompt("small")
|
||||
prompt["inputs"] = []any{map[string]any{"name": "transcript", "required": true}}
|
||||
prompts := map[string]any{
|
||||
"inspection": map[string]any{"1": prompt},
|
||||
}
|
||||
return configurationShape{
|
||||
main: configkit.Input{Source: configkit.Mapping("inspection definitions", map[string]any{
|
||||
"profiles": profiles,
|
||||
"prompts": prompts,
|
||||
})},
|
||||
inputEntries: 2, effectiveEntries: 2, tiers: 1,
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkInspectionSelection(b *testing.B) {
|
||||
for _, scenario := range []struct {
|
||||
name string
|
||||
entries int
|
||||
}{
|
||||
{name: "small_extra", entries: 2},
|
||||
{name: "nested_extra", entries: 64},
|
||||
} {
|
||||
shape := inspectionDefinitions(inspectionExtraParams(scenario.entries))
|
||||
result, err := load(shape.main, nil)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
ctx := context.Background()
|
||||
definition, hash, err := result.GetPromptSnapshot(ctx, "inspection", "1")
|
||||
if err != nil || definition.ID != "inspection" || definition.Version != "1" || hash == "" ||
|
||||
len(definition.Inputs) != 1 || definition.Inputs[0].Name != "transcript" || !definition.Inputs[0].Required {
|
||||
b.Fatalf("prompt lookup fixture is invalid: definition=%+v hash=%q err=%v", definition, hash, err)
|
||||
}
|
||||
profile, err := result.GetProfile(ctx, "small")
|
||||
if err != nil || profile.ID != "small" || profile.ResolvedTarget.Model != "inspection-model" ||
|
||||
len(profile.ResolvedTarget.ExtraParams["provider_options"].(map[string]any)) != scenario.entries {
|
||||
b.Fatalf("profile lookup fixture is invalid: profile=%+v err=%v", profile, err)
|
||||
}
|
||||
inspection, err := result.GetPromptInspection(ctx, "inspection", "1")
|
||||
if err != nil || inspection.PromptID != "inspection" || inspection.PromptVersion != "1" ||
|
||||
len(inspection.Inputs) != 1 || inspection.Inputs[0].Name != "transcript" {
|
||||
b.Fatalf("prompt inspection fixture is invalid: inspection=%+v err=%v", inspection, err)
|
||||
}
|
||||
identity, err := result.GetProfileIdentity(ctx, "small")
|
||||
if err != nil || identity != "small" {
|
||||
b.Fatalf("profile identity fixture is invalid: identity=%q err=%v", identity, err)
|
||||
}
|
||||
|
||||
b.Run("prompt/"+scenario.name, func(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
for b.Loop() {
|
||||
definition, hash, err := result.GetPromptSnapshot(ctx, "inspection", "1")
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
selectedPrompt = definition
|
||||
selectedPromptHash = hash
|
||||
}
|
||||
b.ReportMetric(float64(scenario.entries), "extra-entries/op")
|
||||
})
|
||||
b.Run("prompt_inspection/"+scenario.name, func(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
for b.Loop() {
|
||||
inspection, err := result.GetPromptInspection(ctx, "inspection", "1")
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
selectedPromptInspection = inspection
|
||||
}
|
||||
b.ReportMetric(float64(scenario.entries), "extra-entries/op")
|
||||
})
|
||||
b.Run("profile/"+scenario.name, func(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
for b.Loop() {
|
||||
profile, err := result.GetProfile(ctx, "small")
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
selectedProfile = profile
|
||||
}
|
||||
b.ReportMetric(float64(scenario.entries), "extra-entries/op")
|
||||
})
|
||||
b.Run("profile_identity/"+scenario.name, func(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
for b.Loop() {
|
||||
identity, err := result.GetProfileIdentity(ctx, "small")
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
selectedProfileIdentity = identity
|
||||
}
|
||||
b.ReportMetric(float64(scenario.entries), "extra-entries/op")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,6 +16,15 @@ type Inspection interface {
|
||||
Explain(configkit.Path) (configkit.Explanation, bool)
|
||||
WriteEffective(io.Writer) error
|
||||
WriteExplanation(io.Writer, configkit.Path) error
|
||||
capturedResult() *configkit.Result[document]
|
||||
}
|
||||
|
||||
type inspectionResult struct {
|
||||
*configkit.Result[document]
|
||||
}
|
||||
|
||||
func (inspection inspectionResult) capturedResult() *configkit.Result[document] {
|
||||
return inspection.Result
|
||||
}
|
||||
|
||||
// Inspection returns only the retained result's inspection operations.
|
||||
@@ -23,7 +32,19 @@ func (state *loaded) Inspection() Inspection {
|
||||
if state == nil {
|
||||
return nil
|
||||
}
|
||||
return state.result
|
||||
return inspectionResult{Result: state.result}
|
||||
}
|
||||
|
||||
// Diff compares two captured ConfigKit results without reconstructing them.
|
||||
func Diff(left, right Inspection) ([]configkit.Difference, error) {
|
||||
var leftResult, rightResult *configkit.Result[document]
|
||||
if left != nil {
|
||||
leftResult = left.capturedResult()
|
||||
}
|
||||
if right != nil {
|
||||
rightResult = right.capturedResult()
|
||||
}
|
||||
return configkit.Diff(leftResult, rightResult)
|
||||
}
|
||||
|
||||
func secretProviderPath(path configkit.Path) bool {
|
||||
|
||||
29
internal/configuration/listing.go
Normal file
29
internal/configuration/listing.go
Normal file
@@ -0,0 +1,29 @@
|
||||
package configuration
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
)
|
||||
|
||||
func (state *loaded) ListProfiles(ctx context.Context) ([]string, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
profiles := append([]string(nil), state.profileIDs...)
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return profiles, nil
|
||||
}
|
||||
|
||||
func (state *loaded) ListPrompts(ctx context.Context) ([]domain.PromptIdentity, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
prompts := append([]domain.PromptIdentity(nil), state.promptIDs...)
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return prompts, nil
|
||||
}
|
||||
142
internal/configuration/listing_test.go
Normal file
142
internal/configuration/listing_test.go
Normal file
@@ -0,0 +1,142 @@
|
||||
package configuration
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
)
|
||||
|
||||
func TestLoadedListsSortedEffectiveIdentities(t *testing.T) {
|
||||
state, err := Load(mapping("identities", map[string]any{
|
||||
"backends": map[string]any{
|
||||
"local": map[string]any{"endpoint": "http://localhost:8080/v1"},
|
||||
},
|
||||
"profiles": map[string]any{
|
||||
"z/profile": map[string]any{"backend": "local", "model": "z-model"},
|
||||
"partial/profile": map[string]any{"backend": "local"},
|
||||
"a/profile": map[string]any{"backend": "local", "model": "a-model"},
|
||||
" z/profile ": map[string]any{"backend": "local", "model": "spaced-model"},
|
||||
},
|
||||
"prompts": map[string]any{
|
||||
"z/prompt": map[string]any{"v1": listingPromptDefinition()},
|
||||
"a/prompt": map[string]any{"v1": listingPromptDefinition()},
|
||||
" z/prompt ": map[string]any{
|
||||
"2": listingPromptDefinition(),
|
||||
"10": listingPromptDefinition(),
|
||||
},
|
||||
},
|
||||
}), nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
profiles, err := state.ListProfiles(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wantProfiles := []string{" z/profile ", "a/profile", "partial/profile", "z/profile"}
|
||||
if !reflect.DeepEqual(profiles, wantProfiles) {
|
||||
t.Fatalf("profile IDs = %#v, want %#v", profiles, wantProfiles)
|
||||
}
|
||||
|
||||
prompts, err := state.ListPrompts(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wantPrompts := []domain.PromptIdentity{
|
||||
{ID: " z/prompt ", Version: "10"},
|
||||
{ID: " z/prompt ", Version: "2"},
|
||||
{ID: "a/prompt", Version: "v1"},
|
||||
{ID: "z/prompt", Version: "v1"},
|
||||
}
|
||||
if !reflect.DeepEqual(prompts, wantPrompts) {
|
||||
t.Fatalf("prompt identities = %#v, want %#v", prompts, wantPrompts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadedListsAreIndependent(t *testing.T) {
|
||||
state, err := Load(mapping("identities", map[string]any{
|
||||
"profiles": map[string]any{"profile": map[string]any{}},
|
||||
"prompts": map[string]any{"example": map[string]any{"v1": listingPromptDefinition()}},
|
||||
}), nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
profiles, err := state.ListProfiles(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
profiles = append(profiles, "caller profile")
|
||||
profiles[0] = "changed profile"
|
||||
|
||||
prompts, err := state.ListPrompts(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
prompts[0].ID = "changed prompt"
|
||||
prompts[0].Version = "changed version"
|
||||
|
||||
profiles, err = state.ListProfiles(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(profiles, []string{"profile"}) {
|
||||
t.Fatalf("caller mutation changed profile inventory: %#v", profiles)
|
||||
}
|
||||
prompts, err = state.ListPrompts(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wantPrompts := []domain.PromptIdentity{{ID: "example", Version: "v1"}}
|
||||
if !reflect.DeepEqual(prompts, wantPrompts) {
|
||||
t.Fatalf("caller mutation changed prompt inventory: %#v", prompts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadedListsEmptyInventory(t *testing.T) {
|
||||
state, err := Load(configkit.Input{Source: configkit.NoSource()}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
profiles, err := state.ListProfiles(context.Background())
|
||||
if err != nil || len(profiles) != 0 {
|
||||
t.Fatalf("empty profile listing = %#v, %v", profiles, err)
|
||||
}
|
||||
prompts, err := state.ListPrompts(context.Background())
|
||||
if err != nil || len(prompts) != 0 {
|
||||
t.Fatalf("empty prompt listing = %#v, %v", prompts, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadedListsHonorCanceledContext(t *testing.T) {
|
||||
state, err := Load(mapping("identities", map[string]any{
|
||||
"profiles": map[string]any{"partial": map[string]any{}},
|
||||
"prompts": map[string]any{"example": map[string]any{"v1": listingPromptDefinition()}},
|
||||
}), nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
|
||||
profiles, err := state.ListProfiles(ctx)
|
||||
if !errors.Is(err, context.Canceled) || len(profiles) != 0 {
|
||||
t.Fatalf("canceled profile listing = %#v, %v", profiles, err)
|
||||
}
|
||||
prompts, err := state.ListPrompts(ctx)
|
||||
if !errors.Is(err, context.Canceled) || len(prompts) != 0 {
|
||||
t.Fatalf("canceled prompt listing = %#v, %v", prompts, err)
|
||||
}
|
||||
}
|
||||
|
||||
func listingPromptDefinition() map[string]any {
|
||||
return map[string]any{
|
||||
"messages": []any{map[string]any{"role": "user", "content": "Hello."}},
|
||||
"output": map[string]any{"format": "text", "validation_mode": "basic"},
|
||||
}
|
||||
}
|
||||
@@ -54,10 +54,10 @@ type promptConfig struct {
|
||||
}
|
||||
|
||||
type promptInputConfig struct {
|
||||
Name string `yaml:"name"`
|
||||
Required bool `yaml:"required"`
|
||||
ContentType string `yaml:"content_type"`
|
||||
Description string `yaml:"description"`
|
||||
Name string `yaml:"name"`
|
||||
Required bool `yaml:"required"`
|
||||
ContentType *string `yaml:"content_type"`
|
||||
Description *string `yaml:"description"`
|
||||
}
|
||||
|
||||
type promptMessageConfig struct {
|
||||
@@ -93,10 +93,13 @@ type loaded struct {
|
||||
result *configkit.Result[document]
|
||||
backends map[string]map[string]fieldState
|
||||
profiles map[string]map[string]fieldState
|
||||
promptMetadata map[string]map[string][]promptInputPresence
|
||||
prompts map[string]map[string]*promptEntry
|
||||
ambiguous map[string]bool
|
||||
registry *backend.Registry
|
||||
indexedProfiles map[string]domain.ExecutionProfile
|
||||
profileIDs []string
|
||||
promptIDs []domain.PromptIdentity
|
||||
}
|
||||
|
||||
var envName = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`)
|
||||
@@ -112,8 +115,9 @@ func Load(main configkit.Input, underlays []configkit.Underlay) (*loaded, error)
|
||||
// load performs one ConfigKit lifecycle over the selected source tiers.
|
||||
func load(main configkit.Input, underlays []configkit.Underlay) (*loaded, error) {
|
||||
state := &loaded{
|
||||
backends: make(map[string]map[string]fieldState),
|
||||
profiles: make(map[string]map[string]fieldState),
|
||||
backends: make(map[string]map[string]fieldState),
|
||||
profiles: make(map[string]map[string]fieldState),
|
||||
promptMetadata: make(map[string]map[string][]promptInputPresence),
|
||||
}
|
||||
result, err := configkit.LoadLayered(main, configkit.LayeredOptions[document]{
|
||||
Underlays: underlays,
|
||||
@@ -122,6 +126,7 @@ func load(main configkit.Input, underlays []configkit.Underlay) (*loaded, error)
|
||||
Resolve: func(config *document, resolution *configkit.Resolution) error {
|
||||
captureFields(config.Backends, "backends", backendFields, resolution, state.backends)
|
||||
captureFields(config.Profiles, "profiles", profileFields, resolution, state.profiles)
|
||||
capturePromptMetadata(config.Prompts, resolution, state.promptMetadata)
|
||||
resolveBackends(config.Backends, resolution)
|
||||
resolveProfiles(config.Profiles, resolution)
|
||||
resolvePrompts(config.Prompts, resolution)
|
||||
@@ -140,9 +145,53 @@ func load(main configkit.Input, underlays []configkit.Underlay) (*loaded, error)
|
||||
if err := state.buildExecutionProfiles(); err != nil {
|
||||
return nil, fmt.Errorf("load configuration: %w", err)
|
||||
}
|
||||
state.profileIDs = sortedKeys(state.indexedProfiles)
|
||||
for _, id := range sortedKeys(state.prompts) {
|
||||
for _, version := range sortedKeys(state.prompts[id]) {
|
||||
state.promptIDs = append(state.promptIDs, domain.PromptIdentity{ID: id, Version: version})
|
||||
}
|
||||
}
|
||||
return state, nil
|
||||
}
|
||||
|
||||
type promptInputPresence struct {
|
||||
contentType fieldState
|
||||
description fieldState
|
||||
}
|
||||
|
||||
func capturePromptMetadata(prompts map[string]map[string]promptConfig, resolution *configkit.Resolution, destination map[string]map[string][]promptInputPresence) {
|
||||
for _, id := range sortedKeys(prompts) {
|
||||
if destination[id] == nil {
|
||||
destination[id] = make(map[string][]promptInputPresence, len(prompts[id]))
|
||||
}
|
||||
for _, version := range sortedKeys(prompts[id]) {
|
||||
inputs := prompts[id][version].Inputs
|
||||
states := make([]promptInputPresence, len(inputs))
|
||||
base := configkit.Keys("prompts", id, version, "inputs")
|
||||
for index := range inputs {
|
||||
path := base.Index(index)
|
||||
states[index] = promptInputPresence{
|
||||
contentType: captureFieldState(path.Key("content_type"), resolution),
|
||||
description: captureFieldState(path.Key("description"), resolution),
|
||||
}
|
||||
}
|
||||
destination[id][version] = states
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func captureFieldState(path configkit.Path, resolution *configkit.Resolution) fieldState {
|
||||
info, exists := resolution.InputInfo(path)
|
||||
switch {
|
||||
case !exists || !info.Explicit:
|
||||
return fieldAbsent
|
||||
case info.Null:
|
||||
return fieldNull
|
||||
default:
|
||||
return fieldValue
|
||||
}
|
||||
}
|
||||
|
||||
func captureFields[T any](entries map[string]T, collection string, fields []string, resolution *configkit.Resolution, destination map[string]map[string]fieldState) {
|
||||
for id := range entries {
|
||||
states := make(map[string]fieldState, len(fields))
|
||||
|
||||
@@ -21,11 +21,12 @@ var (
|
||||
)
|
||||
|
||||
type promptEntry struct {
|
||||
definition domain.PromptDefinition
|
||||
inspection domain.PromptInspection
|
||||
hash string
|
||||
session *template.Template
|
||||
messages []*template.Template
|
||||
definition domain.PromptDefinition
|
||||
inspection domain.PromptInspection
|
||||
requirementInputs []prompt.InputMetadata
|
||||
hash string
|
||||
session *template.Template
|
||||
messages []*template.Template
|
||||
}
|
||||
|
||||
func resolvePrompts(prompts map[string]map[string]promptConfig, resolution *configkit.Resolution) {
|
||||
@@ -48,18 +49,24 @@ func resolvePrompts(prompts map[string]map[string]promptConfig, resolution *conf
|
||||
for index := range definition.Inputs {
|
||||
input := &definition.Inputs[index]
|
||||
inputPath := base.Key("inputs").Index(index)
|
||||
normalizedName := strings.TrimSpace(input.Name)
|
||||
if normalizedName != input.Name {
|
||||
configkit.Normalize(resolution, inputPath.Key("name"), &input.Name, normalizedName, "normalize input metadata")
|
||||
inputsChanged = true
|
||||
}
|
||||
for _, field := range []struct {
|
||||
name string
|
||||
value *string
|
||||
value **string
|
||||
}{
|
||||
{"name", &input.Name},
|
||||
{"content_type", &input.ContentType},
|
||||
{"description", &input.Description},
|
||||
} {
|
||||
normalized := strings.TrimSpace(*field.value)
|
||||
if normalized != *field.value {
|
||||
configkit.Normalize(resolution, inputPath.Key(field.name), field.value, normalized, "normalize input metadata")
|
||||
inputsChanged = true
|
||||
if *field.value != nil {
|
||||
normalized := strings.TrimSpace(**field.value)
|
||||
if normalized != **field.value {
|
||||
configkit.Normalize(resolution, inputPath.Key(field.name), field.value, &normalized, "normalize input metadata")
|
||||
inputsChanged = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -132,6 +139,10 @@ func (state *loaded) materializePrompt(id, version string, raw promptConfig, loo
|
||||
}
|
||||
seenInputs := make(map[string]bool, len(raw.Inputs))
|
||||
definition.Inputs = make([]domain.PromptInput, 0, len(raw.Inputs))
|
||||
entry := &promptEntry{
|
||||
messages: make([]*template.Template, 0, len(raw.Messages)),
|
||||
requirementInputs: make([]prompt.InputMetadata, 0, len(raw.Inputs)),
|
||||
}
|
||||
for index, input := range raw.Inputs {
|
||||
path := base.Key("inputs").Index(index)
|
||||
if input.Name == "" {
|
||||
@@ -141,11 +152,27 @@ func (state *loaded) materializePrompt(id, version string, raw promptConfig, loo
|
||||
return nil, fmt.Errorf("%s: input name is duplicated", path.Key("name"))
|
||||
}
|
||||
seenInputs[input.Name] = true
|
||||
contentType := ""
|
||||
if input.ContentType != nil {
|
||||
contentType = *input.ContentType
|
||||
}
|
||||
description := ""
|
||||
if input.Description != nil {
|
||||
description = *input.Description
|
||||
}
|
||||
definition.Inputs = append(definition.Inputs, domain.PromptInput{
|
||||
Name: input.Name, Required: input.Required, ContentType: input.ContentType, Description: input.Description,
|
||||
Name: input.Name, Required: input.Required, ContentType: contentType, Description: description,
|
||||
})
|
||||
presence := state.promptMetadata[id][version][index]
|
||||
metadata := prompt.InputMetadata{Name: input.Name, Required: input.Required}
|
||||
if presence.contentType == fieldValue {
|
||||
metadata.ContentType = stringPointer(contentType)
|
||||
}
|
||||
if presence.description == fieldValue {
|
||||
metadata.Description = stringPointer(description)
|
||||
}
|
||||
entry.requirementInputs = append(entry.requirementInputs, metadata)
|
||||
}
|
||||
entry := &promptEntry{messages: make([]*template.Template, 0, len(raw.Messages))}
|
||||
definition.Templates = make([]domain.PromptMessageTemplate, 0, len(raw.Messages))
|
||||
for index, message := range raw.Messages {
|
||||
path := base.Key("messages").Index(index)
|
||||
@@ -233,6 +260,11 @@ func (state *loaded) materializePrompt(id, version string, raw promptConfig, loo
|
||||
return entry, nil
|
||||
}
|
||||
|
||||
func stringPointer(value string) *string {
|
||||
copy := value
|
||||
return ©
|
||||
}
|
||||
|
||||
func (state *loaded) readContent(fieldPath configkit.Path, name string, lookupOrigin func(configkit.Path) (configkit.FieldInfo, bool)) (string, error) {
|
||||
if strings.TrimSpace(name) == "" {
|
||||
return "", fmt.Errorf("%s: content_file path must not be blank", fieldPath)
|
||||
@@ -265,14 +297,14 @@ func (state *loaded) readContent(fieldPath configkit.Path, name string, lookupOr
|
||||
return string(content), nil
|
||||
}
|
||||
|
||||
func (state *loaded) prompt(id, version string) (*domain.PromptDefinition, domain.PromptInspection, error) {
|
||||
func (state *loaded) lookupPromptEntry(id, version string) (*promptEntry, error) {
|
||||
if state == nil || strings.TrimSpace(id) == "" || (version != "" && strings.TrimSpace(version) == "") {
|
||||
return nil, domain.PromptInspection{}, errPromptNotFound
|
||||
return nil, errPromptNotFound
|
||||
}
|
||||
versions := state.prompts[id]
|
||||
if version == "" {
|
||||
if state.ambiguous[id] {
|
||||
return nil, domain.PromptInspection{}, errPromptAmbiguous
|
||||
return nil, errPromptAmbiguous
|
||||
}
|
||||
for selected := range versions {
|
||||
version = selected
|
||||
@@ -280,9 +312,12 @@ func (state *loaded) prompt(id, version string) (*domain.PromptDefinition, domai
|
||||
}
|
||||
entry := versions[version]
|
||||
if entry == nil {
|
||||
return nil, domain.PromptInspection{}, errPromptNotFound
|
||||
return nil, errPromptNotFound
|
||||
}
|
||||
definition := entry.definition
|
||||
return entry, nil
|
||||
}
|
||||
|
||||
func ownPromptDefinition(definition domain.PromptDefinition) *domain.PromptDefinition {
|
||||
definition.Inputs = append([]domain.PromptInput(nil), definition.Inputs...)
|
||||
definition.Templates = append([]domain.PromptMessageTemplate(nil), definition.Templates...)
|
||||
for index := range definition.Templates {
|
||||
@@ -291,9 +326,27 @@ func (state *loaded) prompt(id, version string) (*domain.PromptDefinition, domai
|
||||
definition.Templates[index].CacheControl = ©
|
||||
}
|
||||
}
|
||||
inspection := entry.inspection
|
||||
return &definition
|
||||
}
|
||||
|
||||
func ownPromptInspection(inspection domain.PromptInspection) *domain.PromptInspection {
|
||||
inspection.Inputs = append([]domain.PromptInput(nil), inspection.Inputs...)
|
||||
return &definition, inspection, nil
|
||||
return &inspection
|
||||
}
|
||||
|
||||
func promptLookupError(err error) error {
|
||||
if errors.Is(err, errPromptNotFound) {
|
||||
return promptdef.ErrPromptDefinitionNotFound
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (state *loaded) prompt(id, version string) (*domain.PromptDefinition, domain.PromptInspection, error) {
|
||||
entry, err := state.lookupPromptEntry(id, version)
|
||||
if err != nil {
|
||||
return nil, domain.PromptInspection{}, err
|
||||
}
|
||||
return ownPromptDefinition(entry.definition), *ownPromptInspection(entry.inspection), nil
|
||||
}
|
||||
|
||||
// GetPromptSnapshot returns an independently owned captured definition and its
|
||||
@@ -302,14 +355,24 @@ func (state *loaded) GetPromptSnapshot(ctx context.Context, id, version string)
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
definition, inspection, err := state.prompt(id, version)
|
||||
if errors.Is(err, errPromptNotFound) {
|
||||
return nil, "", promptdef.ErrPromptDefinitionNotFound
|
||||
}
|
||||
entry, err := state.lookupPromptEntry(id, version)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
return nil, "", promptLookupError(err)
|
||||
}
|
||||
return definition, inspection.PromptHash, nil
|
||||
return ownPromptDefinition(entry.definition), entry.hash, nil
|
||||
}
|
||||
|
||||
// GetPromptInspection returns an independently owned inspection of one
|
||||
// captured prompt without copying its execution templates.
|
||||
func (state *loaded) GetPromptInspection(ctx context.Context, id, version string) (*domain.PromptInspection, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
entry, err := state.lookupPromptEntry(id, version)
|
||||
if err != nil {
|
||||
return nil, promptLookupError(err)
|
||||
}
|
||||
return ownPromptInspection(entry.inspection), nil
|
||||
}
|
||||
|
||||
// GetPromptDefinition implements the runner's prompt lookup boundary.
|
||||
@@ -329,3 +392,33 @@ func (state *loaded) GetCompiledTemplates(id, version string) (prompt.CompiledTe
|
||||
}
|
||||
return prompt.CompiledTemplates{Session: entry.session, Messages: entry.messages}, true
|
||||
}
|
||||
|
||||
// GetRequirementSnapshot returns captured templates and input metadata for
|
||||
// one exact prompt identity. Returned metadata and slices are independently
|
||||
// owned; the compiled templates are shared immutable construction-time parses.
|
||||
func (state *loaded) GetRequirementSnapshot(id, version string) (prompt.RequirementSnapshot, bool) {
|
||||
if state == nil {
|
||||
return prompt.RequirementSnapshot{}, false
|
||||
}
|
||||
entry := state.prompts[id][version]
|
||||
if entry == nil {
|
||||
return prompt.RequirementSnapshot{}, false
|
||||
}
|
||||
snapshot := prompt.RequirementSnapshot{
|
||||
Templates: prompt.CompiledTemplates{
|
||||
Session: entry.session,
|
||||
Messages: append([]*template.Template(nil), entry.messages...),
|
||||
},
|
||||
Inputs: make([]prompt.InputMetadata, len(entry.requirementInputs)),
|
||||
}
|
||||
for index, input := range entry.requirementInputs {
|
||||
snapshot.Inputs[index] = input
|
||||
if input.ContentType != nil {
|
||||
snapshot.Inputs[index].ContentType = stringPointer(*input.ContentType)
|
||||
}
|
||||
if input.Description != nil {
|
||||
snapshot.Inputs[index].Description = stringPointer(*input.Description)
|
||||
}
|
||||
}
|
||||
return snapshot, true
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package configuration
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io/fs"
|
||||
"os"
|
||||
@@ -10,6 +11,7 @@ import (
|
||||
"testing/fstest"
|
||||
|
||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/promptdef"
|
||||
)
|
||||
|
||||
func TestLoadCapturesInheritedImportedContent(t *testing.T) {
|
||||
@@ -134,6 +136,94 @@ func TestLoadRejectsMalformedStaticSyntaxAndAmbiguousSelection(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestPromptInspectionLookupMatchesSnapshotSelectionAndOwnsInputs(t *testing.T) {
|
||||
makePrompt := func(content string) map[string]any {
|
||||
return map[string]any{
|
||||
"default_profile": "partial",
|
||||
"inputs": []any{map[string]any{
|
||||
"name": "document", "required": true,
|
||||
"content_type": " text/plain ", "description": " source document ",
|
||||
}},
|
||||
"messages": []any{map[string]any{"role": "user", "content": content}},
|
||||
"output": map[string]any{"format": "text", "validation_mode": "none"},
|
||||
}
|
||||
}
|
||||
state, err := load(mapping("prompt inspection lookup", map[string]any{
|
||||
"profiles": map[string]any{"partial": map[string]any{}},
|
||||
"prompts": map[string]any{
|
||||
"multi": map[string]any{"v1": makePrompt("first"), "v2": makePrompt("second")},
|
||||
"single": map[string]any{"v1": makePrompt("only")},
|
||||
},
|
||||
}), nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
for _, selection := range []struct {
|
||||
id, version string
|
||||
}{
|
||||
{id: "multi", version: "v2"},
|
||||
{id: "single"},
|
||||
} {
|
||||
inspection, err := state.GetPromptInspection(context.Background(), selection.id, selection.version)
|
||||
if err != nil {
|
||||
t.Fatalf("inspect %q/%q: %v", selection.id, selection.version, err)
|
||||
}
|
||||
definition, hash, err := state.GetPromptSnapshot(context.Background(), selection.id, selection.version)
|
||||
if err != nil {
|
||||
t.Fatalf("snapshot %q/%q: %v", selection.id, selection.version, err)
|
||||
}
|
||||
if inspection.PromptID != definition.ID || inspection.PromptVersion != definition.Version ||
|
||||
inspection.PromptHash != hash || inspection.DefaultProfileID != "partial" ||
|
||||
len(inspection.Inputs) != 1 || inspection.Inputs[0].Name != "document" ||
|
||||
inspection.Inputs[0].ContentType != "text/plain" || inspection.Inputs[0].Description != "source document" {
|
||||
t.Fatalf("inspection did not match the selected snapshot: inspection=%+v definition=%+v hash=%q", inspection, definition, hash)
|
||||
}
|
||||
if len(definition.Inputs) != len(inspection.Inputs) || definition.Inputs[0].Name != inspection.Inputs[0].Name {
|
||||
t.Fatalf("definition and inspection inputs diverged: definition=%+v inspection=%+v", definition.Inputs, inspection.Inputs)
|
||||
}
|
||||
}
|
||||
|
||||
owned, err := state.GetPromptInspection(context.Background(), "single", "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
owned.Inputs[0].Name = "caller mutation"
|
||||
owned.Inputs[0].Description = "caller mutation"
|
||||
again, err := state.GetPromptInspection(context.Background(), "single", "v1")
|
||||
if err != nil || again.Inputs[0].Name != "document" || again.Inputs[0].Description != "source document" {
|
||||
t.Fatalf("inspection mutation escaped into captured state: inspection=%+v err=%v", again, err)
|
||||
}
|
||||
|
||||
for _, selection := range []struct {
|
||||
id, version string
|
||||
}{
|
||||
{id: "missing", version: "v1"},
|
||||
{id: "multi", version: "V2"},
|
||||
{id: " ", version: "v1"},
|
||||
} {
|
||||
inspection, err := state.GetPromptInspection(context.Background(), selection.id, selection.version)
|
||||
if inspection != nil || !errors.Is(err, promptdef.ErrPromptDefinitionNotFound) {
|
||||
t.Errorf("inspection selection %q/%q=(%+v, %v), want not found", selection.id, selection.version, inspection, err)
|
||||
}
|
||||
definition, _, err := state.GetPromptSnapshot(context.Background(), selection.id, selection.version)
|
||||
if definition != nil || !errors.Is(err, promptdef.ErrPromptDefinitionNotFound) {
|
||||
t.Errorf("snapshot selection %q/%q=(%+v, %v), want not found", selection.id, selection.version, definition, err)
|
||||
}
|
||||
}
|
||||
if _, err := state.GetPromptInspection(context.Background(), "multi", ""); !errors.Is(err, errPromptAmbiguous) {
|
||||
t.Errorf("ambiguous inspection selection error=%v", err)
|
||||
}
|
||||
if _, _, err := state.GetPromptSnapshot(context.Background(), "multi", ""); !errors.Is(err, errPromptAmbiguous) {
|
||||
t.Errorf("ambiguous snapshot selection error=%v", err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
if _, err := state.GetPromptInspection(ctx, "single", "v1"); !errors.Is(err, context.Canceled) {
|
||||
t.Errorf("canceled inspection selection error=%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadAcceptsEmptyInlineContentAndRejectsInvalidPromptFields(t *testing.T) {
|
||||
state, err := load(mapping("empty-content", map[string]any{"prompts": map[string]any{" exact.id ": map[string]any{"v1": map[string]any{
|
||||
"inputs": []any{map[string]any{"name": " note ", "required": true}},
|
||||
@@ -170,6 +260,113 @@ func TestLoadAcceptsEmptyInlineContentAndRejectsInvalidPromptFields(t *testing.T
|
||||
}
|
||||
}
|
||||
|
||||
func TestRequirementSnapshotCapturesMetadataPresenceAndNormalizedValues(t *testing.T) {
|
||||
state, err := load(mapping("requirement-metadata", map[string]any{
|
||||
"prompts": map[string]any{"example": map[string]any{"v1": map[string]any{
|
||||
"session_id": "{{.session}}",
|
||||
"inputs": []any{
|
||||
map[string]any{"name": "absent"},
|
||||
map[string]any{"name": "null", "content_type": nil, "description": nil},
|
||||
map[string]any{"name": "empty", "content_type": "", "description": ""},
|
||||
map[string]any{"name": "normalized", "content_type": " text/plain ", "description": " final text "},
|
||||
},
|
||||
"messages": []any{map[string]any{"role": "user", "content": "{{input \"absent\"}}"}},
|
||||
"output": map[string]any{"format": "text", "validation_mode": "basic"},
|
||||
}}},
|
||||
}), nil)
|
||||
if err != nil {
|
||||
var loadError *configkit.LoadError
|
||||
if errors.As(err, &loadError) {
|
||||
t.Fatalf("%v: %+v", err, loadError.Diagnostics())
|
||||
}
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
snapshot, ok := state.GetRequirementSnapshot("example", "v1")
|
||||
if !ok || len(snapshot.Inputs) != 4 || len(snapshot.Templates.Messages) != 1 || snapshot.Templates.Session == nil {
|
||||
t.Fatalf("requirement snapshot was not captured: %+v, %t", snapshot, ok)
|
||||
}
|
||||
for _, index := range []int{0, 1} {
|
||||
if snapshot.Inputs[index].ContentType != nil || snapshot.Inputs[index].Description != nil {
|
||||
t.Errorf("input %q should not have explicit string metadata: %+v", snapshot.Inputs[index].Name, snapshot.Inputs[index])
|
||||
}
|
||||
}
|
||||
if snapshot.Inputs[2].ContentType == nil || *snapshot.Inputs[2].ContentType != "" ||
|
||||
snapshot.Inputs[2].Description == nil || *snapshot.Inputs[2].Description != "" {
|
||||
t.Errorf("explicit empty metadata presence was lost: %+v", snapshot.Inputs[2])
|
||||
}
|
||||
if snapshot.Inputs[3].ContentType == nil || *snapshot.Inputs[3].ContentType != "text/plain" ||
|
||||
snapshot.Inputs[3].Description == nil || *snapshot.Inputs[3].Description != "final text" {
|
||||
t.Errorf("snapshot did not use normalized final metadata: %+v", snapshot.Inputs[3])
|
||||
}
|
||||
|
||||
definition, inspection, err := state.prompt("example", "v1")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(definition.Inputs) != 4 || len(inspection.Inputs) != 4 || inspection.Inputs[2].ContentType != "" || inspection.Inputs[2].Description != "" ||
|
||||
inspection.Inputs[3].ContentType != "text/plain" || inspection.Inputs[3].Description != "final text" {
|
||||
t.Fatalf("existing prompt inspection shape or normalized values changed: %+v", inspection)
|
||||
}
|
||||
|
||||
*snapshot.Inputs[2].ContentType = "caller mutation"
|
||||
snapshot.Inputs[3].Name = "caller mutation"
|
||||
snapshot.Templates.Messages[0] = nil
|
||||
again, ok := state.GetRequirementSnapshot("example", "v1")
|
||||
if !ok || *again.Inputs[2].ContentType != "" || again.Inputs[3].Name != "normalized" || again.Templates.Messages[0] == nil {
|
||||
t.Fatalf("snapshot mutation escaped into captured state: %+v, %t", again, ok)
|
||||
}
|
||||
if _, ok := state.GetRequirementSnapshot("example", "V1"); ok {
|
||||
t.Fatal("snapshot lookup unexpectedly changed exact version matching")
|
||||
}
|
||||
}
|
||||
|
||||
func TestInputMetadataPresenceDoesNotChangePromptHashOrInspection(t *testing.T) {
|
||||
variants := []struct {
|
||||
name string
|
||||
fields map[string]any
|
||||
}{
|
||||
{name: "absent"},
|
||||
{name: "null", fields: map[string]any{"content_type": nil, "description": nil}},
|
||||
{name: "empty", fields: map[string]any{"content_type": "", "description": ""}},
|
||||
}
|
||||
var expectedHash string
|
||||
for _, variant := range variants {
|
||||
t.Run(variant.name, func(t *testing.T) {
|
||||
input := map[string]any{"name": "document"}
|
||||
for name, value := range variant.fields {
|
||||
input[name] = value
|
||||
}
|
||||
state, err := load(mapping("hash-"+variant.name, map[string]any{
|
||||
"prompts": map[string]any{"same": map[string]any{"v1": map[string]any{
|
||||
"inputs": []any{input},
|
||||
"messages": []any{map[string]any{"role": "user", "content": "hello"}},
|
||||
"output": map[string]any{"format": "text", "validation_mode": "basic"},
|
||||
}}},
|
||||
}), nil)
|
||||
if err != nil {
|
||||
var loadError *configkit.LoadError
|
||||
if errors.As(err, &loadError) {
|
||||
t.Fatalf("%v: %+v", err, loadError.Diagnostics())
|
||||
}
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, inspection, err := state.prompt("same", "v1")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(inspection.Inputs) != 1 || inspection.Inputs[0].ContentType != "" || inspection.Inputs[0].Description != "" {
|
||||
t.Fatalf("old inspection metadata changed: %+v", inspection.Inputs)
|
||||
}
|
||||
if expectedHash == "" {
|
||||
expectedHash = inspection.PromptHash
|
||||
} else if inspection.PromptHash != expectedHash {
|
||||
t.Fatalf("metadata presence changed the definition hash: got %q, want %q", inspection.PromptHash, expectedHash)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func promptWithFile(name string) map[string]any {
|
||||
return map[string]any{"prompts": map[string]any{"example": map[string]any{"v1": map[string]any{
|
||||
"messages": []any{map[string]any{"role": "user", "content_file": name}},
|
||||
|
||||
@@ -147,18 +147,26 @@ func overlayProviderMap(lower, upper map[string]any) (map[string]any, error) {
|
||||
return merged, nil
|
||||
}
|
||||
|
||||
// GetProfile returns an independently owned indexed profile for the internal
|
||||
// runner's lookup boundary.
|
||||
func (state *loaded) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) {
|
||||
func (state *loaded) lookupProfile(ctx context.Context, id string) (domain.ExecutionProfile, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
return domain.ExecutionProfile{}, err
|
||||
}
|
||||
if state == nil {
|
||||
return nil, fmt.Errorf("%w: %q", profilepkg.ErrProfileNotFound, id)
|
||||
return domain.ExecutionProfile{}, fmt.Errorf("%w: %q", profilepkg.ErrProfileNotFound, id)
|
||||
}
|
||||
profile, found := state.indexedProfiles[id]
|
||||
if !found {
|
||||
return nil, fmt.Errorf("%w: %q", profilepkg.ErrProfileNotFound, id)
|
||||
return domain.ExecutionProfile{}, fmt.Errorf("%w: %q", profilepkg.ErrProfileNotFound, id)
|
||||
}
|
||||
return profile, nil
|
||||
}
|
||||
|
||||
// GetProfile returns an independently owned indexed profile for the internal
|
||||
// runner's lookup boundary.
|
||||
func (state *loaded) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) {
|
||||
profile, err := state.lookupProfile(ctx, id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
target := *profile.ResolvedTarget
|
||||
params, err := jsonvalue.CopyMap(target.ExtraParams)
|
||||
@@ -170,6 +178,15 @@ func (state *loaded) GetProfile(ctx context.Context, id string) (*domain.Executi
|
||||
return &profile, nil
|
||||
}
|
||||
|
||||
// GetProfileIdentity returns the exact lookup ID for a captured profile
|
||||
// without copying its execution target.
|
||||
func (state *loaded) GetProfileIdentity(ctx context.Context, id string) (string, error) {
|
||||
if _, err := state.lookupProfile(ctx, id); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return id, nil
|
||||
}
|
||||
|
||||
// GetBackend returns an independently owned effective backend definition.
|
||||
func (state *loaded) GetBackend(id string) (domain.Backend, error) {
|
||||
if state == nil {
|
||||
|
||||
@@ -3,6 +3,7 @@ package configuration
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"reflect"
|
||||
@@ -11,6 +12,7 @@ import (
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/defaults"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/llm"
|
||||
profilepkg "gitea.maximumdirect.net/eric/promptkit/internal/profile"
|
||||
)
|
||||
|
||||
func TestLoadBuildsEffectiveTargetsAfterInheritance(t *testing.T) {
|
||||
@@ -89,6 +91,18 @@ func TestLoadBuildsEffectiveTargetsAfterInheritance(t *testing.T) {
|
||||
if partial.ResolvedTarget.Endpoint != "" || partial.ResolvedTarget.Model != "" {
|
||||
t.Fatalf("partial profile gained a runnable target: %+v", partial.ResolvedTarget)
|
||||
}
|
||||
identity, err := state.GetProfileIdentity(context.Background(), "partial")
|
||||
if err != nil || identity != "partial" {
|
||||
t.Fatalf("partial profile identity=(%q, %v)", identity, err)
|
||||
}
|
||||
if _, err := state.GetProfileIdentity(context.Background(), "missing"); !errors.Is(err, profilepkg.ErrProfileNotFound) {
|
||||
t.Fatalf("missing profile identity error=%v", err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
if _, err := state.GetProfileIdentity(ctx, "partial"); !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("canceled profile identity lookup error=%v", err)
|
||||
}
|
||||
endpointOnly, _ := state.GetProfile(context.Background(), "endpoint-only")
|
||||
if endpointOnly.ResolvedTarget.Endpoint != "https://standalone.example/v1" ||
|
||||
endpointOnly.ResolvedTarget.BackendID != "" || endpointOnly.ResolvedTarget.Model != "" {
|
||||
|
||||
@@ -135,6 +135,12 @@ type Artifact struct {
|
||||
Hash string
|
||||
}
|
||||
|
||||
// PromptIdentity identifies one configured prompt version.
|
||||
type PromptIdentity struct {
|
||||
ID string
|
||||
Version string
|
||||
}
|
||||
|
||||
// PromptDefinition represents a configured prompt execution definition.
|
||||
type PromptDefinition struct {
|
||||
ID string `yaml:"id"`
|
||||
|
||||
58
internal/domain/requirements.go
Normal file
58
internal/domain/requirements.go
Normal file
@@ -0,0 +1,58 @@
|
||||
package domain
|
||||
|
||||
// RequirementKind classifies how a prompt may depend on an input or variable.
|
||||
type RequirementKind string
|
||||
|
||||
const (
|
||||
RequirementRequired RequirementKind = "required"
|
||||
RequirementConditional RequirementKind = "conditional"
|
||||
RequirementOptional RequirementKind = "optional"
|
||||
)
|
||||
|
||||
// InputRequirement describes one declared or template-referenced input.
|
||||
type InputRequirement struct {
|
||||
Name string
|
||||
Kind RequirementKind
|
||||
Declared bool
|
||||
ContentType *string
|
||||
Description *string
|
||||
Explanation string
|
||||
}
|
||||
|
||||
// VariableRequirement describes one caller variable referenced by a prompt.
|
||||
type VariableRequirement struct {
|
||||
Name string
|
||||
Kind RequirementKind
|
||||
Supplied bool
|
||||
Explanation string
|
||||
}
|
||||
|
||||
// RequirementIssueCode identifies a limit that may affect requirement
|
||||
// discovery.
|
||||
type RequirementIssueCode string
|
||||
|
||||
const (
|
||||
RequirementIssueMissingVariable RequirementIssueCode = "missing_variable"
|
||||
RequirementIssueInputContent RequirementIssueCode = "input_content"
|
||||
RequirementIssueDynamicInputName RequirementIssueCode = "dynamic_input_name"
|
||||
RequirementIssueUnsupportedConstruct RequirementIssueCode = "unsupported_construct"
|
||||
)
|
||||
|
||||
// RequirementIssue describes one limitation on requirement discovery.
|
||||
type RequirementIssue struct {
|
||||
Code RequirementIssueCode
|
||||
InputName string
|
||||
VariableName string
|
||||
Description string
|
||||
}
|
||||
|
||||
// RequirementInspection contains the inventories discovered for one prompt.
|
||||
type RequirementInspection struct {
|
||||
PromptID string
|
||||
PromptVersion string
|
||||
ProfileID string
|
||||
Inputs []InputRequirement
|
||||
Variables []VariableRequirement
|
||||
Complete bool
|
||||
Issues []RequirementIssue
|
||||
}
|
||||
@@ -83,6 +83,12 @@ func (o *Options) SetSchemaFile(path string) error {
|
||||
type Engine struct {
|
||||
runner *usecase.Runner
|
||||
inspection configuration.Inspection
|
||||
listing identityInventory
|
||||
}
|
||||
|
||||
type identityInventory interface {
|
||||
ListProfiles(context.Context) ([]string, error)
|
||||
ListPrompts(context.Context) ([]domain.PromptIdentity, error)
|
||||
}
|
||||
|
||||
// New assembles an engine from facade-selected configuration and collaborators.
|
||||
@@ -135,6 +141,7 @@ func New(cfg Config, options Options) (*Engine, error) {
|
||||
return &Engine{
|
||||
runner: usecase.NewRunnerWithRepairer(snapshot, snapshot, reader, prompt.NewCompiledRenderer(snapshot), client, validator, usecase.NewDefaultOutputRepairer(client), manager, defaultRetries),
|
||||
inspection: snapshot.Inspection(),
|
||||
listing: snapshot,
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -178,9 +185,23 @@ func fileSource(name string) (fs.FS, string, string, error) {
|
||||
func (e *Engine) InspectPrompt(ctx context.Context, id, version string) (*domain.PromptInspection, error) {
|
||||
return e.runner.InspectPrompt(ctx, id, version)
|
||||
}
|
||||
func (e *Engine) InspectRequirements(
|
||||
ctx context.Context,
|
||||
request usecase.RequirementInspectionRequest,
|
||||
) (*domain.RequirementInspection, error) {
|
||||
return e.runner.InspectRequirements(ctx, request)
|
||||
}
|
||||
func (e *Engine) InspectProfile(ctx context.Context, id string) (*domain.ProfileInspection, error) {
|
||||
return e.runner.InspectProfile(ctx, id)
|
||||
}
|
||||
|
||||
func (e *Engine) ListProfiles(ctx context.Context) ([]string, error) {
|
||||
return e.listing.ListProfiles(ctx)
|
||||
}
|
||||
|
||||
func (e *Engine) ListPrompts(ctx context.Context) ([]domain.PromptIdentity, error) {
|
||||
return e.listing.ListPrompts(ctx)
|
||||
}
|
||||
func (e *Engine) Prepare(ctx context.Context, request domain.RunRequest) (*domain.PreparedRun, error) {
|
||||
return e.runner.Prepare(ctx, request)
|
||||
}
|
||||
|
||||
@@ -152,6 +152,11 @@ func (c *OpenAICompatibleClient) PrepareGeneration(req domain.GenerateRequest) (
|
||||
|
||||
payload, err := json.Marshal(wirePayload)
|
||||
if err != nil {
|
||||
for key, value := range req.Target.ExtraParams {
|
||||
if _, extraErr := json.Marshal(value); extraErr != nil {
|
||||
return nil, fmt.Errorf("%w: extra_params.%s must be JSON-serializable: %v", ErrInvalidRequest, key, extraErr)
|
||||
}
|
||||
}
|
||||
return nil, fmt.Errorf("%w: failed to encode request: %v", ErrRequestFailed, err)
|
||||
}
|
||||
apiKey := strings.TrimSpace(req.Target.APIKey)
|
||||
@@ -446,9 +451,6 @@ func openAIChatRequestPayload(req openAIChatRequest) (map[string]any, error) {
|
||||
if IsReservedOpenAIChatRequestField(key) {
|
||||
return nil, fmt.Errorf("extra_params key %q collides with reserved request field", key)
|
||||
}
|
||||
if _, err := json.Marshal(value); err != nil {
|
||||
return nil, fmt.Errorf("extra_params.%s must be JSON-serializable: %w", key, err)
|
||||
}
|
||||
out[key] = value
|
||||
}
|
||||
|
||||
|
||||
@@ -1113,6 +1113,13 @@ func TestOpenAICompatibleClientRejectsInvalidExtraParamsBeforeProviderCall(t *te
|
||||
}{
|
||||
{name: "empty key", extraParams: map[string]any{"": "empty"}, want: "key must not be empty"},
|
||||
{name: "unserializable value", extraParams: map[string]any{"bad": math.Inf(1)}, want: "JSON-serializable"},
|
||||
{
|
||||
name: "nested unserializable value",
|
||||
extraParams: map[string]any{
|
||||
"nested": map[string]any{"values": []any{"valid", math.NaN()}},
|
||||
},
|
||||
want: "extra_params.nested",
|
||||
},
|
||||
}
|
||||
for _, key := range []string{
|
||||
"model",
|
||||
@@ -1161,6 +1168,56 @@ func TestOpenAICompatibleClientRejectsInvalidExtraParamsBeforeProviderCall(t *te
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAICompatibleClientClassifiesSchemaEncodingFailures(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
extraParams map[string]any
|
||||
want error
|
||||
unwanted error
|
||||
}{
|
||||
{
|
||||
name: "serializable extra keeps full-request failure",
|
||||
extraParams: map[string]any{"client_option": "accepted"},
|
||||
want: ErrRequestFailed,
|
||||
unwanted: ErrInvalidRequest,
|
||||
},
|
||||
{
|
||||
name: "invalid extra keeps invalid-request failure",
|
||||
extraParams: map[string]any{"nested": map[string]any{"bad": math.Inf(1)}},
|
||||
want: ErrInvalidRequest,
|
||||
unwanted: ErrRequestFailed,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
provider := newRecordingProvider(t)
|
||||
client := newProviderClient(t, provider, OpenAICompatibleConfig{})
|
||||
request := ordinaryGenerateRequest()
|
||||
request.Target.ExtraParams = tc.extraParams
|
||||
request.StructuredOutput = &domain.StructuredOutputSpec{
|
||||
Type: domain.StructuredOutputJSONSchema,
|
||||
JSONSchema: &domain.StructuredOutputJSONSpec{
|
||||
Name: "invalid-schema",
|
||||
Strict: true,
|
||||
Schema: json.RawMessage("{"),
|
||||
},
|
||||
}
|
||||
|
||||
_, err := client.Generate(context.Background(), request)
|
||||
if err == nil || !errors.Is(err, tc.want) {
|
||||
t.Fatalf("error = %v, want identity %v", err, tc.want)
|
||||
}
|
||||
if errors.Is(err, tc.unwanted) {
|
||||
t.Fatalf("error = %v, unexpectedly has identity %v", err, tc.unwanted)
|
||||
}
|
||||
if calls := provider.requestCount(); calls != 0 {
|
||||
t.Fatalf("provider calls = %d, want 0", calls)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func checkConfiguredModelFallback(t *testing.T) {
|
||||
provider := newRecordingProvider(t)
|
||||
client := newProviderClient(t, provider, OpenAICompatibleConfig{Model: "default-model"})
|
||||
|
||||
119
internal/llm/prepare_generation_benchmark_test.go
Normal file
119
internal/llm/prepare_generation_benchmark_test.go
Normal file
@@ -0,0 +1,119 @@
|
||||
package llm
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
)
|
||||
|
||||
var prepareGenerationBenchmarkSink *PreparedGeneration
|
||||
|
||||
func BenchmarkPrepareGeneration(b *testing.B) {
|
||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||
BaseURL: "https://api.example.test/v1",
|
||||
Model: "benchmark-model",
|
||||
})
|
||||
if err != nil {
|
||||
b.Fatalf("construct benchmark client: %v", err)
|
||||
}
|
||||
|
||||
prompt := domain.RenderedPrompt{
|
||||
Messages: []domain.RenderedMessage{{Role: "user", Content: "Summarize this request."}},
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
request domain.GenerateRequest
|
||||
}{
|
||||
{
|
||||
name: "no_extras",
|
||||
request: domain.GenerateRequest{
|
||||
Prompt: prompt,
|
||||
Target: domain.ExecutionTarget{Model: "benchmark-model"},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "small_extras",
|
||||
request: domain.GenerateRequest{
|
||||
Prompt: prompt,
|
||||
Target: domain.ExecutionTarget{Model: "benchmark-model", ExtraParams: map[string]any{
|
||||
"priority": "low",
|
||||
"attempt": 3,
|
||||
"enabled": true,
|
||||
}},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "1MiB_string_extra",
|
||||
request: domain.GenerateRequest{
|
||||
Prompt: prompt,
|
||||
Target: domain.ExecutionTarget{Model: "benchmark-model", ExtraParams: map[string]any{
|
||||
"payload": strings.Repeat("x", 1<<20),
|
||||
}},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "nested_json_extras",
|
||||
request: domain.GenerateRequest{
|
||||
Prompt: prompt,
|
||||
Target: domain.ExecutionTarget{Model: "benchmark-model", ExtraParams: map[string]any{
|
||||
"options": map[string]any{
|
||||
"enabled": true,
|
||||
"limits": []any{2, 4, 8},
|
||||
"tags": []string{"summary", "offline"},
|
||||
},
|
||||
}},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "structured_output",
|
||||
request: domain.GenerateRequest{
|
||||
Prompt: prompt,
|
||||
Target: domain.ExecutionTarget{Model: "benchmark-model"},
|
||||
StructuredOutput: &domain.StructuredOutputSpec{
|
||||
Type: domain.StructuredOutputJSONSchema,
|
||||
JSONSchema: &domain.StructuredOutputJSONSpec{
|
||||
Name: "summary",
|
||||
Strict: true,
|
||||
Schema: map[string]any{
|
||||
"type": "object",
|
||||
"properties": map[string]any{
|
||||
"summary": map[string]any{"type": "string"},
|
||||
"count": map[string]any{"type": "integer"},
|
||||
},
|
||||
"required": []string{"summary", "count"},
|
||||
"additionalProperties": false,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
b.Run(tc.name, func(b *testing.B) {
|
||||
prepared, err := client.PrepareGeneration(tc.request)
|
||||
if err != nil {
|
||||
b.Fatalf("check prepared request: %v", err)
|
||||
}
|
||||
if !json.Valid(prepared.payload) {
|
||||
b.Fatal("prepared request is not valid JSON")
|
||||
}
|
||||
|
||||
b.ReportAllocs()
|
||||
b.SetBytes(int64(len(prepared.payload)))
|
||||
b.ResetTimer()
|
||||
for range b.N {
|
||||
prepared, err := client.PrepareGeneration(tc.request)
|
||||
if err != nil {
|
||||
b.Fatalf("prepare generation: %v", err)
|
||||
}
|
||||
prepareGenerationBenchmarkSink = prepared
|
||||
}
|
||||
b.StopTimer()
|
||||
runtime.KeepAlive(prepareGenerationBenchmarkSink)
|
||||
})
|
||||
}
|
||||
}
|
||||
254
internal/prompt/compiled_renderer_benchmark_test.go
Normal file
254
internal/prompt/compiled_renderer_benchmark_test.go
Normal file
@@ -0,0 +1,254 @@
|
||||
package prompt
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"testing"
|
||||
"text/template"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
)
|
||||
|
||||
const rendererBenchmarkArtifactSize = 256 << 10
|
||||
|
||||
var (
|
||||
renderedPromptBenchmarkSink *domain.RenderedPrompt
|
||||
renderedPromptsBenchmarkSink []*domain.RenderedPrompt
|
||||
)
|
||||
|
||||
type rendererBenchmarkIdentity struct {
|
||||
id string
|
||||
version string
|
||||
}
|
||||
|
||||
type rendererBenchmarkLookup struct {
|
||||
identity rendererBenchmarkIdentity
|
||||
templates CompiledTemplates
|
||||
}
|
||||
|
||||
func (lookup rendererBenchmarkLookup) GetCompiledTemplates(id, version string) (CompiledTemplates, bool) {
|
||||
if id != lookup.identity.id || version != lookup.identity.version {
|
||||
return CompiledTemplates{}, false
|
||||
}
|
||||
return lookup.templates, true
|
||||
}
|
||||
|
||||
type compiledRenderingBenchmarkCase struct {
|
||||
name string
|
||||
renderer Renderer
|
||||
definition *domain.PromptDefinition
|
||||
inputs map[string]*domain.Artifact
|
||||
vars map[string]string
|
||||
ctx context.Context
|
||||
outputBytes int
|
||||
check func(*testing.B, *domain.RenderedPrompt)
|
||||
}
|
||||
|
||||
func BenchmarkCompiledRendering(b *testing.B) {
|
||||
body := bytes.Repeat([]byte("x"), rendererBenchmarkArtifactSize)
|
||||
bodyText := string(body)
|
||||
|
||||
small := newCompiledRenderingBenchmarkCase(b,
|
||||
"small_message", "", []string{"Summary for {{.audience}}."},
|
||||
&domain.PromptDefinition{
|
||||
ID: "compiled-render-benchmark", Version: "v1",
|
||||
Templates: []domain.PromptMessageTemplate{{Role: "user", Content: "Summary for {{.audience}}."}},
|
||||
}, nil, map[string]string{"audience": "project leads"},
|
||||
len("Summary for project leads."),
|
||||
func(b *testing.B, rendered *domain.RenderedPrompt) {
|
||||
if rendered.SessionID != "" || len(rendered.Messages) != 1 || rendered.Messages[0].Content != "Summary for project leads." {
|
||||
b.Fatalf("unexpected small compiled render: %+v", rendered)
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
artifactInputs := map[string]*domain.Artifact{
|
||||
"document": {Name: "document", Body: body},
|
||||
}
|
||||
largeContent := `{{input "document"}}`
|
||||
largeDefinition := &domain.PromptDefinition{
|
||||
ID: "compiled-render-benchmark", Version: "v1",
|
||||
Inputs: []domain.PromptInput{{Name: "document"}},
|
||||
Templates: []domain.PromptMessageTemplate{{Role: "user", Content: largeContent}},
|
||||
}
|
||||
large := newCompiledRenderingBenchmarkCase(b,
|
||||
"artifact_backed_256KiB", "", []string{largeContent},
|
||||
largeDefinition, artifactInputs, nil, len(bodyText),
|
||||
func(b *testing.B, rendered *domain.RenderedPrompt) {
|
||||
if len(rendered.Messages) != 1 {
|
||||
b.Fatalf("artifact render message count=%d, want 1", len(rendered.Messages))
|
||||
}
|
||||
if rendered.Messages[0].Content != bodyText {
|
||||
b.Fatalf("artifact render length=%d, want %d", len(rendered.Messages[0].Content), len(bodyText))
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
repeatedSession := `document-size={{len (input "document")}}`
|
||||
repeatedMessages := []string{
|
||||
`{{input "document"}}`,
|
||||
`{{input "document"}}|{{input "document"}}`,
|
||||
}
|
||||
repeatedDefinition := &domain.PromptDefinition{
|
||||
ID: "compiled-render-benchmark", Version: "v1", SessionID: repeatedSession,
|
||||
Inputs: []domain.PromptInput{{Name: "document"}},
|
||||
Templates: []domain.PromptMessageTemplate{
|
||||
{Role: "system", Content: repeatedMessages[0]},
|
||||
{Role: "user", Content: repeatedMessages[1]},
|
||||
},
|
||||
}
|
||||
repeatedOutputBytes := len(bodyText)*3 + 1
|
||||
repeated := newCompiledRenderingBenchmarkCase(b,
|
||||
"repeated_across_session_and_messages", repeatedSession, repeatedMessages,
|
||||
repeatedDefinition, artifactInputs, nil, repeatedOutputBytes,
|
||||
func(b *testing.B, rendered *domain.RenderedPrompt) {
|
||||
if rendered.SessionID != "document-size="+strconv.Itoa(len(bodyText)) || len(rendered.Messages) != 2 ||
|
||||
rendered.Messages[0].Content != bodyText || rendered.Messages[1].Content != bodyText+"|"+bodyText {
|
||||
b.Fatalf("unexpected repeated compiled render: session=%q message lengths=%v", rendered.SessionID, renderedMessageLengths(rendered))
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
for _, benchmark := range []compiledRenderingBenchmarkCase{small, large, repeated} {
|
||||
b.Run(benchmark.name, func(b *testing.B) {
|
||||
rendered, err := benchmark.renderer.Render(benchmark.ctx, benchmark.definition, benchmark.inputs, benchmark.vars)
|
||||
if err != nil {
|
||||
b.Fatalf("check compiled renderer output: %v", err)
|
||||
}
|
||||
benchmark.check(b, rendered)
|
||||
|
||||
b.ReportAllocs()
|
||||
b.SetBytes(int64(benchmark.outputBytes))
|
||||
b.ResetTimer()
|
||||
for range b.N {
|
||||
rendered, err := benchmark.renderer.Render(benchmark.ctx, benchmark.definition, benchmark.inputs, benchmark.vars)
|
||||
if err != nil {
|
||||
b.Fatalf("render compiled template: %v", err)
|
||||
}
|
||||
renderedPromptBenchmarkSink = rendered
|
||||
}
|
||||
b.StopTimer()
|
||||
runtime.KeepAlive(renderedPromptBenchmarkSink)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// BenchmarkRetainedRenderedOutput reports post-GC live heap for a fixed batch
|
||||
// of rendered outputs above the same fixture's setup-only baseline.
|
||||
func BenchmarkRetainedRenderedOutput(b *testing.B) {
|
||||
body := bytes.Repeat([]byte("x"), rendererBenchmarkArtifactSize)
|
||||
bodyText := string(body)
|
||||
content := `{{input "document"}}`
|
||||
definition := &domain.PromptDefinition{
|
||||
ID: "retained-render-benchmark", Version: "v1",
|
||||
Inputs: []domain.PromptInput{{Name: "document"}},
|
||||
Templates: []domain.PromptMessageTemplate{{Role: "user", Content: content}},
|
||||
}
|
||||
compiled := CompiledTemplates{Messages: []*template.Template{mustCompileRendererBenchmarkTemplate(b, content)}}
|
||||
renderer := NewCompiledRenderer(rendererBenchmarkLookup{
|
||||
identity: rendererBenchmarkIdentity{id: definition.ID, version: definition.Version},
|
||||
templates: compiled,
|
||||
})
|
||||
inputs := map[string]*domain.Artifact{"document": {Name: "document", Body: body}}
|
||||
ctx := context.Background()
|
||||
const retainedBatchSize = 4
|
||||
retained := make([]*domain.RenderedPrompt, retainedBatchSize)
|
||||
|
||||
check, err := renderer.Render(ctx, definition, inputs, nil)
|
||||
if err != nil {
|
||||
b.Fatalf("check retained-output fixture: %v", err)
|
||||
}
|
||||
if len(check.Messages) != 1 || check.Messages[0].Content != bodyText {
|
||||
b.Fatalf("retained-output fixture returned unexpected content")
|
||||
}
|
||||
check = nil
|
||||
// Repeated -count runs must start from the same setup-only heap baseline.
|
||||
renderedPromptBenchmarkSink = nil
|
||||
renderedPromptsBenchmarkSink = nil
|
||||
|
||||
runtime.GC()
|
||||
var setupMemory runtime.MemStats
|
||||
runtime.ReadMemStats(&setupMemory)
|
||||
runtime.KeepAlive(renderer)
|
||||
runtime.KeepAlive(definition)
|
||||
runtime.KeepAlive(inputs)
|
||||
|
||||
b.ReportAllocs()
|
||||
b.SetBytes(int64(len(bodyText)))
|
||||
b.ResetTimer()
|
||||
for iteration := 0; iteration < b.N; iteration++ {
|
||||
rendered, err := renderer.Render(ctx, definition, inputs, nil)
|
||||
if err != nil {
|
||||
b.Fatalf("render retained-output fixture: %v", err)
|
||||
}
|
||||
retained[iteration%len(retained)] = rendered
|
||||
}
|
||||
b.StopTimer()
|
||||
for i, rendered := range retained {
|
||||
if rendered != nil {
|
||||
continue
|
||||
}
|
||||
rendered, err := renderer.Render(ctx, definition, inputs, nil)
|
||||
if err != nil {
|
||||
b.Fatalf("fill retained-output fixture: %v", err)
|
||||
}
|
||||
retained[i] = rendered
|
||||
}
|
||||
renderedPromptsBenchmarkSink = retained
|
||||
runtime.GC()
|
||||
var retainedMemory runtime.MemStats
|
||||
runtime.ReadMemStats(&retainedMemory)
|
||||
b.ReportMetric(float64(int64(retainedMemory.HeapAlloc)-int64(setupMemory.HeapAlloc)), "retained-B")
|
||||
runtime.KeepAlive(retained)
|
||||
runtime.KeepAlive(renderer)
|
||||
runtime.KeepAlive(definition)
|
||||
runtime.KeepAlive(inputs)
|
||||
}
|
||||
|
||||
func newCompiledRenderingBenchmarkCase(
|
||||
b *testing.B,
|
||||
name string,
|
||||
session string,
|
||||
messages []string,
|
||||
definition *domain.PromptDefinition,
|
||||
inputs map[string]*domain.Artifact,
|
||||
vars map[string]string,
|
||||
outputBytes int,
|
||||
check func(*testing.B, *domain.RenderedPrompt),
|
||||
) compiledRenderingBenchmarkCase {
|
||||
compiled := CompiledTemplates{}
|
||||
if session != "" {
|
||||
compiled.Session = mustCompileRendererBenchmarkTemplate(b, session)
|
||||
}
|
||||
for _, content := range messages {
|
||||
compiled.Messages = append(compiled.Messages, mustCompileRendererBenchmarkTemplate(b, content))
|
||||
}
|
||||
return compiledRenderingBenchmarkCase{
|
||||
name: name,
|
||||
renderer: NewCompiledRenderer(rendererBenchmarkLookup{
|
||||
identity: rendererBenchmarkIdentity{id: definition.ID, version: definition.Version},
|
||||
templates: compiled,
|
||||
}),
|
||||
definition: definition, inputs: inputs, vars: vars, ctx: context.Background(),
|
||||
outputBytes: outputBytes, check: check,
|
||||
}
|
||||
}
|
||||
|
||||
func mustCompileRendererBenchmarkTemplate(b *testing.B, source string) *template.Template {
|
||||
b.Helper()
|
||||
compiled, err := CompileTemplate("benchmark", source)
|
||||
if err != nil {
|
||||
b.Fatalf("compile benchmark template: %v", err)
|
||||
}
|
||||
return compiled
|
||||
}
|
||||
|
||||
func renderedMessageLengths(rendered *domain.RenderedPrompt) []int {
|
||||
lengths := make([]int, len(rendered.Messages))
|
||||
for i, message := range rendered.Messages {
|
||||
lengths[i] = len(message.Content)
|
||||
}
|
||||
return lengths
|
||||
}
|
||||
@@ -1,7 +1,6 @@
|
||||
package prompt
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
@@ -126,7 +125,7 @@ func (r *goRenderer) Render(ctx context.Context, definition *domain.PromptDefini
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var buf bytes.Buffer
|
||||
var buf strings.Builder
|
||||
executeErr := tmpl.Execute(&buf, vars)
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
@@ -235,7 +234,7 @@ func executeSessionTemplate(ctx context.Context, tmpl *template.Template, vars m
|
||||
if err := ctx.Err(); err != nil {
|
||||
return "", err
|
||||
}
|
||||
var buf bytes.Buffer
|
||||
var buf strings.Builder
|
||||
executeErr := tmpl.Execute(&buf, vars)
|
||||
if err := ctx.Err(); err != nil {
|
||||
return "", err
|
||||
|
||||
@@ -4,8 +4,11 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"text/template"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
)
|
||||
@@ -501,6 +504,77 @@ func TestGoRendererArtifactTextLifecycle(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompiledRendererKeepsConcurrentRequestsIndependent(t *testing.T) {
|
||||
const (
|
||||
promptID = "concurrent-render"
|
||||
version = "v1"
|
||||
)
|
||||
sessionTemplate, err := CompileTemplate("session", "session-{{.request}}")
|
||||
if err != nil {
|
||||
t.Fatalf("compile session template: %v", err)
|
||||
}
|
||||
messageTemplate, err := CompileTemplate("message", `request={{.request}};document={{input "document"}}`)
|
||||
if err != nil {
|
||||
t.Fatalf("compile message template: %v", err)
|
||||
}
|
||||
renderer := NewCompiledRenderer(compiledRendererTestLookup{
|
||||
id: promptID,
|
||||
version: version,
|
||||
templates: CompiledTemplates{
|
||||
Session: sessionTemplate,
|
||||
Messages: []*template.Template{messageTemplate},
|
||||
},
|
||||
})
|
||||
definition := &domain.PromptDefinition{
|
||||
ID: promptID, Version: version, SessionID: "session-{{.request}}",
|
||||
Inputs: []domain.PromptInput{{Name: "document"}},
|
||||
Templates: []domain.PromptMessageTemplate{{Role: "user", Content: `request={{.request}};document={{input "document"}}`}},
|
||||
}
|
||||
|
||||
const requestCount = 24
|
||||
errs := make(chan error, requestCount)
|
||||
var wait sync.WaitGroup
|
||||
wait.Add(requestCount)
|
||||
for i := 0; i < requestCount; i++ {
|
||||
go func(i int) {
|
||||
defer wait.Done()
|
||||
request := fmt.Sprintf("request-%d", i)
|
||||
document := fmt.Sprintf("document-%d", i)
|
||||
rendered, err := renderer.Render(context.Background(), definition,
|
||||
map[string]*domain.Artifact{"document": {Body: []byte(document)}},
|
||||
map[string]string{"request": request},
|
||||
)
|
||||
if err != nil {
|
||||
errs <- fmt.Errorf("render %s: %w", request, err)
|
||||
return
|
||||
}
|
||||
if rendered.SessionID != "session-"+request || len(rendered.Messages) != 1 ||
|
||||
rendered.Messages[0].Content != "request="+request+";document="+document {
|
||||
errs <- fmt.Errorf("render %s returned session %q and messages %#v", request, rendered.SessionID, rendered.Messages)
|
||||
}
|
||||
}(i)
|
||||
}
|
||||
wait.Wait()
|
||||
close(errs)
|
||||
for err := range errs {
|
||||
t.Error(err)
|
||||
}
|
||||
}
|
||||
|
||||
type compiledRendererTestLookup struct {
|
||||
id, version string
|
||||
templates CompiledTemplates
|
||||
}
|
||||
|
||||
func (lookup compiledRendererTestLookup) GetCompiledTemplates(id, version string) (CompiledTemplates, bool) {
|
||||
if id != lookup.id || version != lookup.version {
|
||||
return CompiledTemplates{}, false
|
||||
}
|
||||
return lookup.templates, true
|
||||
}
|
||||
|
||||
// BenchmarkGoRendererArtifactReferences measures NewGoRenderer's per-call
|
||||
// template-parsing fallback rather than the captured compiled path.
|
||||
func BenchmarkGoRendererArtifactReferences(b *testing.B) {
|
||||
body := bytes.Repeat([]byte("document content "), (artifactTextChunkSize*4)/len("document content "))
|
||||
inputs := map[string]*domain.Artifact{
|
||||
|
||||
978
internal/prompt/requirements.go
Normal file
978
internal/prompt/requirements.go
Normal file
@@ -0,0 +1,978 @@
|
||||
package prompt
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sort"
|
||||
"text/template"
|
||||
"text/template/parse"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
)
|
||||
|
||||
type requirementReachability uint8
|
||||
|
||||
const (
|
||||
requirementDefinite requirementReachability = iota
|
||||
requirementPossible
|
||||
requirementInactive
|
||||
)
|
||||
|
||||
type dotIdentity uint8
|
||||
|
||||
const (
|
||||
dotRoot dotIdentity = iota
|
||||
dotNonRoot
|
||||
dotUnknown
|
||||
)
|
||||
|
||||
type inputEvidence struct {
|
||||
name string
|
||||
kind domain.RequirementKind
|
||||
declared bool
|
||||
applicable bool
|
||||
contentType *string
|
||||
description *string
|
||||
explanation string
|
||||
}
|
||||
|
||||
type variableEvidence struct {
|
||||
name string
|
||||
kind domain.RequirementKind
|
||||
applicable bool
|
||||
explanation string
|
||||
}
|
||||
|
||||
type requirementIssueKey struct {
|
||||
code domain.RequirementIssueCode
|
||||
inputName string
|
||||
variableName string
|
||||
}
|
||||
|
||||
type conditionEvaluation struct {
|
||||
value any
|
||||
known bool
|
||||
unsupported bool
|
||||
inputNameUnknown bool
|
||||
missingVariables map[string]struct{}
|
||||
contentInputs map[string]struct{}
|
||||
}
|
||||
|
||||
func (evaluation conditionEvaluation) truth() bool {
|
||||
truth, ok := template.IsTrue(evaluation.value)
|
||||
return evaluation.known && ok && truth
|
||||
}
|
||||
|
||||
func (walker *requirementWalker) evaluateCondition(pipe *parse.PipeNode, reach requirementReachability, dot dotIdentity) conditionEvaluation {
|
||||
if pipe == nil {
|
||||
return conditionEvaluation{unsupported: true}
|
||||
}
|
||||
if len(pipe.Cmds) == 1 && len(pipe.Decl) == 0 && !pipe.IsAssign {
|
||||
return walker.evaluateCommand(pipe.Cmds[0], reach, dot)
|
||||
}
|
||||
if name, ok := directInputPipeline(pipe); ok {
|
||||
return walker.evaluateInputName(name, reach, dot)
|
||||
}
|
||||
before := walker.dependencies
|
||||
visitReach := requirementPossible
|
||||
if reach == requirementInactive {
|
||||
visitReach = requirementInactive
|
||||
}
|
||||
if err := walker.visitPipe(pipe, visitReach, dot); err != nil {
|
||||
return conditionEvaluation{unsupported: true}
|
||||
}
|
||||
return conditionEvaluation{unsupported: walker.dependencies != before}
|
||||
}
|
||||
|
||||
func (walker *requirementWalker) evaluateCommand(command *parse.CommandNode, reach requirementReachability, dot dotIdentity) conditionEvaluation {
|
||||
if command == nil || len(command.Args) == 0 {
|
||||
return conditionEvaluation{unsupported: true}
|
||||
}
|
||||
identifier, isIdentifier := command.Args[0].(*parse.IdentifierNode)
|
||||
if isIdentifier {
|
||||
switch identifier.Ident {
|
||||
case "input":
|
||||
if reach == requirementInactive {
|
||||
if len(command.Args) == 2 {
|
||||
if name, ok := literalStringValue(command.Args[1]); ok {
|
||||
walker.recordInput(name, requirementInactive)
|
||||
return conditionEvaluation{}
|
||||
}
|
||||
}
|
||||
for _, argument := range command.Args[1:] {
|
||||
_ = walker.visit(argument, requirementInactive, dot)
|
||||
}
|
||||
return conditionEvaluation{}
|
||||
}
|
||||
if len(command.Args) == 2 {
|
||||
if name, ok := command.Args[1].(*parse.StringNode); ok {
|
||||
walker.recordInput(name.Text, reach)
|
||||
return conditionEvaluation{contentInputs: map[string]struct{}{name.Text: {}}}
|
||||
}
|
||||
return walker.evaluateInputName(command.Args[1], reach, dot)
|
||||
}
|
||||
for _, argument := range command.Args[1:] {
|
||||
_ = walker.visit(argument, requirementPossible, dot)
|
||||
}
|
||||
return walker.unresolvedInputName(conditionEvaluation{unsupported: true})
|
||||
case "index":
|
||||
if len(command.Args) == 3 {
|
||||
if key, ok := command.Args[2].(*parse.StringNode); ok && isRootMap(command.Args[1], dot, walker.rootKnown) {
|
||||
walker.recordOptionalVariable(key.Text, reach)
|
||||
value, supplied := walker.vars[key.Text]
|
||||
if !supplied {
|
||||
value = ""
|
||||
}
|
||||
return conditionEvaluation{value: value, known: true}
|
||||
}
|
||||
}
|
||||
case "not":
|
||||
if len(command.Args) == 2 {
|
||||
evaluation := walker.evaluateValue(command.Args[1], reach, dot)
|
||||
if evaluation.known {
|
||||
truth, ok := template.IsTrue(evaluation.value)
|
||||
if ok {
|
||||
evaluation.value = !truth
|
||||
} else {
|
||||
evaluation.known = false
|
||||
evaluation.unsupported = true
|
||||
}
|
||||
}
|
||||
return evaluation
|
||||
}
|
||||
case "eq", "ne":
|
||||
if len(command.Args) == 3 {
|
||||
left := walker.evaluateValue(command.Args[1], reach, dot)
|
||||
right := walker.evaluateValue(command.Args[2], reach, dot)
|
||||
result := mergeConditionEvaluations(left, right)
|
||||
if left.known && right.known {
|
||||
leftString, leftOK := left.value.(string)
|
||||
rightString, rightOK := right.value.(string)
|
||||
if leftOK && rightOK {
|
||||
equal := leftString == rightString
|
||||
if identifier.Ident == "ne" {
|
||||
equal = !equal
|
||||
}
|
||||
result.value = equal
|
||||
result.known = true
|
||||
return result
|
||||
}
|
||||
result.unsupported = true
|
||||
}
|
||||
return result
|
||||
}
|
||||
}
|
||||
}
|
||||
if !isIdentifier && len(command.Args) == 1 {
|
||||
return walker.evaluateValue(command.Args[0], reach, dot)
|
||||
}
|
||||
if reach == requirementInactive {
|
||||
_ = walker.visitCommand(command, reach, dot)
|
||||
return conditionEvaluation{}
|
||||
}
|
||||
before := walker.dependencies
|
||||
_ = walker.visitCommand(command, requirementPossible, dot)
|
||||
return conditionEvaluation{unsupported: walker.dependencies != before}
|
||||
}
|
||||
|
||||
func (walker *requirementWalker) evaluateValue(node parse.Node, reach requirementReachability, dot dotIdentity) conditionEvaluation {
|
||||
switch node := node.(type) {
|
||||
case *parse.DotNode:
|
||||
if dot == dotRoot {
|
||||
return conditionEvaluation{value: walker.vars, known: true}
|
||||
}
|
||||
if dot == dotUnknown {
|
||||
walker.unresolvedVariableScope(reach)
|
||||
}
|
||||
return conditionEvaluation{unsupported: true}
|
||||
case *parse.StringNode:
|
||||
return conditionEvaluation{value: node.Text, known: true}
|
||||
case *parse.BoolNode:
|
||||
return conditionEvaluation{value: node.True, known: true}
|
||||
case *parse.NilNode:
|
||||
return conditionEvaluation{value: nil, known: true}
|
||||
case *parse.FieldNode:
|
||||
if dot != dotRoot || len(node.Ident) == 0 {
|
||||
if dot == dotUnknown {
|
||||
walker.unresolvedVariableScope(reach)
|
||||
}
|
||||
return conditionEvaluation{unsupported: true}
|
||||
}
|
||||
name := node.Ident[0]
|
||||
walker.recordVariable(name, reach)
|
||||
if len(node.Ident) > 1 {
|
||||
walker.addIssue("", name, "A chained field access is outside the direct-key scope.")
|
||||
return conditionEvaluation{unsupported: true}
|
||||
}
|
||||
value, supplied := walker.vars[name]
|
||||
if !supplied {
|
||||
return conditionEvaluation{missingVariables: map[string]struct{}{name: {}}}
|
||||
}
|
||||
return conditionEvaluation{value: value, known: true}
|
||||
case *parse.VariableNode:
|
||||
if len(node.Ident) == 1 && node.Ident[0] == "$" && walker.rootKnown {
|
||||
return conditionEvaluation{value: walker.vars, known: true}
|
||||
}
|
||||
if len(node.Ident) < 2 || node.Ident[0] != "$" || !walker.rootKnown {
|
||||
walker.visitVariable(node, reach)
|
||||
return conditionEvaluation{unsupported: true}
|
||||
}
|
||||
name := node.Ident[1]
|
||||
walker.recordVariable(name, reach)
|
||||
if len(node.Ident) > 2 {
|
||||
walker.addIssue("", name, "A chained field access is outside the direct-key scope.")
|
||||
return conditionEvaluation{unsupported: true}
|
||||
}
|
||||
value, supplied := walker.vars[name]
|
||||
if !supplied {
|
||||
return conditionEvaluation{missingVariables: map[string]struct{}{name: {}}}
|
||||
}
|
||||
return conditionEvaluation{value: value, known: true}
|
||||
case *parse.CommandNode:
|
||||
return walker.evaluateCommand(node, reach, dot)
|
||||
case *parse.PipeNode:
|
||||
return walker.evaluateCondition(node, reach, dot)
|
||||
case *parse.ChainNode:
|
||||
before := walker.dependencies
|
||||
visitReach := requirementPossible
|
||||
if reach == requirementInactive {
|
||||
visitReach = requirementInactive
|
||||
}
|
||||
_ = walker.visit(node.Node, visitReach, dot)
|
||||
return conditionEvaluation{unsupported: walker.dependencies != before}
|
||||
default:
|
||||
return conditionEvaluation{unsupported: true}
|
||||
}
|
||||
}
|
||||
|
||||
func (walker *requirementWalker) evaluateInputName(node parse.Node, reach requirementReachability, dot dotIdentity) conditionEvaluation {
|
||||
evaluation := walker.evaluateValue(node, reach, dot)
|
||||
if evaluation.known {
|
||||
if name, ok := evaluation.value.(string); ok {
|
||||
walker.recordInput(name, reach)
|
||||
return conditionEvaluation{contentInputs: map[string]struct{}{name: {}}}
|
||||
}
|
||||
evaluation.unsupported = true
|
||||
}
|
||||
// Value analysis has already visited recognizable references, including
|
||||
// the conservative fallback for unsupported expressions. Do not rescan
|
||||
// the same subtree: nested computed names would multiply that work.
|
||||
return walker.unresolvedInputName(evaluation)
|
||||
}
|
||||
|
||||
func (walker *requirementWalker) unresolvedInputName(evaluation conditionEvaluation) conditionEvaluation {
|
||||
for name := range evaluation.missingVariables {
|
||||
walker.addIssueCode(domain.RequirementIssueMissingVariable, "", name, "A missing variable prevents resolving an input name.")
|
||||
}
|
||||
for name := range evaluation.contentInputs {
|
||||
walker.addIssueCode(domain.RequirementIssueInputContent, name, "", "Captured input content affects an input name.")
|
||||
}
|
||||
walker.addIssueCode(domain.RequirementIssueDynamicInputName, "", "", "An input helper name is not statically available.")
|
||||
walker.unknownInputEffect = true
|
||||
evaluation.inputNameUnknown = true
|
||||
return evaluation
|
||||
}
|
||||
|
||||
func directInputPipeline(pipe *parse.PipeNode) (parse.Node, bool) {
|
||||
if pipe == nil || len(pipe.Cmds) != 2 || len(pipe.Decl) != 0 || pipe.IsAssign {
|
||||
return nil, false
|
||||
}
|
||||
input := pipe.Cmds[1]
|
||||
if len(input.Args) != 1 {
|
||||
return nil, false
|
||||
}
|
||||
identifier, ok := input.Args[0].(*parse.IdentifierNode)
|
||||
if !ok || identifier.Ident != "input" {
|
||||
return nil, false
|
||||
}
|
||||
value := pipe.Cmds[0]
|
||||
if len(value.Args) > 0 {
|
||||
valueIdentifier, isValueIdentifier := value.Args[0].(*parse.IdentifierNode)
|
||||
if isValueIdentifier && valueIdentifier.Ident == "input" && len(value.Args) == 2 {
|
||||
return value, true
|
||||
}
|
||||
}
|
||||
if len(value.Args) == 1 {
|
||||
return value.Args[0], true
|
||||
}
|
||||
if len(value.Args) == 3 {
|
||||
function, ok := value.Args[0].(*parse.IdentifierNode)
|
||||
if ok && function.Ident == "index" {
|
||||
return value, true
|
||||
}
|
||||
}
|
||||
return nil, false
|
||||
}
|
||||
|
||||
func literalStringValue(node parse.Node) (string, bool) {
|
||||
switch node := node.(type) {
|
||||
case *parse.StringNode:
|
||||
return node.Text, true
|
||||
case *parse.PipeNode:
|
||||
if len(node.Cmds) == 1 && len(node.Cmds[0].Args) == 1 && len(node.Decl) == 0 && !node.IsAssign {
|
||||
return literalStringValue(node.Cmds[0].Args[0])
|
||||
}
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
func mergeConditionEvaluations(left, right conditionEvaluation) conditionEvaluation {
|
||||
result := conditionEvaluation{
|
||||
unsupported: left.unsupported || right.unsupported,
|
||||
inputNameUnknown: left.inputNameUnknown || right.inputNameUnknown,
|
||||
missingVariables: make(map[string]struct{}, len(left.missingVariables)+len(right.missingVariables)),
|
||||
contentInputs: make(map[string]struct{}, len(left.contentInputs)+len(right.contentInputs)),
|
||||
}
|
||||
for name := range left.missingVariables {
|
||||
result.missingVariables[name] = struct{}{}
|
||||
}
|
||||
for name := range right.missingVariables {
|
||||
result.missingVariables[name] = struct{}{}
|
||||
}
|
||||
for name := range left.contentInputs {
|
||||
result.contentInputs[name] = struct{}{}
|
||||
}
|
||||
for name := range right.contentInputs {
|
||||
result.contentInputs[name] = struct{}{}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func branchReach(parent requirementReachability, known, truth, selected bool) requirementReachability {
|
||||
if parent == requirementInactive {
|
||||
return requirementInactive
|
||||
}
|
||||
if !known {
|
||||
return requirementPossible
|
||||
}
|
||||
if truth == selected {
|
||||
if parent == requirementPossible {
|
||||
return requirementPossible
|
||||
}
|
||||
return requirementDefinite
|
||||
}
|
||||
return requirementInactive
|
||||
}
|
||||
|
||||
type requirementWalker struct {
|
||||
ctx context.Context
|
||||
vars map[string]string
|
||||
inputs map[string]*inputEvidence
|
||||
variables map[string]*variableEvidence
|
||||
issues map[requirementIssueKey]domain.RequirementIssue
|
||||
complete bool
|
||||
rootKnown bool
|
||||
unknownInputEffect bool
|
||||
unknownVariableEffect bool
|
||||
dependencies int
|
||||
templateSet *template.Template
|
||||
visitedTrees map[*parse.Tree]bool
|
||||
}
|
||||
|
||||
// AnalyzeRequirements collects the prompt dependencies recognizable from its
|
||||
// captured syntax. Branch bodies remain conditional until their conditions are
|
||||
// classified; unsupported dependency-bearing syntax makes the result partial.
|
||||
func AnalyzeRequirements(
|
||||
ctx context.Context,
|
||||
definition *domain.PromptDefinition,
|
||||
snapshot RequirementSnapshot,
|
||||
vars map[string]string,
|
||||
sessionBypassed bool,
|
||||
) (domain.RequirementInspection, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return domain.RequirementInspection{}, err
|
||||
}
|
||||
result := domain.RequirementInspection{Complete: true}
|
||||
if definition != nil {
|
||||
result.PromptID = definition.ID
|
||||
result.PromptVersion = definition.Version
|
||||
}
|
||||
walker := &requirementWalker{
|
||||
ctx: ctx,
|
||||
vars: vars,
|
||||
inputs: make(map[string]*inputEvidence, len(snapshot.Inputs)),
|
||||
variables: make(map[string]*variableEvidence),
|
||||
issues: make(map[requirementIssueKey]domain.RequirementIssue),
|
||||
complete: true,
|
||||
rootKnown: true,
|
||||
}
|
||||
for _, input := range snapshot.Inputs {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return domain.RequirementInspection{}, err
|
||||
}
|
||||
kind := domain.RequirementOptional
|
||||
if input.Required {
|
||||
kind = domain.RequirementRequired
|
||||
}
|
||||
walker.inputs[input.Name] = &inputEvidence{
|
||||
name: input.Name,
|
||||
kind: kind,
|
||||
declared: true,
|
||||
contentType: copyString(input.ContentType),
|
||||
description: copyString(input.Description),
|
||||
}
|
||||
if kind == domain.RequirementOptional {
|
||||
walker.inputs[input.Name].explanation = "Declared as an optional input."
|
||||
}
|
||||
}
|
||||
for _, message := range snapshot.Templates.Messages {
|
||||
if err := walker.walkTemplate(message); err != nil {
|
||||
return domain.RequirementInspection{}, err
|
||||
}
|
||||
}
|
||||
if !sessionBypassed && snapshot.Templates.Session != nil {
|
||||
if err := walker.walkTemplate(snapshot.Templates.Session); err != nil {
|
||||
return domain.RequirementInspection{}, err
|
||||
}
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return domain.RequirementInspection{}, err
|
||||
}
|
||||
return walker.result(result), nil
|
||||
}
|
||||
|
||||
func (walker *requirementWalker) walkTemplate(compiled *template.Template) error {
|
||||
if err := walker.ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if compiled == nil || compiled.Tree == nil || compiled.Tree.Root == nil {
|
||||
walker.addIssue("", "", "A captured template has no parsed root.")
|
||||
return nil
|
||||
}
|
||||
previousSet := walker.templateSet
|
||||
previousVisited := walker.visitedTrees
|
||||
walker.templateSet = compiled
|
||||
walker.visitedTrees = map[*parse.Tree]bool{compiled.Tree: true}
|
||||
walker.rootKnown = true
|
||||
err := walker.visit(compiled.Tree.Root, requirementDefinite, dotRoot)
|
||||
walker.templateSet = previousSet
|
||||
walker.visitedTrees = previousVisited
|
||||
return err
|
||||
}
|
||||
|
||||
func (walker *requirementWalker) visit(node parse.Node, reach requirementReachability, dot dotIdentity) error {
|
||||
if err := walker.ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if node == nil {
|
||||
return nil
|
||||
}
|
||||
switch node := node.(type) {
|
||||
case *parse.ListNode:
|
||||
if node == nil {
|
||||
return nil
|
||||
}
|
||||
for _, child := range node.Nodes {
|
||||
if err := walker.visit(child, reach, dot); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
case *parse.ActionNode:
|
||||
return walker.visit(node.Pipe, reach, dot)
|
||||
case *parse.IfNode:
|
||||
if reach == requirementInactive {
|
||||
if err := walker.visit(node.Pipe, requirementInactive, dot); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := walker.visit(node.List, requirementInactive, dot); err != nil {
|
||||
return err
|
||||
}
|
||||
return walker.visit(node.ElseList, requirementInactive, dot)
|
||||
}
|
||||
condition := walker.evaluateCondition(node.Pipe, reach, dot)
|
||||
before := walker.dependencies
|
||||
thenReach := branchReach(reach, condition.known, condition.truth(), true)
|
||||
elseReach := branchReach(reach, condition.known, condition.truth(), false)
|
||||
if err := walker.visit(node.List, thenReach, dot); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := walker.visit(node.ElseList, elseReach, dot); err != nil {
|
||||
return err
|
||||
}
|
||||
if !condition.known && walker.dependencies != before {
|
||||
walker.recordConditionLimitation(condition)
|
||||
}
|
||||
return nil
|
||||
case *parse.WithNode:
|
||||
if reach == requirementInactive {
|
||||
if err := walker.visit(node.Pipe, requirementInactive, dot); err != nil {
|
||||
return err
|
||||
}
|
||||
bodyDot := walker.withDotIdentity(node.Pipe, dot, conditionEvaluation{})
|
||||
if err := walker.visit(node.List, requirementInactive, bodyDot); err != nil {
|
||||
return err
|
||||
}
|
||||
return walker.visit(node.ElseList, requirementInactive, dot)
|
||||
}
|
||||
condition := walker.evaluateCondition(node.Pipe, reach, dot)
|
||||
before := walker.dependencies
|
||||
thenReach := branchReach(reach, condition.known, condition.truth(), true)
|
||||
elseReach := branchReach(reach, condition.known, condition.truth(), false)
|
||||
bodyDot := walker.withDotIdentity(node.Pipe, dot, condition)
|
||||
if err := walker.visit(node.List, thenReach, bodyDot); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := walker.visit(node.ElseList, elseReach, dot); err != nil {
|
||||
return err
|
||||
}
|
||||
if !condition.known && walker.dependencies != before {
|
||||
walker.recordConditionLimitation(condition)
|
||||
}
|
||||
return nil
|
||||
case *parse.RangeNode:
|
||||
if err := walker.visit(node.Pipe, reach, dot); err != nil {
|
||||
return err
|
||||
}
|
||||
before := walker.dependencies
|
||||
bodyReach := requirementPossible
|
||||
if reach == requirementInactive {
|
||||
bodyReach = requirementInactive
|
||||
}
|
||||
if err := walker.visit(node.List, bodyReach, dotNonRoot); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := walker.visit(node.ElseList, bodyReach, dot); err != nil {
|
||||
return err
|
||||
}
|
||||
if reach != requirementInactive && walker.dependencies != before {
|
||||
walker.addIssue("", "", "Dependencies inside a range are only conservatively enumerated.")
|
||||
}
|
||||
case *parse.TemplateNode:
|
||||
pipeReach := requirementPossible
|
||||
if reach == requirementInactive {
|
||||
pipeReach = requirementInactive
|
||||
}
|
||||
if err := walker.visit(node.Pipe, pipeReach, dot); err != nil {
|
||||
return err
|
||||
}
|
||||
if reach == requirementInactive {
|
||||
return nil
|
||||
}
|
||||
called := walker.templateSet.Lookup(node.Name)
|
||||
if called == nil || called.Tree == nil || called.Tree.Root == nil {
|
||||
walker.addIssue("", "", "An invoked named template is unavailable for syntax inspection.")
|
||||
return nil
|
||||
}
|
||||
if walker.visitedTrees[called.Tree] {
|
||||
walker.addIssue("", "", "An invoked named template is scanned conservatively.")
|
||||
return nil
|
||||
}
|
||||
walker.visitedTrees[called.Tree] = true
|
||||
rootKnown := walker.rootKnown
|
||||
walker.rootKnown = false
|
||||
walker.unknownVariableEffect = true
|
||||
err := walker.visit(called.Tree.Root, requirementPossible, dotNonRoot)
|
||||
walker.rootKnown = rootKnown
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
walker.addIssue("", "", "An invoked named template is scanned conservatively.")
|
||||
case *parse.PipeNode:
|
||||
return walker.visitPipe(node, reach, dot)
|
||||
case *parse.CommandNode:
|
||||
return walker.visitCommand(node, reach, dot)
|
||||
case *parse.FieldNode:
|
||||
if dot == dotRoot && len(node.Ident) > 0 {
|
||||
walker.recordVariable(node.Ident[0], reach)
|
||||
if len(node.Ident) > 1 && reach != requirementInactive {
|
||||
walker.addIssue("", node.Ident[0], "A chained field access is outside the direct-key scope.")
|
||||
}
|
||||
}
|
||||
if dot == dotUnknown {
|
||||
walker.unresolvedVariableScope(reach)
|
||||
}
|
||||
case *parse.DotNode:
|
||||
if dot == dotUnknown {
|
||||
walker.unresolvedVariableScope(reach)
|
||||
}
|
||||
case *parse.VariableNode:
|
||||
walker.visitVariable(node, reach)
|
||||
case *parse.ChainNode:
|
||||
before := walker.dependencies
|
||||
chainReach := requirementPossible
|
||||
if reach == requirementInactive {
|
||||
chainReach = requirementInactive
|
||||
}
|
||||
if err := walker.visit(node.Node, chainReach, dot); err != nil {
|
||||
return err
|
||||
}
|
||||
if reach != requirementInactive && walker.dependencies != before {
|
||||
walker.addIssue("", "", "A chained expression is not interpreted.")
|
||||
}
|
||||
case *parse.BreakNode, *parse.ContinueNode:
|
||||
if reach != requirementInactive {
|
||||
walker.addIssue("", "", "Loop control is outside the supported syntax scope.")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (walker *requirementWalker) visitPipe(pipe *parse.PipeNode, reach requirementReachability, dot dotIdentity) error {
|
||||
if pipe == nil {
|
||||
return nil
|
||||
}
|
||||
if len(pipe.Cmds) == 1 && len(pipe.Decl) == 0 && !pipe.IsAssign {
|
||||
return walker.visitCommand(pipe.Cmds[0], reach, dot)
|
||||
}
|
||||
if name, ok := directInputPipeline(pipe); ok {
|
||||
if reach == requirementInactive {
|
||||
if literal, known := literalStringValue(name); known {
|
||||
walker.recordInput(literal, requirementInactive)
|
||||
return nil
|
||||
}
|
||||
return walker.visit(name, requirementInactive, dot)
|
||||
}
|
||||
walker.evaluateInputName(name, reach, dot)
|
||||
return nil
|
||||
}
|
||||
before := walker.dependencies
|
||||
commandReach := requirementPossible
|
||||
if reach == requirementInactive {
|
||||
commandReach = requirementInactive
|
||||
}
|
||||
for _, command := range pipe.Cmds {
|
||||
if err := walker.visitCommand(command, commandReach, dot); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if reach != requirementInactive && walker.dependencies != before {
|
||||
walker.addIssue("", "", "A pipeline or variable assignment is not interpreted.")
|
||||
}
|
||||
if reach != requirementInactive {
|
||||
for _, declaration := range pipe.Decl {
|
||||
if len(declaration.Ident) > 0 && declaration.Ident[0] == "$" {
|
||||
walker.rootKnown = false
|
||||
walker.unknownVariableEffect = true
|
||||
walker.addIssue("", "", "The root template binding is reassigned.")
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (walker *requirementWalker) visitCommand(command *parse.CommandNode, reach requirementReachability, dot dotIdentity) error {
|
||||
if command == nil || len(command.Args) == 0 {
|
||||
return nil
|
||||
}
|
||||
identifier, isIdentifier := command.Args[0].(*parse.IdentifierNode)
|
||||
if isIdentifier && identifier.Ident == "input" {
|
||||
if reach == requirementInactive {
|
||||
if len(command.Args) == 2 {
|
||||
if name, ok := literalStringValue(command.Args[1]); ok {
|
||||
walker.recordInput(name, reach)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
for _, argument := range command.Args[1:] {
|
||||
if err := walker.visit(argument, requirementInactive, dot); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if len(command.Args) == 2 {
|
||||
if name, ok := command.Args[1].(*parse.StringNode); ok {
|
||||
walker.recordInput(name.Text, reach)
|
||||
return nil
|
||||
}
|
||||
walker.evaluateInputName(command.Args[1], reach, dot)
|
||||
return nil
|
||||
}
|
||||
for _, argument := range command.Args[1:] {
|
||||
if err := walker.visit(argument, requirementPossible, dot); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
walker.unresolvedInputName(conditionEvaluation{unsupported: true})
|
||||
return nil
|
||||
}
|
||||
if isIdentifier && identifier.Ident == "index" && len(command.Args) == 3 {
|
||||
if _, ok := command.Args[2].(*parse.StringNode); ok && isRootMap(command.Args[1], dot, walker.rootKnown) {
|
||||
key := command.Args[2].(*parse.StringNode).Text
|
||||
walker.recordOptionalVariable(key, reach)
|
||||
return nil
|
||||
}
|
||||
// A local map may alias the caller root even though its value is not
|
||||
// tracked. A lookup on it must not vanish from complete discovery.
|
||||
if variable, ok := unwrapSingleValue(command.Args[1]).(*parse.VariableNode); ok && len(variable.Ident) == 1 {
|
||||
walker.unresolvedVariableScope(reach)
|
||||
}
|
||||
}
|
||||
if !isIdentifier && len(command.Args) == 1 {
|
||||
return walker.visit(command.Args[0], reach, dot)
|
||||
}
|
||||
before := walker.dependencies
|
||||
for index, argument := range command.Args {
|
||||
if isIdentifier && index == 0 {
|
||||
continue
|
||||
}
|
||||
argumentReach := requirementPossible
|
||||
if reach == requirementInactive {
|
||||
argumentReach = requirementInactive
|
||||
}
|
||||
if err := walker.visit(argument, argumentReach, dot); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if reach != requirementInactive && walker.dependencies != before {
|
||||
walker.addIssue("", "", "A function call or computed expression is not interpreted.")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (walker *requirementWalker) visitVariable(variable *parse.VariableNode, reach requirementReachability) {
|
||||
if len(variable.Ident) < 2 {
|
||||
return
|
||||
}
|
||||
if variable.Ident[0] != "$" {
|
||||
walker.unresolvedVariableScope(reach)
|
||||
return
|
||||
}
|
||||
if !walker.rootKnown {
|
||||
if reach != requirementInactive {
|
||||
walker.unknownVariableEffect = true
|
||||
walker.addIssue("", "", "The root template binding is not known.")
|
||||
}
|
||||
return
|
||||
}
|
||||
walker.recordVariable(variable.Ident[1], reach)
|
||||
if len(variable.Ident) > 2 && reach != requirementInactive {
|
||||
walker.addIssue("", variable.Ident[1], "A chained field access is outside the direct-key scope.")
|
||||
}
|
||||
}
|
||||
|
||||
func (walker *requirementWalker) unresolvedVariableScope(reach requirementReachability) {
|
||||
if reach != requirementInactive {
|
||||
walker.unknownVariableEffect = true
|
||||
walker.addIssue("", "", "A caller-variable scope is not statically known.")
|
||||
}
|
||||
}
|
||||
|
||||
func (walker *requirementWalker) withDotIdentity(pipe *parse.PipeNode, dot dotIdentity, condition conditionEvaluation) dotIdentity {
|
||||
if pipe != nil && len(pipe.Cmds) == 1 && len(pipe.Decl) == 0 && !pipe.IsAssign && len(pipe.Cmds[0].Args) == 1 &&
|
||||
isRootMap(pipe.Cmds[0].Args[0], dot, walker.rootKnown) {
|
||||
return dotRoot
|
||||
}
|
||||
if condition.known {
|
||||
if _, root := condition.value.(map[string]string); root {
|
||||
return dotRoot
|
||||
}
|
||||
return dotNonRoot
|
||||
}
|
||||
if condition.inputNameUnknown ||
|
||||
!condition.unsupported && (len(condition.missingVariables) > 0 || len(condition.contentInputs) > 0) {
|
||||
// Supported caller values and input bodies are strings; supported
|
||||
// comparisons and negation return booleans, never the root map.
|
||||
return dotNonRoot
|
||||
}
|
||||
return dotUnknown
|
||||
}
|
||||
|
||||
func isRootMap(node parse.Node, dot dotIdentity, rootKnown bool) bool {
|
||||
switch node := unwrapSingleValue(node).(type) {
|
||||
case *parse.DotNode:
|
||||
return dot == dotRoot
|
||||
case *parse.VariableNode:
|
||||
return rootKnown && len(node.Ident) == 1 && node.Ident[0] == "$"
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func unwrapSingleValue(node parse.Node) parse.Node {
|
||||
for {
|
||||
pipe, ok := node.(*parse.PipeNode)
|
||||
if !ok || len(pipe.Cmds) != 1 || len(pipe.Decl) != 0 || pipe.IsAssign || len(pipe.Cmds[0].Args) != 1 {
|
||||
return node
|
||||
}
|
||||
node = pipe.Cmds[0].Args[0]
|
||||
}
|
||||
}
|
||||
|
||||
func (walker *requirementWalker) recordInput(name string, reach requirementReachability) {
|
||||
evidence := walker.inputs[name]
|
||||
if evidence == nil {
|
||||
evidence = &inputEvidence{
|
||||
name: name, kind: domain.RequirementOptional,
|
||||
explanation: "Declared as an optional input.",
|
||||
}
|
||||
walker.inputs[name] = evidence
|
||||
}
|
||||
if reach == requirementInactive {
|
||||
if !evidence.applicable && evidence.kind != domain.RequirementRequired {
|
||||
evidence.explanation = "This input is referenced only in a skipped template path."
|
||||
}
|
||||
} else {
|
||||
evidence.applicable = true
|
||||
walker.promoteInput(evidence, kindForReach(reach))
|
||||
if evidence.kind == domain.RequirementRequired {
|
||||
evidence.explanation = ""
|
||||
} else if reach == requirementPossible {
|
||||
evidence.explanation = "This reference is not known to be reached on every execution path."
|
||||
}
|
||||
}
|
||||
if reach != requirementInactive {
|
||||
walker.dependencies++
|
||||
}
|
||||
}
|
||||
|
||||
func (walker *requirementWalker) recordVariable(name string, reach requirementReachability) {
|
||||
evidence := walker.variables[name]
|
||||
if evidence == nil {
|
||||
evidence = &variableEvidence{name: name, kind: domain.RequirementOptional}
|
||||
walker.variables[name] = evidence
|
||||
}
|
||||
if reach == requirementInactive {
|
||||
if !evidence.applicable {
|
||||
evidence.explanation = "This variable is referenced only in a skipped template path."
|
||||
}
|
||||
} else {
|
||||
evidence.applicable = true
|
||||
walker.promoteVariable(evidence, kindForReach(reach))
|
||||
if evidence.kind == domain.RequirementRequired {
|
||||
evidence.explanation = ""
|
||||
} else if reach == requirementPossible {
|
||||
evidence.explanation = "This reference is not known to be reached on every execution path."
|
||||
}
|
||||
}
|
||||
if reach != requirementInactive {
|
||||
walker.dependencies++
|
||||
}
|
||||
}
|
||||
|
||||
func (walker *requirementWalker) recordOptionalVariable(name string, reach requirementReachability) {
|
||||
evidence := walker.variables[name]
|
||||
if evidence == nil {
|
||||
evidence = &variableEvidence{name: name, kind: domain.RequirementOptional}
|
||||
walker.variables[name] = evidence
|
||||
}
|
||||
if reach == requirementInactive {
|
||||
if !evidence.applicable {
|
||||
evidence.explanation = "This variable is referenced only in a skipped template path."
|
||||
}
|
||||
} else {
|
||||
evidence.applicable = true
|
||||
}
|
||||
if evidence.kind == domain.RequirementOptional && reach != requirementInactive {
|
||||
evidence.explanation = "This key is read through an optional map lookup."
|
||||
}
|
||||
if reach != requirementInactive {
|
||||
walker.dependencies++
|
||||
}
|
||||
}
|
||||
|
||||
func (walker *requirementWalker) promoteInput(evidence *inputEvidence, kind domain.RequirementKind) {
|
||||
if requirementRank(kind) > requirementRank(evidence.kind) {
|
||||
evidence.kind = kind
|
||||
}
|
||||
if evidence.kind == domain.RequirementRequired {
|
||||
evidence.explanation = ""
|
||||
}
|
||||
}
|
||||
|
||||
func (walker *requirementWalker) promoteVariable(evidence *variableEvidence, kind domain.RequirementKind) {
|
||||
if requirementRank(kind) > requirementRank(evidence.kind) {
|
||||
evidence.kind = kind
|
||||
}
|
||||
if evidence.kind == domain.RequirementRequired {
|
||||
evidence.explanation = ""
|
||||
}
|
||||
}
|
||||
|
||||
func kindForReach(reach requirementReachability) domain.RequirementKind {
|
||||
if reach == requirementPossible {
|
||||
return domain.RequirementConditional
|
||||
}
|
||||
return domain.RequirementRequired
|
||||
}
|
||||
|
||||
func requirementRank(kind domain.RequirementKind) int {
|
||||
switch kind {
|
||||
case domain.RequirementRequired:
|
||||
return 3
|
||||
case domain.RequirementConditional:
|
||||
return 2
|
||||
default:
|
||||
return 1
|
||||
}
|
||||
}
|
||||
|
||||
func (walker *requirementWalker) addIssue(inputName, variableName, description string) {
|
||||
walker.addIssueCode(domain.RequirementIssueUnsupportedConstruct, inputName, variableName, description)
|
||||
}
|
||||
|
||||
func (walker *requirementWalker) addIssueCode(code domain.RequirementIssueCode, inputName, variableName, description string) {
|
||||
walker.complete = false
|
||||
issue := domain.RequirementIssue{
|
||||
Code: code,
|
||||
InputName: inputName,
|
||||
VariableName: variableName,
|
||||
Description: description,
|
||||
}
|
||||
key := requirementIssueKey{code: issue.Code, inputName: inputName, variableName: variableName}
|
||||
walker.issues[key] = issue
|
||||
}
|
||||
|
||||
func (walker *requirementWalker) recordConditionLimitation(condition conditionEvaluation) {
|
||||
for name := range condition.missingVariables {
|
||||
walker.addIssueCode(domain.RequirementIssueMissingVariable, "", name, "A missing variable prevents selecting a dependency-bearing branch.")
|
||||
}
|
||||
for name := range condition.contentInputs {
|
||||
walker.addIssueCode(domain.RequirementIssueInputContent, name, "", "Captured input content affects branch selection.")
|
||||
}
|
||||
if condition.unsupported && !condition.inputNameUnknown ||
|
||||
(len(condition.missingVariables) == 0 && len(condition.contentInputs) == 0 && !condition.inputNameUnknown) {
|
||||
walker.addIssue("", "", "A dependency-bearing branch condition is not resolved by the supported syntax analysis.")
|
||||
}
|
||||
}
|
||||
|
||||
func (walker *requirementWalker) result(result domain.RequirementInspection) domain.RequirementInspection {
|
||||
result.Complete = walker.complete
|
||||
result.Inputs = make([]domain.InputRequirement, 0, len(walker.inputs))
|
||||
for _, evidence := range walker.inputs {
|
||||
if walker.unknownInputEffect && evidence.kind != domain.RequirementRequired {
|
||||
evidence.kind = domain.RequirementConditional
|
||||
evidence.explanation = "An unresolved input name could refer to this input."
|
||||
}
|
||||
result.Inputs = append(result.Inputs, domain.InputRequirement{
|
||||
Name: evidence.name, Kind: evidence.kind, Declared: evidence.declared,
|
||||
ContentType: copyString(evidence.contentType), Description: copyString(evidence.description),
|
||||
Explanation: evidence.explanation,
|
||||
})
|
||||
}
|
||||
sort.Slice(result.Inputs, func(i, j int) bool { return result.Inputs[i].Name < result.Inputs[j].Name })
|
||||
result.Variables = make([]domain.VariableRequirement, 0, len(walker.variables))
|
||||
for _, evidence := range walker.variables {
|
||||
if walker.unknownVariableEffect && evidence.kind != domain.RequirementRequired {
|
||||
evidence.kind = domain.RequirementConditional
|
||||
evidence.explanation = "An unresolved caller-variable scope could refer to this variable."
|
||||
}
|
||||
_, supplied := walker.vars[evidence.name]
|
||||
result.Variables = append(result.Variables, domain.VariableRequirement{
|
||||
Name: evidence.name, Kind: evidence.kind, Supplied: supplied, Explanation: evidence.explanation,
|
||||
})
|
||||
}
|
||||
sort.Slice(result.Variables, func(i, j int) bool { return result.Variables[i].Name < result.Variables[j].Name })
|
||||
result.Issues = make([]domain.RequirementIssue, 0, len(walker.issues))
|
||||
for _, issue := range walker.issues {
|
||||
result.Issues = append(result.Issues, issue)
|
||||
}
|
||||
sort.Slice(result.Issues, func(i, j int) bool {
|
||||
left, right := result.Issues[i], result.Issues[j]
|
||||
if left.Code != right.Code {
|
||||
return left.Code < right.Code
|
||||
}
|
||||
if left.InputName != right.InputName {
|
||||
return left.InputName < right.InputName
|
||||
}
|
||||
return left.VariableName < right.VariableName
|
||||
})
|
||||
return result
|
||||
}
|
||||
|
||||
func copyString(value *string) *string {
|
||||
if value == nil {
|
||||
return nil
|
||||
}
|
||||
copy := *value
|
||||
return ©
|
||||
}
|
||||
671
internal/prompt/requirements_test.go
Normal file
671
internal/prompt/requirements_test.go
Normal file
@@ -0,0 +1,671 @@
|
||||
package prompt
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"text/template"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
)
|
||||
|
||||
func TestAnalyzeRequirementsDoesNotHideCallerMapScopes(t *testing.T) {
|
||||
for _, source := range []string{
|
||||
`{{with .}}{{.audience}}{{end}}`,
|
||||
`{{with $}}{{.audience}}{{end}}`,
|
||||
`{{with (.)}}{{.audience}}{{end}}`,
|
||||
`{{$alias := .}}{{with $alias}}{{.audience}}{{end}}`,
|
||||
`{{with .}}{{index . "audience"}}{{end}}`,
|
||||
`{{$alias := .}}{{index $alias "audience"}}`,
|
||||
`{{$alias := .}}{{index ($alias) "audience"}}`,
|
||||
`{{$alias := .}}{{$alias.audience}}`,
|
||||
} {
|
||||
t.Run(source, func(t *testing.T) {
|
||||
compiled := compileRequirementTemplate(t, "scope", source)
|
||||
vars := map[string]string{"audience": "visible"}
|
||||
var rendered strings.Builder
|
||||
if err := compiled.Execute(&rendered, vars); err != nil || rendered.String() != "visible" {
|
||||
t.Fatalf("caller-map reference did not render: %q, %v", rendered.String(), err)
|
||||
}
|
||||
result, err := AnalyzeRequirements(context.Background(), &domain.PromptDefinition{}, RequirementSnapshot{
|
||||
Templates: CompiledTemplates{Messages: []*template.Template{compiled}},
|
||||
}, vars, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result.Complete {
|
||||
if len(result.Variables) != 1 || result.Variables[0].Name != "audience" {
|
||||
t.Fatalf("complete discovery lost the caller key: %+v", result)
|
||||
}
|
||||
} else if len(result.Issues) == 0 {
|
||||
t.Fatalf("partial discovery did not explain its limitation: %+v", result)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnalyzeRequirementsUnknownScopesAffectOptionalEntriesOnlyWhenActive(t *testing.T) {
|
||||
for _, access := range []string{`{{with $alias}}{{.audience}}{{end}}`, `{{index $alias "audience"}}`, `{{index ($alias) "audience"}}`, `{{$alias.audience}}`} {
|
||||
for _, active := range []bool{false, true} {
|
||||
source := `{{if false}}{{.optional}}{{end}}{{$alias := .}}` + access
|
||||
if !active {
|
||||
source = `{{if false}}` + source + `{{end}}`
|
||||
}
|
||||
compiled := compileRequirementTemplate(t, "unknown-scope", source)
|
||||
result, err := AnalyzeRequirements(context.Background(), &domain.PromptDefinition{}, RequirementSnapshot{
|
||||
Templates: CompiledTemplates{Messages: []*template.Template{compiled}},
|
||||
}, map[string]string{"audience": "visible"}, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wantKind := domain.RequirementOptional
|
||||
if active {
|
||||
wantKind = domain.RequirementConditional
|
||||
}
|
||||
if result.Complete == active || len(result.Variables) != 1 || result.Variables[0].Name != "optional" || result.Variables[0].Kind != wantKind {
|
||||
t.Fatalf("active=%v, source=%s, result=%+v", active, source, result)
|
||||
}
|
||||
if active && (len(result.Issues) == 0 || result.Issues[0].Code != domain.RequirementIssueUnsupportedConstruct) {
|
||||
t.Fatalf("active unknown scope lacks a structured issue: %+v", result)
|
||||
}
|
||||
if !active && len(result.Issues) != 0 {
|
||||
t.Fatalf("skipped unknown scope introduced uncertainty: %+v", result)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRequirementWalkerDoesNotRescanNestedUnsupportedNames(t *testing.T) {
|
||||
for _, depth := range []int{6, 12} {
|
||||
expression := ".selector"
|
||||
for level := 0; level < depth; level++ {
|
||||
expression = `input (printf "%s" (` + expression + `))`
|
||||
}
|
||||
source := "{{" + expression + "}}"
|
||||
compiled := compileRequirementTemplate(t, "computed-name", source)
|
||||
walker := &requirementWalker{
|
||||
ctx: context.Background(), vars: map[string]string{"selector": "binding"},
|
||||
inputs: make(map[string]*inputEvidence), variables: make(map[string]*variableEvidence),
|
||||
issues: make(map[requirementIssueKey]domain.RequirementIssue), complete: true, rootKnown: true,
|
||||
}
|
||||
if err := walker.walkTemplate(compiled); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// A generous source-relative bound detects exponential rescanning, not
|
||||
// an exact visit count or a production analysis limit.
|
||||
if walker.dependencies > len(source)*8 {
|
||||
t.Fatalf("depth=%d: %d accesses for %d bytes of source", depth, walker.dependencies, len(source))
|
||||
}
|
||||
result := walker.result(domain.RequirementInspection{})
|
||||
if result.Complete || len(result.Variables) != 1 || result.Variables[0].Name != "selector" ||
|
||||
result.Variables[0].Kind != domain.RequirementConditional || len(result.Issues) == 0 {
|
||||
t.Fatalf("nested unsupported names lost discovery metadata: %+v", result)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnalyzeRequirementsCollectsDeclaredAndLiteralInputs(t *testing.T) {
|
||||
description := "retained metadata"
|
||||
definition := &domain.PromptDefinition{ID: "capture", Version: "v1"}
|
||||
snapshot := RequirementSnapshot{
|
||||
Inputs: []InputMetadata{
|
||||
{Name: "optional", ContentType: stringPointerForTest("text/plain"), Description: &description},
|
||||
{Name: "unused", Required: true},
|
||||
{Name: "transcript", Required: true},
|
||||
},
|
||||
Templates: CompiledTemplates{Messages: []*template.Template{
|
||||
compileRequirementTemplate(t, "message", `{{input "transcript"}} {{input "extra"}} {{input "extra"}}`),
|
||||
}},
|
||||
}
|
||||
result, err := AnalyzeRequirements(context.Background(), definition, snapshot, nil, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result.PromptID != "capture" || result.PromptVersion != "v1" || !result.Complete || len(result.Issues) != 0 {
|
||||
t.Fatalf("unexpected result identity or completeness: %+v", result)
|
||||
}
|
||||
wantNames := []string{"extra", "optional", "transcript", "unused"}
|
||||
if got := requirementInputNames(result.Inputs); !reflect.DeepEqual(got, wantNames) {
|
||||
t.Fatalf("input names = %v, want %v", got, wantNames)
|
||||
}
|
||||
if got := result.Inputs[0]; got.Kind != domain.RequirementRequired || got.Declared || got.ContentType != nil || got.Description != nil {
|
||||
t.Fatalf("undeclared literal input was not required independently: %+v", got)
|
||||
}
|
||||
if got := result.Inputs[1]; got.Kind != domain.RequirementOptional || !got.Declared || got.ContentType == nil || *got.ContentType != "text/plain" ||
|
||||
got.Description == nil || *got.Description != description || got.Explanation == "" {
|
||||
t.Fatalf("optional declaration metadata was not retained: %+v", got)
|
||||
}
|
||||
if result.Inputs[2].Kind != domain.RequirementRequired || !result.Inputs[2].Declared || result.Inputs[3].Kind != domain.RequirementRequired {
|
||||
t.Fatalf("required declarations or literal use were lost: %+v", result.Inputs)
|
||||
}
|
||||
*result.Inputs[1].Description = "caller mutation"
|
||||
if *snapshot.Inputs[0].Description != description {
|
||||
t.Fatal("result metadata pointer aliases the captured snapshot")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnalyzeRequirementsDistinguishesDirectAndOptionalCallerKeys(t *testing.T) {
|
||||
result, err := AnalyzeRequirements(context.Background(), &domain.PromptDefinition{}, RequirementSnapshot{
|
||||
Templates: CompiledTemplates{Messages: []*template.Template{
|
||||
compileRequirementTemplate(t, "message", `{{.missing}} {{$.present}} {{index . "lookup"}}`),
|
||||
}},
|
||||
}, map[string]string{"present": ""}, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !result.Complete || len(result.Issues) != 0 {
|
||||
t.Fatalf("plain root and literal lookup syntax should be complete: %+v", result)
|
||||
}
|
||||
wantNames := []string{"lookup", "missing", "present"}
|
||||
if got := requirementVariableNames(result.Variables); !reflect.DeepEqual(got, wantNames) {
|
||||
t.Fatalf("variable names = %v, want %v", got, wantNames)
|
||||
}
|
||||
if result.Variables[0].Kind != domain.RequirementOptional || result.Variables[0].Supplied || result.Variables[0].Explanation == "" {
|
||||
t.Errorf("literal index lookup should be optional and absent: %+v", result.Variables[0])
|
||||
}
|
||||
if result.Variables[1].Kind != domain.RequirementRequired || result.Variables[1].Supplied {
|
||||
t.Errorf("missing direct variable should be required but not supplied: %+v", result.Variables[1])
|
||||
}
|
||||
if result.Variables[2].Kind != domain.RequirementRequired || !result.Variables[2].Supplied {
|
||||
t.Errorf("empty supplied direct variable was not detected by presence: %+v", result.Variables[2])
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnalyzeRequirementsSeparatesRootAndLocalTemplateValues(t *testing.T) {
|
||||
result, err := AnalyzeRequirements(context.Background(), &domain.PromptDefinition{}, RequirementSnapshot{
|
||||
Templates: CompiledTemplates{Messages: []*template.Template{
|
||||
compileRequirementTemplate(t, "message", `{{$local := "local"}}{{with .context}}{{$local}} {{.child}} {{$.root}}{{end}}`),
|
||||
}},
|
||||
}, nil, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wantNames := []string{"context", "root"}
|
||||
if got := requirementVariableNames(result.Variables); !reflect.DeepEqual(got, wantNames) {
|
||||
t.Fatalf("root variable names = %v, want %v", got, wantNames)
|
||||
}
|
||||
if result.Variables[0].Kind != domain.RequirementRequired || result.Variables[1].Kind != domain.RequirementConditional || result.Complete {
|
||||
t.Fatalf("with scope and unresolved branch should be conservative: %+v", result)
|
||||
}
|
||||
if len(result.Issues) != 1 || result.Issues[0].Code != domain.RequirementIssueMissingVariable || result.Issues[0].VariableName != "context" {
|
||||
t.Fatalf("conditional root use should have one associated limitation: %+v", result.Issues)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnalyzeRequirementsMarksBranchAndUnsupportedScopeReferences(t *testing.T) {
|
||||
result, err := AnalyzeRequirements(context.Background(), &domain.PromptDefinition{}, RequirementSnapshot{
|
||||
Templates: CompiledTemplates{Messages: []*template.Template{
|
||||
compileRequirementTemplate(t, "message", `{{if .enabled}}{{input "maybe"}}{{end}}{{range .items}}{{input "inside"}} {{.local}} {{$.root}}{{end}}`),
|
||||
}},
|
||||
}, nil, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wantInputs := []string{"inside", "maybe"}
|
||||
wantVars := []string{"enabled", "items", "root"}
|
||||
if got := requirementInputNames(result.Inputs); !reflect.DeepEqual(got, wantInputs) {
|
||||
t.Fatalf("input names = %v, want %v", got, wantInputs)
|
||||
}
|
||||
if got := requirementVariableNames(result.Variables); !reflect.DeepEqual(got, wantVars) {
|
||||
t.Fatalf("variable names = %v, want %v", got, wantVars)
|
||||
}
|
||||
if result.Complete || len(result.Issues) < 2 {
|
||||
t.Fatalf("branch/range limitations were not reported: %+v", result)
|
||||
}
|
||||
for _, input := range result.Inputs {
|
||||
if input.Kind != domain.RequirementConditional {
|
||||
t.Errorf("input %q should be conservative conditional: %+v", input.Name, input)
|
||||
}
|
||||
}
|
||||
if result.Variables[0].Kind != domain.RequirementRequired || result.Variables[1].Kind != domain.RequirementRequired ||
|
||||
result.Variables[2].Kind != domain.RequirementConditional {
|
||||
t.Errorf("root and loop-scope variables were misclassified: %+v", result.Variables)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnalyzeRequirementsExcludesUnusedDefinitionsAndBypassedSession(t *testing.T) {
|
||||
message := compileRequirementTemplate(t, "message", `{{define "unused"}}{{input "unused"}}{{end}}main`)
|
||||
session := compileRequirementTemplate(t, "session", `{{input "session_only"}}`)
|
||||
result, err := AnalyzeRequirements(context.Background(), &domain.PromptDefinition{}, RequirementSnapshot{
|
||||
Templates: CompiledTemplates{Messages: []*template.Template{message}, Session: session},
|
||||
}, nil, true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !result.Complete || len(result.Inputs) != 0 {
|
||||
t.Fatalf("unused definitions or bypassed session were included: %+v", result)
|
||||
}
|
||||
result, err = AnalyzeRequirements(context.Background(), &domain.PromptDefinition{}, RequirementSnapshot{
|
||||
Templates: CompiledTemplates{Messages: []*template.Template{message}, Session: session},
|
||||
}, nil, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(result.Inputs) != 1 || result.Inputs[0].Name != "session_only" || result.Inputs[0].Kind != domain.RequirementRequired {
|
||||
t.Fatalf("applicable session template was not visited: %+v", result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnalyzeRequirementsScansInvokedDefinitionsConservatively(t *testing.T) {
|
||||
message := compileRequirementTemplate(t, "message", `{{define "unused"}}{{input "unused"}}{{end}}{{define "called"}}{{input "called"}} {{.local}}{{end}}main {{template "called" .}}`)
|
||||
result, err := AnalyzeRequirements(context.Background(), &domain.PromptDefinition{}, RequirementSnapshot{
|
||||
Templates: CompiledTemplates{Messages: []*template.Template{message}},
|
||||
}, nil, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result.Complete || len(result.Inputs) != 1 || result.Inputs[0].Name != "called" ||
|
||||
result.Inputs[0].Kind != domain.RequirementConditional || len(result.Variables) != 0 {
|
||||
t.Fatalf("called definition was not scanned conservatively: %+v", result)
|
||||
}
|
||||
if len(result.Issues) < 1 || result.Issues[0].Code != domain.RequirementIssueUnsupportedConstruct {
|
||||
t.Fatalf("named-template limitation was not reported: %+v", result.Issues)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnalyzeRequirementsClassifiesKnownConditions(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
condition string
|
||||
vars map[string]string
|
||||
wantActive string
|
||||
wantSkipped string
|
||||
}{
|
||||
{name: "string false is truthy", condition: `.flag`, vars: map[string]string{"flag": "false"}, wantActive: "active", wantSkipped: "inactive"},
|
||||
{name: "empty direct value is false", condition: `.flag`, vars: map[string]string{"flag": ""}, wantActive: "inactive", wantSkipped: "active"},
|
||||
{name: "simple equality", condition: `eq .mode "active"`, vars: map[string]string{"mode": "active"}, wantActive: "active", wantSkipped: "inactive"},
|
||||
{name: "simple inequality", condition: `ne .mode "active"`, vars: map[string]string{"mode": "other"}, wantActive: "active", wantSkipped: "inactive"},
|
||||
{name: "not direct value", condition: `not .flag`, vars: map[string]string{"flag": ""}, wantActive: "active", wantSkipped: "inactive"},
|
||||
{name: "literal true", condition: `true`, wantActive: "active", wantSkipped: "inactive"},
|
||||
{name: "literal false", condition: `false`, wantActive: "inactive", wantSkipped: "active"},
|
||||
{name: "missing index is empty", condition: `index . "toggle"`, wantActive: "inactive", wantSkipped: "active"},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
definition := &domain.PromptDefinition{Templates: []domain.PromptMessageTemplate{{
|
||||
Role: "user", Content: `{{if ` + test.condition + `}}{{input "active"}}{{else}}{{input "inactive"}}{{end}}`,
|
||||
}}}
|
||||
snapshot := RequirementSnapshot{Templates: CompiledTemplates{Messages: []*template.Template{
|
||||
compileRequirementTemplate(t, "message", definition.Templates[0].Content),
|
||||
}}}
|
||||
result, err := AnalyzeRequirements(context.Background(), definition, snapshot, test.vars, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !result.Complete || len(result.Issues) != 0 {
|
||||
t.Fatalf("known condition should produce a complete result: %+v", result)
|
||||
}
|
||||
inputs := make(map[string]domain.InputRequirement, len(result.Inputs))
|
||||
for _, input := range result.Inputs {
|
||||
inputs[input.Name] = input
|
||||
}
|
||||
if inputs[test.wantActive].Kind != domain.RequirementRequired || inputs[test.wantSkipped].Kind != domain.RequirementOptional || inputs[test.wantSkipped].Explanation == "" {
|
||||
t.Fatalf("selected/skipped inputs were misclassified: %+v", result.Inputs)
|
||||
}
|
||||
if test.condition == `index . "toggle"` {
|
||||
if len(result.Variables) != 1 || result.Variables[0].Name != "toggle" || result.Variables[0].Kind != domain.RequirementOptional || result.Variables[0].Supplied {
|
||||
t.Fatalf("missing index key was not retained as optional: %+v", result.Variables)
|
||||
}
|
||||
}
|
||||
|
||||
artifacts := map[string]*domain.Artifact{
|
||||
"active": {Name: "active", Body: []byte("active")},
|
||||
"inactive": {Name: "inactive", Body: []byte("inactive")},
|
||||
}
|
||||
rendered, err := NewGoRenderer().Render(context.Background(), definition, artifacts, test.vars)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
selected := "active"
|
||||
if inputs[selected].Kind != domain.RequirementRequired {
|
||||
selected = "inactive"
|
||||
}
|
||||
if len(rendered.Messages) != 1 || rendered.Messages[0].Content != selected {
|
||||
t.Fatalf("analysis and renderer disagree: result=%+v rendered=%+v", result, rendered)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnalyzeRequirementsHandlesWithDotAndNestedReachability(t *testing.T) {
|
||||
definition := &domain.PromptDefinition{Templates: []domain.PromptMessageTemplate{{
|
||||
Role: "user", Content: `{{with .context}}{{input "inside"}} {{.child}} {{$.root}}{{else}}{{input "empty"}}{{end}}`,
|
||||
}}}
|
||||
snapshot := RequirementSnapshot{Templates: CompiledTemplates{Messages: []*template.Template{
|
||||
compileRequirementTemplate(t, "message", definition.Templates[0].Content),
|
||||
}}}
|
||||
result, err := AnalyzeRequirements(context.Background(), definition, snapshot, map[string]string{"context": "value", "root": "root"}, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !result.Complete || len(result.Inputs) != 2 || len(result.Variables) != 2 {
|
||||
t.Fatalf("known with condition should have a complete inventory: %+v", result)
|
||||
}
|
||||
if result.Inputs[0].Name != "empty" || result.Inputs[0].Kind != domain.RequirementOptional ||
|
||||
result.Inputs[1].Name != "inside" || result.Inputs[1].Kind != domain.RequirementRequired {
|
||||
t.Fatalf("with body and else reachability is incorrect: %+v", result.Inputs)
|
||||
}
|
||||
if result.Variables[0].Name != "context" || result.Variables[0].Kind != domain.RequirementRequired ||
|
||||
result.Variables[1].Name != "root" || result.Variables[1].Kind != domain.RequirementRequired {
|
||||
t.Fatalf("with dot was confused with the root caller map: %+v", result.Variables)
|
||||
}
|
||||
|
||||
nested := compileRequirementTemplate(t, "nested", `{{if .outer}}{{if .inner}}{{input "inner"}}{{else}}{{input "inner_else"}}{{end}}{{else}}{{input "outer_else"}}{{end}}`)
|
||||
nestedResult, err := AnalyzeRequirements(context.Background(), &domain.PromptDefinition{}, RequirementSnapshot{
|
||||
Templates: CompiledTemplates{Messages: []*template.Template{nested}},
|
||||
}, map[string]string{"outer": "yes", "inner": "yes"}, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !nestedResult.Complete || nestedResult.Inputs[0].Name != "inner" || nestedResult.Inputs[0].Kind != domain.RequirementRequired ||
|
||||
nestedResult.Inputs[1].Name != "inner_else" || nestedResult.Inputs[1].Kind != domain.RequirementOptional ||
|
||||
nestedResult.Inputs[2].Name != "outer_else" || nestedResult.Inputs[2].Kind != domain.RequirementOptional {
|
||||
t.Fatalf("known nested conditions lost selected or inactive reachability: %+v", nestedResult)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnalyzeRequirementsReportsUnresolvedDecisionAndContentConditions(t *testing.T) {
|
||||
missing := compileRequirementTemplate(t, "missing", `{{if eq .mode "active"}}{{input "yes"}}{{else}}{{input "no"}}{{end}}`)
|
||||
missingResult, err := AnalyzeRequirements(context.Background(), &domain.PromptDefinition{}, RequirementSnapshot{
|
||||
Templates: CompiledTemplates{Messages: []*template.Template{missing}},
|
||||
}, nil, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if missingResult.Complete || len(missingResult.Issues) != 1 || missingResult.Issues[0].Code != domain.RequirementIssueMissingVariable ||
|
||||
missingResult.Issues[0].VariableName != "mode" || missingResult.Inputs[0].Kind != domain.RequirementConditional ||
|
||||
missingResult.Inputs[1].Kind != domain.RequirementConditional {
|
||||
t.Fatalf("missing decision value did not produce conditional dependencies: %+v", missingResult)
|
||||
}
|
||||
|
||||
content := compileRequirementTemplate(t, "content", `{{if input "selector"}}{{input "yes"}}{{else}}{{input "no"}}{{end}}`)
|
||||
contentResult, err := AnalyzeRequirements(context.Background(), &domain.PromptDefinition{}, RequirementSnapshot{
|
||||
Templates: CompiledTemplates{Messages: []*template.Template{content}},
|
||||
}, nil, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
contentInputs := make(map[string]domain.InputRequirement, len(contentResult.Inputs))
|
||||
for _, input := range contentResult.Inputs {
|
||||
contentInputs[input.Name] = input
|
||||
}
|
||||
if contentResult.Complete || len(contentResult.Issues) != 1 || contentResult.Issues[0].Code != domain.RequirementIssueInputContent ||
|
||||
contentResult.Issues[0].InputName != "selector" || contentInputs["selector"].Kind != domain.RequirementRequired ||
|
||||
contentInputs["yes"].Kind != domain.RequirementConditional || contentInputs["no"].Kind != domain.RequirementConditional {
|
||||
t.Fatalf("content-dependent condition did not preserve known dependencies: %+v", contentResult)
|
||||
}
|
||||
|
||||
textOnly := compileRequirementTemplate(t, "text-only", `{{if .missing}}plain text{{else}}also plain text{{end}}`)
|
||||
textResult, err := AnalyzeRequirements(context.Background(), &domain.PromptDefinition{}, RequirementSnapshot{
|
||||
Templates: CompiledTemplates{Messages: []*template.Template{textOnly}},
|
||||
}, nil, false)
|
||||
if err != nil || !textResult.Complete || len(textResult.Issues) != 0 {
|
||||
t.Fatalf("unresolved text-only branch should not affect completeness: %+v, %v", textResult, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnalyzeRequirementsRetainsInactiveEntriesAndPromotesActiveUses(t *testing.T) {
|
||||
description := "declared for a reason"
|
||||
definition := &domain.PromptDefinition{Templates: []domain.PromptMessageTemplate{{
|
||||
Role: "user", Content: `{{if false}}{{input "shared"}} {{.shared_var}} {{input "declared"}}{{input .dynamic}}{{and .left .right}}{{end}}{{input "shared"}} {{.shared_var}}`,
|
||||
}}}
|
||||
snapshot := RequirementSnapshot{
|
||||
Inputs: []InputMetadata{{Name: "declared", Required: true, Description: &description}},
|
||||
Templates: CompiledTemplates{Messages: []*template.Template{
|
||||
compileRequirementTemplate(t, "message", definition.Templates[0].Content),
|
||||
}},
|
||||
}
|
||||
result, err := AnalyzeRequirements(context.Background(), definition, snapshot, map[string]string{"shared_var": "present"}, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !result.Complete || len(result.Issues) != 0 {
|
||||
t.Fatalf("skipped unsupported expressions introduced uncertainty: %+v", result)
|
||||
}
|
||||
inputs := make(map[string]domain.InputRequirement, len(result.Inputs))
|
||||
for _, input := range result.Inputs {
|
||||
inputs[input.Name] = input
|
||||
}
|
||||
if len(inputs) != 2 || inputs["shared"].Kind != domain.RequirementRequired || inputs["shared"].Explanation != "" ||
|
||||
inputs["declared"].Kind != domain.RequirementRequired || inputs["declared"].Description == nil || *inputs["declared"].Description != description {
|
||||
t.Fatalf("active or declared requirements did not override inactivity: %+v", result.Inputs)
|
||||
}
|
||||
variables := make(map[string]domain.VariableRequirement, len(result.Variables))
|
||||
for _, variable := range result.Variables {
|
||||
variables[variable.Name] = variable
|
||||
}
|
||||
for _, name := range []string{"dynamic", "left", "right", "shared_var"} {
|
||||
if _, ok := variables[name]; !ok {
|
||||
t.Errorf("inactive caller key %q was not inventoried", name)
|
||||
}
|
||||
}
|
||||
if variables["shared_var"].Kind != domain.RequirementRequired || variables["shared_var"].Explanation != "" ||
|
||||
variables["dynamic"].Kind != domain.RequirementOptional || variables["dynamic"].Explanation == "" {
|
||||
t.Fatalf("inactive caller metadata was not promoted or explained: %+v", result.Variables)
|
||||
}
|
||||
|
||||
allOptional := compileRequirementTemplate(t, "optional", `{{index . "lookup"}}`)
|
||||
optionalResult, err := AnalyzeRequirements(context.Background(), &domain.PromptDefinition{}, RequirementSnapshot{
|
||||
Inputs: []InputMetadata{{Name: "optional"}}, Templates: CompiledTemplates{Messages: []*template.Template{allOptional}},
|
||||
}, nil, false)
|
||||
if err != nil || !optionalResult.Complete || len(optionalResult.Issues) != 0 || optionalResult.Inputs[0].Kind != domain.RequirementOptional ||
|
||||
optionalResult.Variables[0].Kind != domain.RequirementOptional {
|
||||
t.Fatalf("all-optional inventory should be complete without artifacts: %+v, %v", optionalResult, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnalyzeRequirementsResolvesSimpleInputNames(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
expression string
|
||||
vars map[string]string
|
||||
kind domain.RequirementKind
|
||||
supplied bool
|
||||
}{
|
||||
{name: "direct dot", expression: `{{input .name}}`, vars: map[string]string{"name": " chosen "}, kind: domain.RequirementRequired, supplied: true},
|
||||
{name: "explicit root", expression: `{{input $.name}}`, vars: map[string]string{"name": "chosen"}, kind: domain.RequirementRequired, supplied: true},
|
||||
{name: "parenthesized direct", expression: `{{input (.name)}}`, vars: map[string]string{"name": "chosen"}, kind: domain.RequirementRequired, supplied: true},
|
||||
{name: "literal lookup", expression: `{{input (index . "name")}}`, vars: map[string]string{"name": "chosen"}, kind: domain.RequirementOptional, supplied: true},
|
||||
{name: "direct pipeline", expression: `{{.name | input}}`, vars: map[string]string{"name": "chosen"}, kind: domain.RequirementRequired, supplied: true},
|
||||
{name: "empty supplied value", expression: `{{input .name}}`, vars: map[string]string{"name": ""}, kind: domain.RequirementRequired, supplied: true},
|
||||
{name: "missing lookup has empty zero value", expression: `{{input (index . "name")}}`, vars: nil, kind: domain.RequirementOptional, supplied: false},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
definition := &domain.PromptDefinition{Templates: []domain.PromptMessageTemplate{{Role: "user", Content: test.expression}}}
|
||||
snapshot := RequirementSnapshot{Templates: CompiledTemplates{Messages: []*template.Template{
|
||||
compileRequirementTemplate(t, "message", test.expression),
|
||||
}}}
|
||||
result, err := AnalyzeRequirements(context.Background(), definition, snapshot, test.vars, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
inputName := "chosen"
|
||||
if test.name == "direct dot" {
|
||||
inputName = " chosen "
|
||||
}
|
||||
if test.name == "empty supplied value" || test.name == "missing lookup has empty zero value" {
|
||||
inputName = ""
|
||||
}
|
||||
if !result.Complete || len(result.Issues) != 0 || len(result.Inputs) != 1 || result.Inputs[0].Name != inputName ||
|
||||
result.Inputs[0].Kind != domain.RequirementRequired {
|
||||
t.Fatalf("simple input name was not resolved exactly: %+v", result)
|
||||
}
|
||||
if len(result.Variables) != 1 || result.Variables[0].Name != "name" || result.Variables[0].Kind != test.kind || result.Variables[0].Supplied != test.supplied {
|
||||
t.Fatalf("name-source caller key classification is incorrect: %+v", result.Variables)
|
||||
}
|
||||
artifacts := map[string]*domain.Artifact{inputName: {Name: inputName, Body: []byte("resolved")}}
|
||||
rendered, err := NewGoRenderer().Render(context.Background(), definition, artifacts, test.vars)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(rendered.Messages) != 1 || rendered.Messages[0].Content != "resolved" {
|
||||
t.Fatalf("analyzed input name differs from renderer: %+v", rendered)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnalyzeRequirementsReportsUnresolvedInputNamesAndPromotesUnknownEffects(t *testing.T) {
|
||||
definition := &domain.PromptDefinition{Templates: []domain.PromptMessageTemplate{{
|
||||
Role: "user", Content: `{{input .missing}}`,
|
||||
}}}
|
||||
result, err := AnalyzeRequirements(context.Background(), definition, RequirementSnapshot{
|
||||
Inputs: []InputMetadata{{Name: "optional"}, {Name: "required", Required: true}},
|
||||
Templates: CompiledTemplates{Messages: []*template.Template{
|
||||
compileRequirementTemplate(t, "message", definition.Templates[0].Content),
|
||||
}},
|
||||
}, nil, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result.Complete || len(result.Issues) != 2 || result.Issues[0].Code != domain.RequirementIssueDynamicInputName ||
|
||||
result.Issues[1].Code != domain.RequirementIssueMissingVariable {
|
||||
t.Fatalf("missing dynamic name did not report its limitations: %+v", result)
|
||||
}
|
||||
if result.Inputs[0].Name != "optional" || result.Inputs[0].Kind != domain.RequirementConditional || result.Inputs[0].Explanation == "" ||
|
||||
result.Inputs[1].Name != "required" || result.Inputs[1].Kind != domain.RequirementRequired || len(result.Variables) != 1 ||
|
||||
result.Variables[0].Name != "missing" || result.Variables[0].Kind != domain.RequirementRequired {
|
||||
t.Fatalf("unknown name effects lost known classifications: %+v", result)
|
||||
}
|
||||
|
||||
computed := compileRequirementTemplate(t, "computed", `{{input (printf "%s" .prefix)}}`)
|
||||
computedResult, err := AnalyzeRequirements(context.Background(), &domain.PromptDefinition{}, RequirementSnapshot{
|
||||
Inputs: []InputMetadata{{Name: "declared"}}, Templates: CompiledTemplates{Messages: []*template.Template{computed}},
|
||||
}, map[string]string{"prefix": "SENSITIVE-VALUE"}, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if computedResult.Complete || len(computedResult.Inputs) != 1 || computedResult.Inputs[0].Name != "declared" ||
|
||||
computedResult.Inputs[0].Kind != domain.RequirementConditional || len(computedResult.Variables) != 1 ||
|
||||
computedResult.Variables[0].Name != "prefix" || computedResult.Variables[0].Kind != domain.RequirementConditional {
|
||||
t.Fatalf("computed-name uncertainty did not affect optional entries: %+v", computedResult)
|
||||
}
|
||||
for _, issue := range computedResult.Issues {
|
||||
if strings.Contains(issue.Description, "SENSITIVE-VALUE") || issue.InputName == "SENSITIVE-VALUE" || issue.VariableName == "SENSITIVE-VALUE" {
|
||||
t.Fatalf("input-name source value leaked into diagnostics: %+v", issue)
|
||||
}
|
||||
}
|
||||
for _, input := range computedResult.Inputs {
|
||||
if strings.Contains(input.Explanation, "SENSITIVE-VALUE") {
|
||||
t.Fatalf("input-name source value leaked into an explanation: %+v", input)
|
||||
}
|
||||
}
|
||||
for _, variable := range computedResult.Variables {
|
||||
if strings.Contains(variable.Explanation, "SENSITIVE-VALUE") {
|
||||
t.Fatalf("input-name source value leaked into an explanation: %+v", variable)
|
||||
}
|
||||
}
|
||||
|
||||
for _, expression := range []string{`{{input (input "selector")}}`, `{{input "selector" | input}}`} {
|
||||
t.Run(expression, func(t *testing.T) {
|
||||
contentName := compileRequirementTemplate(t, "content-name", expression)
|
||||
contentResult, err := AnalyzeRequirements(context.Background(), &domain.PromptDefinition{}, RequirementSnapshot{
|
||||
Templates: CompiledTemplates{Messages: []*template.Template{contentName}},
|
||||
}, nil, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if contentResult.Complete || len(contentResult.Issues) != 2 ||
|
||||
contentResult.Issues[0].Code != domain.RequirementIssueDynamicInputName ||
|
||||
contentResult.Issues[1].Code != domain.RequirementIssueInputContent || contentResult.Issues[1].InputName != "selector" || len(contentResult.Inputs) != 1 ||
|
||||
contentResult.Inputs[0].Name != "selector" || contentResult.Inputs[0].Kind != domain.RequirementRequired {
|
||||
t.Fatalf("input-body-derived name did not preserve the known source dependency: %+v", contentResult)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnalyzeRequirementsDoesNotResolveSkippedComputedInputNames(t *testing.T) {
|
||||
compiled := compileRequirementTemplate(t, "inactive-name", `{{if false}}{{input .name}}{{input (printf "%s" .other)}}{{"piped-literal" | input}}{{template "missing" .arg}}{{end}}`)
|
||||
result, err := AnalyzeRequirements(context.Background(), &domain.PromptDefinition{}, RequirementSnapshot{
|
||||
Templates: CompiledTemplates{Messages: []*template.Template{compiled}},
|
||||
}, nil, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !result.Complete || len(result.Issues) != 0 || len(result.Inputs) != 1 || result.Inputs[0].Name != "piped-literal" ||
|
||||
result.Inputs[0].Kind != domain.RequirementOptional || result.Inputs[0].Explanation == "" || len(result.Variables) != 3 {
|
||||
t.Fatalf("skipped computed expressions introduced uncertainty or names: %+v", result)
|
||||
}
|
||||
for _, variable := range result.Variables {
|
||||
if variable.Kind != domain.RequirementOptional || variable.Explanation == "" {
|
||||
t.Errorf("skipped variable %q was not retained as inactive optional: %+v", variable.Name, variable)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnalyzeRequirementsPromotesOptionalVariablesWhenRootScopeIsUnknown(t *testing.T) {
|
||||
compiled := compileRequirementTemplate(t, "scope", `{{index . "optional"}} {{.required}} {{template "called" .}}{{define "called"}}{{$.hidden}}{{end}}`)
|
||||
result, err := AnalyzeRequirements(context.Background(), &domain.PromptDefinition{}, RequirementSnapshot{
|
||||
Templates: CompiledTemplates{Messages: []*template.Template{compiled}},
|
||||
}, map[string]string{"required": "value"}, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result.Complete || len(result.Variables) != 2 || result.Variables[0].Name != "optional" ||
|
||||
result.Variables[0].Kind != domain.RequirementConditional || result.Variables[0].Explanation == "" ||
|
||||
result.Variables[1].Name != "required" || result.Variables[1].Kind != domain.RequirementRequired {
|
||||
t.Fatalf("unknown caller scope did not affect only optional known entries: %+v", result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnalyzeRequirementsLeavesAssignmentDerivedNamesPartial(t *testing.T) {
|
||||
compiled := compileRequirementTemplate(t, "assignment", `{{$alias := .source}}{{input $alias}}`)
|
||||
result, err := AnalyzeRequirements(context.Background(), &domain.PromptDefinition{}, RequirementSnapshot{
|
||||
Inputs: []InputMetadata{{Name: "known-optional"}}, Templates: CompiledTemplates{Messages: []*template.Template{compiled}},
|
||||
}, nil, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result.Complete || len(result.Issues) == 0 || len(result.Inputs) != 1 ||
|
||||
result.Inputs[0].Name != "known-optional" || result.Inputs[0].Kind != domain.RequirementConditional ||
|
||||
len(result.Variables) != 1 || result.Variables[0].Name != "source" || result.Variables[0].Kind != domain.RequirementConditional {
|
||||
t.Fatalf("assignment-derived input name was interpreted or lost: %+v", result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnalyzeRequirementsSupportsEmptyTemplatesAndCancellation(t *testing.T) {
|
||||
result, err := AnalyzeRequirements(context.Background(), &domain.PromptDefinition{}, RequirementSnapshot{
|
||||
Templates: CompiledTemplates{Messages: []*template.Template{compileRequirementTemplate(t, "message", "plain text")}},
|
||||
}, nil, false)
|
||||
if err != nil || !result.Complete || len(result.Inputs) != 0 || len(result.Variables) != 0 {
|
||||
t.Fatalf("input-free template should have a complete empty inventory: %+v, %v", result, err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
if _, err := AnalyzeRequirements(ctx, &domain.PromptDefinition{}, RequirementSnapshot{}, nil, false); !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("canceled context returned %v, want context.Canceled", err)
|
||||
}
|
||||
}
|
||||
|
||||
func compileRequirementTemplate(t *testing.T, name, source string) *template.Template {
|
||||
t.Helper()
|
||||
compiled, err := CompileTemplate(name, source)
|
||||
if err != nil {
|
||||
t.Fatalf("compile template: %v", err)
|
||||
}
|
||||
return compiled
|
||||
}
|
||||
|
||||
func requirementInputNames(inputs []domain.InputRequirement) []string {
|
||||
names := make([]string, len(inputs))
|
||||
for index, input := range inputs {
|
||||
names[index] = input.Name
|
||||
}
|
||||
return names
|
||||
}
|
||||
|
||||
func requirementVariableNames(variables []domain.VariableRequirement) []string {
|
||||
names := make([]string, len(variables))
|
||||
for index, variable := range variables {
|
||||
names[index] = variable.Name
|
||||
}
|
||||
return names
|
||||
}
|
||||
|
||||
func stringPointerForTest(value string) *string {
|
||||
return &value
|
||||
}
|
||||
@@ -9,6 +9,22 @@ type CompiledTemplates struct {
|
||||
Messages []*template.Template
|
||||
}
|
||||
|
||||
// InputMetadata is the captured declaration metadata for one prompt input.
|
||||
// Nil metadata pointers mean the corresponding field was absent or null.
|
||||
type InputMetadata struct {
|
||||
Name string
|
||||
Required bool
|
||||
ContentType *string
|
||||
Description *string
|
||||
}
|
||||
|
||||
// RequirementSnapshot provides the captured syntax and declarations needed
|
||||
// for requirement inspection without reopening prompt resources.
|
||||
type RequirementSnapshot struct {
|
||||
Templates CompiledTemplates
|
||||
Inputs []InputMetadata
|
||||
}
|
||||
|
||||
// TemplateLookup provides parsed templates for a captured prompt identity.
|
||||
type TemplateLookup interface {
|
||||
GetCompiledTemplates(id, version string) (CompiledTemplates, bool)
|
||||
|
||||
@@ -50,10 +50,12 @@ func (r *Runner) PrepareExecution(ctx context.Context, req domain.RunRequest) (*
|
||||
return nil, err
|
||||
}
|
||||
|
||||
executionSnapshot, err := clonePreparedRun(operation.run)
|
||||
executionSnapshot := operation.run
|
||||
extraParams, err := jsonvalue.CopyMap(executionSnapshot.EffectiveModelParams.ExtraParams)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: failed to copy prepared execution: %v", ErrInvalidRequest, err)
|
||||
}
|
||||
executionSnapshot.EffectiveModelParams.ExtraParams = extraParams
|
||||
|
||||
details, err := clonePreparedRun(executionSnapshot)
|
||||
if err != nil {
|
||||
|
||||
@@ -631,6 +631,124 @@ func TestRunnerPreparedExecutionOwnershipUseAndDiscard(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerPreparedExecutionDetachesTransferredSnapshotSources(t *testing.T) {
|
||||
newExtras := func(value string) map[string]any {
|
||||
return map[string]any{"provider_options": map[string]any{
|
||||
"provider": map[string]any{"settings": map[string]any{"labels": []any{value}}},
|
||||
}}
|
||||
}
|
||||
extraLabel := func(extraParams map[string]any) string {
|
||||
return extraParams["provider_options"].(map[string]any)["provider"].(map[string]any)["settings"].(map[string]any)["labels"].([]any)[0].(string)
|
||||
}
|
||||
mutateExtra := func(extraParams map[string]any, value string) {
|
||||
extraParams["provider_options"].(map[string]any)["provider"].(map[string]any)["settings"].(map[string]any)["labels"].([]any)[0] = value
|
||||
}
|
||||
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
requestOverride bool
|
||||
}{
|
||||
{name: "profile target"},
|
||||
{name: "request override", requestOverride: true},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
profileExtras := newExtras("profile source")
|
||||
requestExtras := newExtras("request source")
|
||||
profile := defaultExecutionProfile()
|
||||
profile.ResolvedTarget.ExtraParams = profileExtras
|
||||
|
||||
schemaDocument := map[string]any{
|
||||
"type": "object",
|
||||
"properties": map[string]any{"value": map[string]any{"type": "string"}},
|
||||
}
|
||||
definition := promptDef(domain.FormatJSON, domain.ValidationJSONSchema, 0)
|
||||
definition.Validation.SchemaPath = "schema.json"
|
||||
control := &domain.CacheControl{Type: domain.CacheControlEphemeral, TTL: "1h"}
|
||||
rendered := &domain.RenderedPrompt{Messages: []domain.RenderedMessage{{
|
||||
Role: domain.RoleUser, Content: "captured rendering", CacheControl: control,
|
||||
}}}
|
||||
renderer := &fakeRenderer{rendered: rendered}
|
||||
sourceArtifact := &domain.Artifact{Body: []byte("captured input"), Hash: "captured-input-hash"}
|
||||
reader := &fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{"source": sourceArtifact}}
|
||||
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: `{"value":"ok"}`}}
|
||||
validator := &recordingValidationPreparer{
|
||||
plan: &recordingPreparedValidation{schemaDocument: schemaDocument},
|
||||
}
|
||||
runner := NewRunner(
|
||||
&fakePromptRepo{def: definition},
|
||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": profile}},
|
||||
reader, renderer, llmClient, validator, nil,
|
||||
)
|
||||
request := domain.RunRequest{
|
||||
PromptID: "p", PromptVersion: "1", ProfileID: "exec",
|
||||
Inputs: map[string]domain.ArtifactRef{
|
||||
"transcript": {Type: domain.ArtifactRefFile, URI: "source"},
|
||||
},
|
||||
}
|
||||
wantLabel := "profile source"
|
||||
if test.requestOverride {
|
||||
request.Execution = &domain.ExecutionTargetOverride{ExtraParams: requestExtras}
|
||||
wantLabel = "request source"
|
||||
}
|
||||
|
||||
prepared, err := runner.PrepareExecution(context.Background(), request)
|
||||
if err != nil {
|
||||
t.Fatalf("prepare execution: %v", err)
|
||||
}
|
||||
mutateExtra(profileExtras, "mutated profile source")
|
||||
mutateExtra(requestExtras, "mutated request source")
|
||||
rendered.Messages[0].Content = "mutated renderer source"
|
||||
control.TTL = "mutated renderer cache metadata"
|
||||
schemaDocument["properties"].(map[string]any)["value"].(map[string]any)["type"] = "integer"
|
||||
sourceArtifact.Hash = "mutated source hash"
|
||||
|
||||
details := prepared.Details()
|
||||
if details == nil || len(details.Messages) != 1 || details.Messages[0].Content != "captured rendering" ||
|
||||
details.Messages[0].CacheControl == nil || details.Messages[0].CacheControl.TTL != "1h" ||
|
||||
details.InputHashes["transcript"] != "captured-input-hash" ||
|
||||
extraLabel(details.EffectiveModelParams.ExtraParams) != wantLabel ||
|
||||
details.StructuredOutput == nil || details.StructuredOutput.JSONSchema == nil ||
|
||||
details.StructuredOutput.JSONSchema.Schema.(map[string]any)["properties"].(map[string]any)["value"].(map[string]any)["type"] != "string" {
|
||||
t.Fatalf("prepared details retained a source alias: %#v", details)
|
||||
}
|
||||
|
||||
result, err := runner.RunPrepared(context.Background(), prepared)
|
||||
if err != nil || result == nil || !result.Validation.IsValid || result.InputHashes["transcript"] != "captured-input-hash" {
|
||||
t.Fatalf("run prepared snapshot=(%+v, %v)", result, err)
|
||||
}
|
||||
if len(llmClient.lastReq.Prompt.Messages) != 1 || llmClient.lastReq.StructuredOutput == nil ||
|
||||
llmClient.lastReq.StructuredOutput.JSONSchema == nil ||
|
||||
extraLabel(llmClient.lastReq.Target.ExtraParams) != wantLabel ||
|
||||
llmClient.lastReq.Prompt.Messages[0].Content != "captured rendering" ||
|
||||
llmClient.lastReq.Prompt.Messages[0].CacheControl == nil || llmClient.lastReq.Prompt.Messages[0].CacheControl.TTL != "1h" ||
|
||||
llmClient.lastReq.StructuredOutput.JSONSchema.Schema.(map[string]any)["properties"].(map[string]any)["value"].(map[string]any)["type"] != "string" {
|
||||
t.Fatalf("execution snapshot retained a source alias: %#v", llmClient.lastReq)
|
||||
}
|
||||
if completedDetails := prepared.Details(); completedDetails == nil ||
|
||||
len(completedDetails.Messages) != 1 || completedDetails.Messages[0].Content != "captured rendering" ||
|
||||
extraLabel(completedDetails.EffectiveModelParams.ExtraParams) != wantLabel {
|
||||
t.Fatalf("completed details changed after source mutation: %#v", completedDetails)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerPrepareExecutionClassifiesExtraCopyFailureAsInvalidRequest(t *testing.T) {
|
||||
profile := defaultExecutionProfile()
|
||||
profile.ResolvedTarget.ExtraParams = map[string]any{"unsupported": func() {}}
|
||||
runner := NewRunner(
|
||||
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
|
||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": profile}},
|
||||
defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil, nil,
|
||||
)
|
||||
prepared, err := runner.PrepareExecution(context.Background(), domain.RunRequest{
|
||||
PromptID: "p", ProfileID: "exec",
|
||||
})
|
||||
if prepared != nil || !errors.Is(err, ErrInvalidRequest) {
|
||||
t.Fatalf("prepared execution copy failure=(%v, %v), want ErrInvalidRequest", prepared, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerPreparedExecutionWithoutValidatorSkipsValidation(t *testing.T) {
|
||||
runner := NewRunner(
|
||||
&fakePromptRepo{def: promptDef(domain.FormatJSON, domain.ValidationJSON, 0)},
|
||||
|
||||
237
internal/usecase/prepared_snapshot_benchmark_test.go
Normal file
237
internal/usecase/prepared_snapshot_benchmark_test.go
Normal file
@@ -0,0 +1,237 @@
|
||||
package usecase
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"runtime"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/validate"
|
||||
)
|
||||
|
||||
var preparedSnapshotHandleSink *PreparedExecution
|
||||
|
||||
const preparedSnapshotRetainedBatchSize = 4
|
||||
|
||||
type preparedSnapshotBenchmarkClient struct{}
|
||||
|
||||
func (preparedSnapshotBenchmarkClient) Generate(
|
||||
context.Context,
|
||||
domain.GenerateRequest,
|
||||
) (*domain.GenerateResponse, error) {
|
||||
return &domain.GenerateResponse{Content: "ok"}, nil
|
||||
}
|
||||
|
||||
type preparedSnapshotBenchmarkValidation struct{ schema any }
|
||||
|
||||
func (validation *preparedSnapshotBenchmarkValidation) Validate(
|
||||
context.Context,
|
||||
*domain.Artifact,
|
||||
) (domain.ValidationResult, error) {
|
||||
return domain.ValidationResult{Status: domain.ValidationPassed, IsValid: true}, nil
|
||||
}
|
||||
|
||||
func (validation *preparedSnapshotBenchmarkValidation) SchemaDocument() any {
|
||||
return validation.schema
|
||||
}
|
||||
|
||||
type preparedSnapshotBenchmarkValidator struct{ schema any }
|
||||
|
||||
func (validator preparedSnapshotBenchmarkValidator) Validate(
|
||||
context.Context,
|
||||
*domain.Artifact,
|
||||
domain.OutputContract,
|
||||
) (domain.ValidationResult, error) {
|
||||
return domain.ValidationResult{Status: domain.ValidationPassed, IsValid: true}, nil
|
||||
}
|
||||
|
||||
func (validator preparedSnapshotBenchmarkValidator) PrepareValidation(
|
||||
context.Context,
|
||||
domain.OutputContract,
|
||||
) (validate.PreparedValidation, error) {
|
||||
return &preparedSnapshotBenchmarkValidation{schema: validator.schema}, nil
|
||||
}
|
||||
|
||||
func preparedSnapshotBenchmarkSchema(properties int) map[string]any {
|
||||
fields := make(map[string]any, properties)
|
||||
for index := range properties {
|
||||
fields[fmt.Sprintf("property-%03d", index)] = map[string]any{"type": "string"}
|
||||
}
|
||||
return map[string]any{
|
||||
"type": "object", "properties": fields, "additionalProperties": false,
|
||||
}
|
||||
}
|
||||
|
||||
func preparedSnapshotBenchmarkExtras(entries int) map[string]any {
|
||||
providers := make(map[string]any, entries)
|
||||
for index := range entries {
|
||||
providers[fmt.Sprintf("provider-%03d", index)] = map[string]any{
|
||||
"enabled": true,
|
||||
"labels": []any{fmt.Sprintf("tier-%d", index%4), fmt.Sprintf("region-%d", index%3)},
|
||||
"limits": map[string]any{"burst": index + 1, "period": "minute"},
|
||||
"routes": []any{map[string]any{"weight": index + 1, "fallback": true}},
|
||||
}
|
||||
}
|
||||
return map[string]any{"provider_options": providers}
|
||||
}
|
||||
|
||||
func newPreparedSnapshotBenchmarkRunner(
|
||||
mutable bool,
|
||||
) (*Runner, domain.RunRequest, any) {
|
||||
messageCount := 1
|
||||
extraEntries := 0
|
||||
schemaProperties := 1
|
||||
if mutable {
|
||||
messageCount = 8
|
||||
extraEntries = 64
|
||||
schemaProperties = 256
|
||||
}
|
||||
|
||||
messages := make([]domain.RenderedMessage, messageCount)
|
||||
for index := range messages {
|
||||
messages[index] = domain.RenderedMessage{
|
||||
Role: domain.RoleUser, Content: fmt.Sprintf("rendered message %d", index),
|
||||
CacheControl: &domain.CacheControl{Type: domain.CacheControlEphemeral, TTL: "1h"},
|
||||
}
|
||||
}
|
||||
renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{
|
||||
SessionID: "snapshot-session", Messages: messages,
|
||||
}}
|
||||
|
||||
profile := defaultExecutionProfile()
|
||||
if extraEntries > 0 {
|
||||
profile.ResolvedTarget.ExtraParams = preparedSnapshotBenchmarkExtras(extraEntries)
|
||||
}
|
||||
profileRepository := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": profile}}
|
||||
definition := promptDef(domain.FormatJSON, domain.ValidationJSONSchema, 0)
|
||||
definition.Validation.SchemaPath = "snapshot-schema.json"
|
||||
schema := preparedSnapshotBenchmarkSchema(schemaProperties)
|
||||
validator := preparedSnapshotBenchmarkValidator{schema: schema}
|
||||
runner := NewRunner(
|
||||
&fakePromptRepo{def: definition},
|
||||
profileRepository,
|
||||
staticArtifactReader{},
|
||||
renderer,
|
||||
preparedSnapshotBenchmarkClient{},
|
||||
validator,
|
||||
nil,
|
||||
)
|
||||
request := domain.RunRequest{
|
||||
PromptID: "p", PromptVersion: "1", ProfileID: "exec",
|
||||
Inputs: singleInputRef(), Vars: map[string]string{"name": "Ada"},
|
||||
AppendedMessages: []domain.RenderedMessage{{
|
||||
Role: domain.RoleAssistant, Content: "appended message",
|
||||
CacheControl: &domain.CacheControl{Type: domain.CacheControlEphemeral, TTL: "1h"},
|
||||
}},
|
||||
}
|
||||
return runner, request, schema
|
||||
}
|
||||
|
||||
func checkPreparedSnapshotBenchmarkFixture(b *testing.B, runner *Runner, request domain.RunRequest, schema any) {
|
||||
b.Helper()
|
||||
prepared, err := runner.PrepareExecution(context.Background(), request)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
details := prepared.Details()
|
||||
if details == nil || details.PromptID != request.PromptID || len(details.Messages) < 2 ||
|
||||
details.Messages[0].Content != "rendered message 0" || details.Messages[len(details.Messages)-1].Content != "appended message" ||
|
||||
details.InputHashes["transcript"] == "" || details.StructuredOutput == nil || details.StructuredOutput.JSONSchema == nil {
|
||||
prepared.Discard()
|
||||
b.Fatalf("prepared snapshot fixture is incomplete: %#v", details)
|
||||
}
|
||||
if got := details.StructuredOutput.JSONSchema.Schema; !reflect.DeepEqual(got, schema) {
|
||||
prepared.Discard()
|
||||
b.Fatalf("prepared schema differs from fixture: %#v", got)
|
||||
}
|
||||
result, err := runner.RunPrepared(context.Background(), prepared)
|
||||
if err != nil || result == nil || result.RawOutput != "ok" || !result.Validation.IsValid {
|
||||
b.Fatalf("completed fixture=(%+v, %v)", result, err)
|
||||
}
|
||||
completedDetails := prepared.Details()
|
||||
if completedDetails == nil || completedDetails.Messages[0].Content != "rendered message 0" ||
|
||||
!reflect.DeepEqual(completedDetails.StructuredOutput.JSONSchema.Schema, schema) {
|
||||
b.Fatalf("completed handle did not retain its details: %#v", completedDetails)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkPreparedSnapshot(b *testing.B) {
|
||||
for _, fixture := range []struct {
|
||||
name string
|
||||
mutable bool
|
||||
}{
|
||||
{name: "small"},
|
||||
{name: "mutable_payload", mutable: true},
|
||||
} {
|
||||
b.Run(fixture.name, func(b *testing.B) {
|
||||
runner, request, schema := newPreparedSnapshotBenchmarkRunner(fixture.mutable)
|
||||
checkPreparedSnapshotBenchmarkFixture(b, runner, request, schema)
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for b.Loop() {
|
||||
prepared, err := runner.PrepareExecution(context.Background(), request)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
prepared.Discard()
|
||||
preparedSnapshotHandleSink = prepared
|
||||
}
|
||||
b.StopTimer()
|
||||
preparedSnapshotHandleSink = nil
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkRetainedPreparedHandles(b *testing.B) {
|
||||
for _, state := range []string{"ready", "completed", "discarded"} {
|
||||
b.Run(state, func(b *testing.B) {
|
||||
runner, request, schema := newPreparedSnapshotBenchmarkRunner(true)
|
||||
checkPreparedSnapshotBenchmarkFixture(b, runner, request, schema)
|
||||
retained := make([]*PreparedExecution, preparedSnapshotRetainedBatchSize)
|
||||
runtime.GC()
|
||||
var setupMemory runtime.MemStats
|
||||
runtime.ReadMemStats(&setupMemory)
|
||||
runtime.KeepAlive(runner)
|
||||
runtime.KeepAlive(request)
|
||||
runtime.KeepAlive(retained)
|
||||
|
||||
retainHandle := func() *PreparedExecution {
|
||||
prepared, err := runner.PrepareExecution(context.Background(), request)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
switch state {
|
||||
case "completed":
|
||||
result, err := runner.RunPrepared(context.Background(), prepared)
|
||||
if err != nil || result == nil || result.RawOutput != "ok" || !result.Validation.IsValid {
|
||||
b.Fatalf("complete retained handle=(%+v, %v)", result, err)
|
||||
}
|
||||
case "discarded":
|
||||
prepared.Discard()
|
||||
}
|
||||
return prepared
|
||||
}
|
||||
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for iteration := 0; iteration < b.N; iteration++ {
|
||||
retained[iteration%len(retained)] = retainHandle()
|
||||
}
|
||||
b.StopTimer()
|
||||
for index, prepared := range retained {
|
||||
if prepared == nil {
|
||||
retained[index] = retainHandle()
|
||||
}
|
||||
}
|
||||
runtime.GC()
|
||||
var retainedMemory runtime.MemStats
|
||||
runtime.ReadMemStats(&retainedMemory)
|
||||
b.ReportMetric(float64(int64(retainedMemory.HeapAlloc)-int64(setupMemory.HeapAlloc)), "retained-B")
|
||||
runtime.KeepAlive(runner)
|
||||
runtime.KeepAlive(request)
|
||||
runtime.KeepAlive(retained)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -15,16 +15,27 @@ type resolvedProfileSelection struct {
|
||||
profile *domain.ExecutionProfile
|
||||
}
|
||||
|
||||
type profileIdentityRepository interface {
|
||||
GetProfileIdentity(context.Context, string) (string, error)
|
||||
}
|
||||
|
||||
func (r *Runner) profileLookupID(profileID string) (string, error) {
|
||||
if strings.TrimSpace(profileID) == "" {
|
||||
return "", fmt.Errorf("%w: profile id is required", ErrInvalidRequest)
|
||||
}
|
||||
if r == nil || r.profiles == nil {
|
||||
return "", fmt.Errorf("%w: profile repository is not configured", ErrProfileLoad)
|
||||
}
|
||||
return profileID, nil
|
||||
}
|
||||
|
||||
func (r *Runner) resolveProfileSelection(
|
||||
ctx context.Context,
|
||||
profileID string,
|
||||
) (*resolvedProfileSelection, error) {
|
||||
lookupID := profileID
|
||||
if strings.TrimSpace(lookupID) == "" {
|
||||
return nil, fmt.Errorf("%w: profile id is required", ErrInvalidRequest)
|
||||
}
|
||||
if r == nil || r.profiles == nil {
|
||||
return nil, fmt.Errorf("%w: profile repository is not configured", ErrProfileLoad)
|
||||
lookupID, err := r.profileLookupID(profileID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
selectedProfile, err := r.profiles.GetProfile(ctx, lookupID)
|
||||
if err != nil {
|
||||
@@ -45,6 +56,25 @@ func (r *Runner) resolveProfileSelection(
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (r *Runner) resolveProfileIdentity(ctx context.Context, profileID string) (string, error) {
|
||||
lookupID, err := r.profileLookupID(profileID)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if indexed, ok := r.profiles.(profileIdentityRepository); ok {
|
||||
identity, err := indexed.GetProfileIdentity(ctx, lookupID)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("%w: %w", ErrProfileLoad, err)
|
||||
}
|
||||
return identity, nil
|
||||
}
|
||||
selection, err := r.resolveProfileSelection(ctx, lookupID)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return selection.id, nil
|
||||
}
|
||||
|
||||
func normalizeResolvedExecutionTarget(target domain.ExecutionTarget) (domain.ExecutionTarget, error) {
|
||||
endpoint, err := domain.NormalizeOpenAICompatibleBaseEndpoint(target.Endpoint)
|
||||
if err != nil {
|
||||
|
||||
@@ -13,6 +13,10 @@ type resolvedPromptDefinition struct {
|
||||
hash string
|
||||
}
|
||||
|
||||
type promptInspectionRepository interface {
|
||||
GetPromptInspection(context.Context, string, string) (*domain.PromptInspection, error)
|
||||
}
|
||||
|
||||
func (r *Runner) resolvePromptDefinition(
|
||||
ctx context.Context,
|
||||
promptID string,
|
||||
@@ -51,6 +55,43 @@ func (r *Runner) resolvePromptDefinition(
|
||||
return &resolvedPromptDefinition{definition: definition, hash: hash}, nil
|
||||
}
|
||||
|
||||
func (r *Runner) resolvePromptInspection(
|
||||
ctx context.Context,
|
||||
promptID string,
|
||||
promptVersion string,
|
||||
) (*domain.PromptInspection, error) {
|
||||
if strings.TrimSpace(promptID) == "" {
|
||||
return nil, fmt.Errorf("%w: prompt id is required", ErrInvalidRequest)
|
||||
}
|
||||
if r == nil || r.promptDefs == nil {
|
||||
return nil, fmt.Errorf("%w: prompt repository is not configured", ErrPromptLoad)
|
||||
}
|
||||
if indexed, ok := r.promptDefs.(promptInspectionRepository); ok {
|
||||
inspection, err := indexed.GetPromptInspection(ctx, promptID, promptVersion)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: %w", ErrPromptLoad, err)
|
||||
}
|
||||
if inspection == nil {
|
||||
return nil, fmt.Errorf("%w: prompt repository returned nil inspection", ErrPromptLoad)
|
||||
}
|
||||
return inspection, nil
|
||||
}
|
||||
|
||||
selection, err := r.resolvePromptDefinition(ctx, promptID, promptVersion)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
definition := selection.definition
|
||||
return &domain.PromptInspection{
|
||||
PromptID: definition.ID,
|
||||
PromptVersion: definition.Version,
|
||||
PromptHash: selection.hash,
|
||||
DefaultProfileID: definition.DefaultProfile,
|
||||
Inputs: append([]domain.PromptInput(nil), definition.Inputs...),
|
||||
OutputContract: definition.Validation,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// InspectPrompt resolves one explicit prompt without execution work.
|
||||
func (r *Runner) InspectPrompt(
|
||||
ctx context.Context,
|
||||
@@ -66,19 +107,5 @@ func (r *Runner) InspectPrompt(
|
||||
default:
|
||||
}
|
||||
|
||||
selection, err := r.resolvePromptDefinition(ctx, promptID, promptVersion)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
inputs := make([]domain.PromptInput, len(selection.definition.Inputs))
|
||||
copy(inputs, selection.definition.Inputs)
|
||||
|
||||
return &domain.PromptInspection{
|
||||
PromptID: selection.definition.ID,
|
||||
PromptVersion: selection.definition.Version,
|
||||
PromptHash: selection.hash,
|
||||
DefaultProfileID: selection.definition.DefaultProfile,
|
||||
Inputs: inputs,
|
||||
OutputContract: selection.definition.Validation,
|
||||
}, nil
|
||||
return r.resolvePromptInspection(ctx, promptID, promptVersion)
|
||||
}
|
||||
|
||||
@@ -18,6 +18,13 @@ type inspectionPromptRepository struct {
|
||||
version string
|
||||
}
|
||||
|
||||
type promptMetadataRepository struct {
|
||||
inspection domain.PromptInspection
|
||||
inspectionErr error
|
||||
lookupID string
|
||||
lookupVersion string
|
||||
}
|
||||
|
||||
func (r *inspectionPromptRepository) GetPromptDefinition(
|
||||
_ context.Context,
|
||||
id string,
|
||||
@@ -32,6 +39,22 @@ func (r *inspectionPromptRepository) GetPromptDefinition(
|
||||
return r.definition, nil
|
||||
}
|
||||
|
||||
func (r *promptMetadataRepository) GetPromptDefinition(context.Context, string, string) (*domain.PromptDefinition, error) {
|
||||
return nil, errors.New("full prompt definition should not be requested")
|
||||
}
|
||||
|
||||
func (r *promptMetadataRepository) GetPromptInspection(
|
||||
_ context.Context,
|
||||
id string,
|
||||
version string,
|
||||
) (*domain.PromptInspection, error) {
|
||||
r.lookupID = id
|
||||
r.lookupVersion = version
|
||||
inspection := r.inspection
|
||||
inspection.Inputs = append([]domain.PromptInput(nil), inspection.Inputs...)
|
||||
return &inspection, r.inspectionErr
|
||||
}
|
||||
|
||||
func TestRunnerInspectPromptResolvesOneDefinitionWithoutExecutionCollaborators(t *testing.T) {
|
||||
definition := &domain.PromptDefinition{
|
||||
ID: "normalized.prompt",
|
||||
@@ -81,6 +104,42 @@ func TestRunnerInspectPromptResolvesOneDefinitionWithoutExecutionCollaborators(t
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerInspectPromptUsesOwnedMetadataWhenRepositorySupportsIt(t *testing.T) {
|
||||
repository := &promptMetadataRepository{inspection: domain.PromptInspection{
|
||||
PromptID: "captured.prompt", PromptVersion: "v2", PromptHash: "cached-prompt-hash",
|
||||
DefaultProfileID: "partial-profile",
|
||||
Inputs: []domain.PromptInput{{Name: "document", Required: true, ContentType: "text/plain", Description: "Captured."}},
|
||||
OutputContract: domain.OutputContract{Format: domain.FormatJSON, ValidationMode: domain.ValidationJSON},
|
||||
}}
|
||||
runner := &Runner{promptDefs: repository}
|
||||
inspection, err := runner.InspectPrompt(context.Background(), " requested-id ", " requested-version ")
|
||||
if err != nil || repository.lookupID != " requested-id " || repository.lookupVersion != " requested-version " ||
|
||||
inspection.PromptID != "captured.prompt" || inspection.PromptVersion != "v2" ||
|
||||
inspection.PromptHash != "cached-prompt-hash" || inspection.DefaultProfileID != "partial-profile" ||
|
||||
!reflect.DeepEqual(inspection.Inputs, repository.inspection.Inputs) || inspection.OutputContract != repository.inspection.OutputContract {
|
||||
t.Fatalf("metadata inspection=(%+v, %v), lookup=(%q,%q)", inspection, err, repository.lookupID, repository.lookupVersion)
|
||||
}
|
||||
inspection.Inputs[0].Name = "caller mutation"
|
||||
inspection.OutputContract.SchemaPath = "caller mutation"
|
||||
second, err := runner.InspectPrompt(context.Background(), "requested-id", "requested-version")
|
||||
if err != nil || second.Inputs[0].Name != "document" || second.OutputContract.SchemaPath != "" {
|
||||
t.Fatalf("metadata inspection was not caller-owned: (%+v, %v)", second, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerInspectPromptUsesCapturedHashWhenMetadataCapabilityIsAbsent(t *testing.T) {
|
||||
repository := &requirementInspectionPromptRepository{
|
||||
definition: &domain.PromptDefinition{
|
||||
ID: "captured.prompt", Version: "v1", Inputs: []domain.PromptInput{{Name: "document"}},
|
||||
},
|
||||
}
|
||||
inspection, err := (&Runner{promptDefs: repository}).InspectPrompt(context.Background(), " requested-id ", " requested-version ")
|
||||
if err != nil || inspection.PromptID != "captured.prompt" || inspection.PromptVersion != "v1" ||
|
||||
inspection.PromptHash != "captured-hash" || inspection.Inputs[0].Name != "document" {
|
||||
t.Fatalf("captured-hash fallback inspection=(%+v, %v)", inspection, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerInspectPromptClassifiesFailuresWithoutRepositoryWorkAfterCancellation(t *testing.T) {
|
||||
t.Run("blank ID", func(t *testing.T) {
|
||||
repository := &inspectionPromptRepository{}
|
||||
|
||||
80
internal/usecase/requirements_inspection.go
Normal file
80
internal/usecase/requirements_inspection.go
Normal file
@@ -0,0 +1,80 @@
|
||||
package usecase
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/prompt"
|
||||
)
|
||||
|
||||
// RequirementInspectionRequest selects a captured prompt and supplies the
|
||||
// context used for dependency classification.
|
||||
type RequirementInspectionRequest struct {
|
||||
PromptID string
|
||||
PromptVersion string
|
||||
ProfileID string
|
||||
Vars map[string]string
|
||||
SessionID string
|
||||
}
|
||||
|
||||
type requirementSnapshotRepository interface {
|
||||
GetRequirementSnapshot(id, version string) (prompt.RequirementSnapshot, bool)
|
||||
}
|
||||
|
||||
// InspectRequirements discovers prompt inputs and caller variables without
|
||||
// loading artifacts or performing execution preparation.
|
||||
func (r *Runner) InspectRequirements(
|
||||
ctx context.Context,
|
||||
req RequirementInspectionRequest,
|
||||
) (*domain.RequirementInspection, error) {
|
||||
if strings.TrimSpace(req.PromptID) == "" {
|
||||
return nil, fmt.Errorf("%w: prompt id is required", ErrInvalidRequest)
|
||||
}
|
||||
directSessionID, err := domain.NormalizeSessionID(req.SessionID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: session_id: %v", ErrInvalidRequest, err)
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, fmt.Errorf("%w: %w", ErrPromptLoad, err)
|
||||
}
|
||||
|
||||
selection, err := r.resolvePromptInspection(ctx, req.PromptID, req.PromptVersion)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, fmt.Errorf("%w: %w", ErrProfileLoad, err)
|
||||
}
|
||||
selectedProfileID := req.ProfileID
|
||||
if strings.TrimSpace(selectedProfileID) == "" {
|
||||
selectedProfileID = selection.DefaultProfileID
|
||||
}
|
||||
if strings.TrimSpace(selectedProfileID) == "" {
|
||||
return nil, fmt.Errorf("%w: %w: profile id is required either in request or prompt default_profile", ErrInvalidRequest, ErrProfileRequired)
|
||||
}
|
||||
profileIdentity, err := r.resolveProfileIdentity(ctx, selectedProfileID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, fmt.Errorf("%w: %w", ErrPromptLoad, err)
|
||||
}
|
||||
|
||||
repository, ok := r.promptDefs.(requirementSnapshotRepository)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("%w: prompt repository does not provide captured requirement snapshots", ErrPromptLoad)
|
||||
}
|
||||
snapshot, ok := repository.GetRequirementSnapshot(selection.PromptID, selection.PromptVersion)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("%w: captured requirement snapshot is unavailable", ErrPromptLoad)
|
||||
}
|
||||
definition := &domain.PromptDefinition{ID: selection.PromptID, Version: selection.PromptVersion}
|
||||
inspection, err := prompt.AnalyzeRequirements(ctx, definition, snapshot, req.Vars, directSessionID != "")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: %w", ErrPromptLoad, err)
|
||||
}
|
||||
inspection.ProfileID = profileIdentity
|
||||
return &inspection, nil
|
||||
}
|
||||
372
internal/usecase/requirements_inspection_test.go
Normal file
372
internal/usecase/requirements_inspection_test.go
Normal file
@@ -0,0 +1,372 @@
|
||||
package usecase
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"text/template"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/prompt"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/promptdef"
|
||||
)
|
||||
|
||||
type requirementInspectionPromptRepository struct {
|
||||
definition *domain.PromptDefinition
|
||||
snapshot prompt.RequirementSnapshot
|
||||
lookupErr error
|
||||
snapshotFound bool
|
||||
lookupCalls int
|
||||
snapshotCalls int
|
||||
lookupID string
|
||||
lookupVersion string
|
||||
snapshotID string
|
||||
snapshotVersion string
|
||||
}
|
||||
|
||||
type requirementPromptMetadataRepository struct {
|
||||
*requirementInspectionPromptRepository
|
||||
inspection domain.PromptInspection
|
||||
lookupID string
|
||||
lookupVersion string
|
||||
}
|
||||
|
||||
type inspectionProfileIdentityRepository struct {
|
||||
*inspectionProfileRepository
|
||||
identity string
|
||||
identityErr error
|
||||
identityID string
|
||||
identityCalls int
|
||||
}
|
||||
|
||||
func (repository *inspectionProfileIdentityRepository) GetProfileIdentity(
|
||||
_ context.Context,
|
||||
id string,
|
||||
) (string, error) {
|
||||
repository.identityCalls++
|
||||
repository.identityID = id
|
||||
return repository.identity, repository.identityErr
|
||||
}
|
||||
|
||||
func (repository *requirementInspectionPromptRepository) GetPromptDefinition(
|
||||
context.Context,
|
||||
string,
|
||||
string,
|
||||
) (*domain.PromptDefinition, error) {
|
||||
return repository.definition, repository.lookupErr
|
||||
}
|
||||
|
||||
func (repository *requirementPromptMetadataRepository) GetPromptInspection(
|
||||
_ context.Context,
|
||||
id string,
|
||||
version string,
|
||||
) (*domain.PromptInspection, error) {
|
||||
repository.lookupID = id
|
||||
repository.lookupVersion = version
|
||||
inspection := repository.inspection
|
||||
inspection.Inputs = append([]domain.PromptInput(nil), inspection.Inputs...)
|
||||
return &inspection, nil
|
||||
}
|
||||
|
||||
func (repository *requirementInspectionPromptRepository) GetPromptSnapshot(
|
||||
_ context.Context,
|
||||
id string,
|
||||
version string,
|
||||
) (*domain.PromptDefinition, string, error) {
|
||||
repository.lookupCalls++
|
||||
repository.lookupID = id
|
||||
repository.lookupVersion = version
|
||||
return repository.definition, "captured-hash", repository.lookupErr
|
||||
}
|
||||
|
||||
func (repository *requirementInspectionPromptRepository) GetRequirementSnapshot(
|
||||
id string,
|
||||
version string,
|
||||
) (prompt.RequirementSnapshot, bool) {
|
||||
repository.snapshotCalls++
|
||||
repository.snapshotID = id
|
||||
repository.snapshotVersion = version
|
||||
return repository.snapshot, repository.snapshotFound
|
||||
}
|
||||
|
||||
func TestRunnerInspectRequirementsUsesCapturedSelectionWithoutExecutionWork(t *testing.T) {
|
||||
message, err := prompt.CompileTemplate("message", `{{input "message-input"}} {{.caller}}`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
session, err := prompt.CompileTemplate("session", `{{input "session-only"}}`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
definition := &domain.PromptDefinition{ID: "selected", Version: "v2", DefaultProfile: "default-profile"}
|
||||
repository := &requirementInspectionPromptRepository{
|
||||
definition: definition,
|
||||
snapshotFound: true,
|
||||
snapshot: prompt.RequirementSnapshot{Templates: prompt.CompiledTemplates{
|
||||
Messages: []*template.Template{message}, Session: session,
|
||||
}},
|
||||
}
|
||||
profiles := &inspectionProfileRepository{profile: &domain.ExecutionProfile{
|
||||
ID: "partial-profile", ResolvedTarget: &domain.ExecutionTarget{Endpoint: ""},
|
||||
}}
|
||||
runner := &Runner{promptDefs: repository, profiles: profiles}
|
||||
inspection, err := runner.InspectRequirements(context.Background(), RequirementInspectionRequest{
|
||||
PromptID: " requested ", PromptVersion: " exact-version ", ProfileID: "partial-profile",
|
||||
Vars: map[string]string{"caller": ""}, SessionID: " direct-session ",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("inspect requirements: %v", err)
|
||||
}
|
||||
if repository.lookupCalls != 1 || repository.lookupID != " requested " || repository.lookupVersion != " exact-version " ||
|
||||
repository.snapshotCalls != 1 || repository.snapshotID != definition.ID || repository.snapshotVersion != definition.Version {
|
||||
t.Fatalf("selection lookup=(%d,%q,%q), snapshot=(%d,%q,%q)", repository.lookupCalls, repository.lookupID,
|
||||
repository.lookupVersion, repository.snapshotCalls, repository.snapshotID, repository.snapshotVersion)
|
||||
}
|
||||
if profiles.calls != 1 || profiles.id != "partial-profile" || inspection.PromptID != definition.ID ||
|
||||
inspection.PromptVersion != definition.Version || inspection.ProfileID != "partial-profile" || !inspection.Complete ||
|
||||
len(inspection.Inputs) != 1 || inspection.Inputs[0].Name != "message-input" || len(inspection.Variables) != 1 ||
|
||||
inspection.Variables[0].Name != "caller" || !inspection.Variables[0].Supplied {
|
||||
t.Fatalf("unexpected requirement inspection: %+v", inspection)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerInspectRequirementsUsesPromptMetadataAndExactRequirementSnapshot(t *testing.T) {
|
||||
message, err := prompt.CompileTemplate("message", `{{input "message-input"}} {{.caller}}`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
session, err := prompt.CompileTemplate("session", `{{input "session-only"}}`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
baseRepository := &requirementInspectionPromptRepository{
|
||||
snapshotFound: true,
|
||||
snapshot: prompt.RequirementSnapshot{Templates: prompt.CompiledTemplates{
|
||||
Messages: []*template.Template{message}, Session: session,
|
||||
}},
|
||||
}
|
||||
promptRepository := &requirementPromptMetadataRepository{
|
||||
requirementInspectionPromptRepository: baseRepository,
|
||||
inspection: domain.PromptInspection{
|
||||
PromptID: "captured-id", PromptVersion: "captured-version", PromptHash: "captured-hash",
|
||||
DefaultProfileID: "default-profile",
|
||||
},
|
||||
}
|
||||
profiles := &inspectionProfileIdentityRepository{
|
||||
inspectionProfileRepository: &inspectionProfileRepository{},
|
||||
identity: "default-profile",
|
||||
}
|
||||
inspection, err := (&Runner{promptDefs: promptRepository, profiles: profiles}).InspectRequirements(
|
||||
context.Background(), RequirementInspectionRequest{
|
||||
PromptID: " requested-id ", PromptVersion: " requested-version ",
|
||||
Vars: map[string]string{"caller": "Ada"}, SessionID: " direct-session ",
|
||||
},
|
||||
)
|
||||
if err != nil || promptRepository.lookupID != " requested-id " || promptRepository.lookupVersion != " requested-version " ||
|
||||
baseRepository.snapshotID != "captured-id" || baseRepository.snapshotVersion != "captured-version" ||
|
||||
profiles.identityID != "default-profile" || inspection.PromptID != "captured-id" ||
|
||||
inspection.PromptVersion != "captured-version" || inspection.ProfileID != "default-profile" ||
|
||||
!inspection.Complete || len(inspection.Inputs) != 1 || inspection.Inputs[0].Name != "message-input" ||
|
||||
len(inspection.Variables) != 1 || inspection.Variables[0].Name != "caller" || !inspection.Variables[0].Supplied {
|
||||
t.Fatalf("metadata-based requirement inspection=(%+v, %v), prompt lookup=(%q,%q), snapshot=(%q,%q), profile=%q",
|
||||
inspection, err, promptRepository.lookupID, promptRepository.lookupVersion,
|
||||
baseRepository.snapshotID, baseRepository.snapshotVersion, profiles.identityID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerInspectRequirementsUsesProfileIdentityCapabilityOrFallback(t *testing.T) {
|
||||
message, err := prompt.CompileTemplate("message", `{{.name}}`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
definition := &domain.PromptDefinition{ID: "selected", Version: "v1"}
|
||||
makePromptRepository := func() *requirementInspectionPromptRepository {
|
||||
return &requirementInspectionPromptRepository{
|
||||
definition: definition, snapshotFound: true,
|
||||
snapshot: prompt.RequirementSnapshot{Templates: prompt.CompiledTemplates{Messages: []*template.Template{message}}},
|
||||
}
|
||||
}
|
||||
requestedID := " literal-profile "
|
||||
|
||||
t.Run("optional capability", func(t *testing.T) {
|
||||
profiles := &inspectionProfileIdentityRepository{
|
||||
inspectionProfileRepository: &inspectionProfileRepository{profile: &domain.ExecutionProfile{
|
||||
ID: "repository-profile", ResolvedTarget: &domain.ExecutionTarget{},
|
||||
}},
|
||||
identity: requestedID,
|
||||
}
|
||||
inspection, err := (&Runner{promptDefs: makePromptRepository(), profiles: profiles}).InspectRequirements(
|
||||
context.Background(), RequirementInspectionRequest{
|
||||
PromptID: "selected", ProfileID: requestedID, Vars: map[string]string{"name": "Ada"},
|
||||
},
|
||||
)
|
||||
if err != nil || inspection.ProfileID != requestedID || profiles.identityCalls != 1 ||
|
||||
profiles.identityID != requestedID || profiles.calls != 0 {
|
||||
t.Fatalf("identity capability inspection=(%+v, %v), capability calls=%d id=%q full lookups=%d",
|
||||
inspection, err, profiles.identityCalls, profiles.identityID, profiles.calls)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("full-profile fallback", func(t *testing.T) {
|
||||
profiles := &inspectionProfileRepository{profile: &domain.ExecutionProfile{
|
||||
ID: "repository-profile", ResolvedTarget: &domain.ExecutionTarget{},
|
||||
}}
|
||||
inspection, err := (&Runner{promptDefs: makePromptRepository(), profiles: profiles}).InspectRequirements(
|
||||
context.Background(), RequirementInspectionRequest{
|
||||
PromptID: "selected", ProfileID: requestedID, Vars: map[string]string{"name": "Ada"},
|
||||
},
|
||||
)
|
||||
if err != nil || inspection.ProfileID != requestedID || profiles.calls != 1 || profiles.id != requestedID {
|
||||
t.Fatalf("fallback inspection=(%+v, %v), full lookup=(%d,%q)", inspection, err, profiles.calls, profiles.id)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("capability failure", func(t *testing.T) {
|
||||
lookupErr := errors.New("profile identity lookup failed")
|
||||
profiles := &inspectionProfileIdentityRepository{
|
||||
inspectionProfileRepository: &inspectionProfileRepository{},
|
||||
identityErr: lookupErr,
|
||||
}
|
||||
_, err := (&Runner{promptDefs: makePromptRepository(), profiles: profiles}).InspectRequirements(
|
||||
context.Background(), RequirementInspectionRequest{PromptID: "selected", ProfileID: requestedID},
|
||||
)
|
||||
if !errors.Is(err, ErrProfileLoad) || !errors.Is(err, lookupErr) || profiles.identityCalls != 1 || profiles.calls != 0 {
|
||||
t.Fatalf("capability failure=(%v), capability calls=%d full lookups=%d", err, profiles.identityCalls, profiles.calls)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestRunnerInspectRequirementsUsesPromptDefaultAndIncludesSessionWhenNotBypassed(t *testing.T) {
|
||||
message, err := prompt.CompileTemplate("message", `{{input "message-input"}}`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
session, err := prompt.CompileTemplate("session", `{{input "session-input"}}`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
definition := &domain.PromptDefinition{ID: "selected", Version: "v1", DefaultProfile: "default-profile"}
|
||||
repository := &requirementInspectionPromptRepository{
|
||||
definition: definition, snapshotFound: true,
|
||||
snapshot: prompt.RequirementSnapshot{Templates: prompt.CompiledTemplates{Messages: []*template.Template{message}, Session: session}},
|
||||
}
|
||||
profiles := &inspectionProfileRepository{profile: &domain.ExecutionProfile{
|
||||
ID: "default-profile", ResolvedTarget: &domain.ExecutionTarget{},
|
||||
}}
|
||||
inspection, err := (&Runner{promptDefs: repository, profiles: profiles}).InspectRequirements(
|
||||
context.Background(), RequirementInspectionRequest{PromptID: "selected"},
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("inspect requirements: %v", err)
|
||||
}
|
||||
if profiles.id != "default-profile" || inspection.ProfileID != "default-profile" || len(inspection.Inputs) != 2 ||
|
||||
inspection.Inputs[0].Name != "message-input" || inspection.Inputs[1].Name != "session-input" {
|
||||
t.Fatalf("prompt-default profile or session template was not selected: %+v", inspection)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerInspectRequirementsReturnsIncompleteDiscoveryAsSuccess(t *testing.T) {
|
||||
compiled, err := prompt.CompileTemplate("message", `{{if .missing}}{{input "left"}}{{else}}{{input "right"}}{{end}}`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
repository := &requirementInspectionPromptRepository{
|
||||
definition: &domain.PromptDefinition{ID: "selected", Version: "v1", DefaultProfile: "profile"},
|
||||
snapshotFound: true,
|
||||
snapshot: prompt.RequirementSnapshot{Templates: prompt.CompiledTemplates{Messages: []*template.Template{compiled}}},
|
||||
}
|
||||
profiles := &inspectionProfileRepository{profile: &domain.ExecutionProfile{ID: "profile", ResolvedTarget: &domain.ExecutionTarget{}}}
|
||||
inspection, err := (&Runner{promptDefs: repository, profiles: profiles}).InspectRequirements(
|
||||
context.Background(), RequirementInspectionRequest{PromptID: "selected"},
|
||||
)
|
||||
if err != nil || inspection == nil || inspection.Complete || len(inspection.Issues) != 1 ||
|
||||
inspection.Issues[0].Code != domain.RequirementIssueMissingVariable {
|
||||
t.Fatalf("incomplete discovery returned (%+v, %v), want successful partial result", inspection, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerInspectRequirementsClassifiesSelectionAndSnapshotFailures(t *testing.T) {
|
||||
t.Run("blank prompt", func(t *testing.T) {
|
||||
repository := &requirementInspectionPromptRepository{}
|
||||
_, err := (&Runner{promptDefs: repository}).InspectRequirements(context.Background(), RequirementInspectionRequest{PromptID: " \t "})
|
||||
if !errors.Is(err, ErrInvalidRequest) || repository.lookupCalls != 0 {
|
||||
t.Fatalf("blank prompt error=%v lookups=%d", err, repository.lookupCalls)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("invalid session", func(t *testing.T) {
|
||||
repository := &requirementInspectionPromptRepository{}
|
||||
_, err := (&Runner{promptDefs: repository}).InspectRequirements(context.Background(), RequirementInspectionRequest{
|
||||
PromptID: "selected", SessionID: string([]byte{0xff}),
|
||||
})
|
||||
if !errors.Is(err, ErrInvalidRequest) || repository.lookupCalls != 0 {
|
||||
t.Fatalf("invalid session error=%v lookups=%d", err, repository.lookupCalls)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("cancellation before lookup", func(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
repository := &requirementInspectionPromptRepository{}
|
||||
_, err := (&Runner{promptDefs: repository}).InspectRequirements(ctx, RequirementInspectionRequest{PromptID: "selected"})
|
||||
if !errors.Is(err, ErrPromptLoad) || !errors.Is(err, context.Canceled) || repository.lookupCalls != 0 {
|
||||
t.Fatalf("canceled inspection=%v lookups=%d", err, repository.lookupCalls)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("no profile selection", func(t *testing.T) {
|
||||
repository := &requirementInspectionPromptRepository{definition: &domain.PromptDefinition{ID: "selected"}}
|
||||
_, err := (&Runner{promptDefs: repository}).InspectRequirements(context.Background(), RequirementInspectionRequest{PromptID: "selected"})
|
||||
if !errors.Is(err, ErrInvalidRequest) || !errors.Is(err, ErrProfileRequired) {
|
||||
t.Fatalf("missing profile error=%v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("ambiguous prompt selection", func(t *testing.T) {
|
||||
ambiguous := errors.New("ambiguous omitted version")
|
||||
repository := &requirementInspectionPromptRepository{lookupErr: ambiguous}
|
||||
_, err := (&Runner{promptDefs: repository}).InspectRequirements(context.Background(), RequirementInspectionRequest{PromptID: "selected"})
|
||||
if !errors.Is(err, ErrPromptLoad) || !errors.Is(err, ambiguous) || repository.lookupVersion != "" {
|
||||
t.Fatalf("ambiguous selection error=%v version=%q", err, repository.lookupVersion)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("captured snapshot unavailable", func(t *testing.T) {
|
||||
repository := &requirementInspectionPromptRepository{definition: &domain.PromptDefinition{ID: "selected", DefaultProfile: "profile"}}
|
||||
profiles := &inspectionProfileRepository{profile: &domain.ExecutionProfile{ID: "profile", ResolvedTarget: &domain.ExecutionTarget{}}}
|
||||
_, err := (&Runner{promptDefs: repository, profiles: profiles}).InspectRequirements(context.Background(), RequirementInspectionRequest{PromptID: "selected"})
|
||||
if !errors.Is(err, ErrPromptLoad) || repository.snapshotCalls != 1 {
|
||||
t.Fatalf("missing snapshot error=%v calls=%d", err, repository.snapshotCalls)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("repository lacks snapshot capability", func(t *testing.T) {
|
||||
_, err := (&Runner{promptDefs: &inspectionPromptRepository{definition: &domain.PromptDefinition{ID: "selected", DefaultProfile: "profile"}},
|
||||
profiles: &inspectionProfileRepository{profile: &domain.ExecutionProfile{ID: "profile", ResolvedTarget: &domain.ExecutionTarget{}}},
|
||||
}).InspectRequirements(context.Background(), RequirementInspectionRequest{PromptID: "selected"})
|
||||
if !errors.Is(err, ErrPromptLoad) {
|
||||
t.Fatalf("missing capability error=%v", err)
|
||||
}
|
||||
})
|
||||
if _, err := (*Runner)(nil).InspectRequirements(context.Background(), RequirementInspectionRequest{PromptID: "selected"}); !errors.Is(err, ErrPromptLoad) {
|
||||
t.Fatalf("nil runner error=%v", err)
|
||||
}
|
||||
if _, err := (&Runner{}).InspectRequirements(context.Background(), RequirementInspectionRequest{PromptID: "selected"}); !errors.Is(err, ErrPromptLoad) {
|
||||
t.Fatalf("nil repository error=%v", err)
|
||||
}
|
||||
if _, err := (&Runner{promptDefs: &requirementInspectionPromptRepository{definition: &domain.PromptDefinition{ID: "selected"}}}).InspectRequirements(
|
||||
context.Background(), RequirementInspectionRequest{PromptID: "selected", ProfileID: "explicit"},
|
||||
); !errors.Is(err, ErrProfileLoad) {
|
||||
t.Fatalf("missing profile error=%v", err)
|
||||
}
|
||||
if _, err := (&Runner{promptDefs: &requirementInspectionPromptRepository{definition: &domain.PromptDefinition{ID: "selected", DefaultProfile: "profile"}},
|
||||
profiles: &inspectionProfileRepository{profile: &domain.ExecutionProfile{ID: "profile"}},
|
||||
}).InspectRequirements(context.Background(), RequirementInspectionRequest{PromptID: "selected"}); !errors.Is(err, ErrProfileLoad) {
|
||||
t.Fatalf("unresolved profile error=%v", err)
|
||||
}
|
||||
if _, err := (&Runner{promptDefs: &requirementInspectionPromptRepository{lookupErr: promptdef.ErrPromptDefinitionNotFound}}).InspectRequirements(
|
||||
context.Background(), RequirementInspectionRequest{PromptID: "missing"},
|
||||
); !errors.Is(err, ErrPromptLoad) || !errors.Is(err, promptdef.ErrPromptDefinitionNotFound) {
|
||||
t.Fatalf("missing prompt error=%v", err)
|
||||
}
|
||||
}
|
||||
@@ -188,7 +188,7 @@ func validateArtifact(ctx context.Context, artifact *domain.Artifact, contract d
|
||||
res.IsValid = true
|
||||
return res, nil
|
||||
case domain.ValidationBasic:
|
||||
empty := strings.TrimSpace(string(artifact.Body)) == ""
|
||||
empty := len(bytes.TrimSpace(artifact.Body)) == 0
|
||||
if err := ctx.Err(); err != nil {
|
||||
return domain.ValidationResult{}, err
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package validate
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/url"
|
||||
@@ -35,32 +36,51 @@ func TestStandardValidatorNoneSkipped(t *testing.T) {
|
||||
|
||||
func TestStandardValidatorBasicSuccess(t *testing.T) {
|
||||
v := NewStandardValidator("")
|
||||
|
||||
res, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte("hello")}, domain.OutputContract{
|
||||
ValidationMode: domain.ValidationBasic,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
if res.Status != domain.ValidationPassed || !res.IsValid {
|
||||
t.Fatalf("expected passed/valid, got status=%q valid=%v", res.Status, res.IsValid)
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
body []byte
|
||||
}{
|
||||
{name: "text", body: []byte("hello")},
|
||||
{name: "invalid UTF-8 byte", body: []byte{0xff}},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
res, err := v.Validate(context.Background(), &domain.Artifact{Body: tc.body}, domain.OutputContract{
|
||||
ValidationMode: domain.ValidationBasic,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
if res.Status != domain.ValidationPassed || !res.IsValid {
|
||||
t.Fatalf("expected passed/valid, got status=%q valid=%v", res.Status, res.IsValid)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestStandardValidatorBasicFailureEmpty(t *testing.T) {
|
||||
v := NewStandardValidator("")
|
||||
|
||||
res, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte(" \n\t ")}, domain.OutputContract{
|
||||
ValidationMode: domain.ValidationBasic,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
if res.Status != domain.ValidationFailed || res.IsValid {
|
||||
t.Fatalf("expected failed/invalid, got status=%q valid=%v", res.Status, res.IsValid)
|
||||
}
|
||||
if len(res.Errors) == 0 {
|
||||
t.Fatal("expected validation errors")
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
body []byte
|
||||
}{
|
||||
{name: "empty bytes", body: []byte{}},
|
||||
{name: "ASCII whitespace", body: []byte(" \n\t ")},
|
||||
{name: "Unicode whitespace", body: []byte("\u2003\u00a0\u3000")},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
res, err := v.Validate(context.Background(), &domain.Artifact{Body: tc.body}, domain.OutputContract{
|
||||
ValidationMode: domain.ValidationBasic,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
if res.Status != domain.ValidationFailed || res.IsValid {
|
||||
t.Fatalf("expected failed/invalid, got status=%q valid=%v", res.Status, res.IsValid)
|
||||
}
|
||||
if len(res.Errors) == 0 {
|
||||
t.Fatal("expected validation errors")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -917,6 +937,49 @@ func BenchmarkJSONValidation(b *testing.B) {
|
||||
}
|
||||
}
|
||||
|
||||
var basicValidationBenchmarkResult domain.ValidationResult
|
||||
|
||||
func BenchmarkBasicValidation(b *testing.B) {
|
||||
largeText := bytes.Repeat([]byte("x"), 1<<20)
|
||||
largeWhitespace := bytes.Repeat([]byte(" \t\n"), (1<<20)/3)
|
||||
unicodeWhitespace := []byte("\u2003\u00a0\u3000")
|
||||
benchmarks := []struct {
|
||||
name string
|
||||
body []byte
|
||||
wantValid bool
|
||||
}{
|
||||
{name: "short_text", body: []byte("A concise response."), wantValid: true},
|
||||
{name: "1MiB_nonempty_text", body: largeText, wantValid: true},
|
||||
{name: "whitespace_only_text", body: largeWhitespace},
|
||||
{name: "unicode_whitespace", body: unicodeWhitespace},
|
||||
}
|
||||
validator := NewStandardValidator("")
|
||||
contract := domain.OutputContract{ValidationMode: domain.ValidationBasic}
|
||||
ctx := context.Background()
|
||||
|
||||
for _, benchmark := range benchmarks {
|
||||
b.Run(benchmark.name, func(b *testing.B) {
|
||||
artifact := &domain.Artifact{Body: benchmark.body}
|
||||
result, err := validator.Validate(ctx, artifact, contract)
|
||||
if err != nil || result.IsValid != benchmark.wantValid {
|
||||
b.Fatalf("validate basic output: result=%+v error=%v", result, err)
|
||||
}
|
||||
|
||||
b.ReportAllocs()
|
||||
// Nonempty input can be classified from its boundary bytes without
|
||||
// scanning the body, so report latency rather than bytes per second.
|
||||
b.ResetTimer()
|
||||
for range b.N {
|
||||
result, err := validator.Validate(ctx, artifact, contract)
|
||||
if err != nil || result.IsValid != benchmark.wantValid {
|
||||
b.Fatalf("validate basic output: result=%+v error=%v", result, err)
|
||||
}
|
||||
basicValidationBenchmarkResult = result
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestJSONSchemaDialectIsDraft2020(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
|
||||
67
listing.go
Normal file
67
listing.go
Normal file
@@ -0,0 +1,67 @@
|
||||
package promptkit
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// PromptIdentity identifies one prompt ID and exact version in the effective
|
||||
// configuration captured when an Engine is constructed. It has no stable JSON
|
||||
// representation.
|
||||
type PromptIdentity struct {
|
||||
ID string
|
||||
Version string
|
||||
}
|
||||
|
||||
// ListProfiles returns profile IDs in the effective configuration captured
|
||||
// when the engine was constructed. Maintained and consumer profiles are
|
||||
// included; an inherited profile retains its own ID. IDs are preserved exactly
|
||||
// and sorted lexically. The returned slice is caller-owned and safe to mutate
|
||||
// while the engine is used concurrently. Empty inventories succeed.
|
||||
//
|
||||
// Listing reports membership, not execution readiness. It does not reload
|
||||
// configuration or perform resource, credential, provider, or execution work.
|
||||
// Profile IDs are application data and may be sensitive. A nil or zero-value
|
||||
// engine returns an error matching ErrInvalidConfig. Cancellation or deadline
|
||||
// expiration returns the context error and no partial result.
|
||||
func (e *Engine) ListProfiles(ctx context.Context) ([]string, error) {
|
||||
if e == nil || e.internal == nil {
|
||||
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
|
||||
}
|
||||
|
||||
profiles, err := e.internal.ListProfiles(ctx)
|
||||
if err != nil {
|
||||
return nil, mapPublicError(err)
|
||||
}
|
||||
return profiles, nil
|
||||
}
|
||||
|
||||
// ListPrompts returns every prompt ID and exact version in the effective
|
||||
// configuration captured when the engine was constructed, sorted lexically by
|
||||
// ID and then version. Versions are not interpreted or prioritized. The
|
||||
// returned slice is caller-owned and safe to mutate while the engine is used
|
||||
// concurrently. Empty inventories succeed.
|
||||
//
|
||||
// Listing reports membership, not execution readiness. It does not reload
|
||||
// configuration or perform resource, credential, provider, or execution work.
|
||||
// Prompt identities are application data and may be sensitive. A nil or
|
||||
// zero-value engine returns an error matching ErrInvalidConfig. Cancellation
|
||||
// or deadline expiration returns the context error and no partial result.
|
||||
func (e *Engine) ListPrompts(ctx context.Context) ([]PromptIdentity, error) {
|
||||
if e == nil || e.internal == nil {
|
||||
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
|
||||
}
|
||||
|
||||
identities, err := e.internal.ListPrompts(ctx)
|
||||
if err != nil {
|
||||
return nil, mapPublicError(err)
|
||||
}
|
||||
public := make([]PromptIdentity, len(identities))
|
||||
for i, identity := range identities {
|
||||
public[i] = PromptIdentity{ID: identity.ID, Version: identity.Version}
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, mapPublicError(err)
|
||||
}
|
||||
return public, nil
|
||||
}
|
||||
379
listing_contract_test.go
Normal file
379
listing_contract_test.go
Normal file
@@ -0,0 +1,379 @@
|
||||
package promptkit_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"sync"
|
||||
"testing"
|
||||
"testing/fstest"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
func TestListIdentitiesBasicContract(t *testing.T) {
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
|
||||
Source: configkit.Mapping("listing", map[string]any{
|
||||
"profiles": map[string]any{"configured-profile": map[string]any{"model": "example-model"}},
|
||||
"prompts": map[string]any{"example-prompt": map[string]any{
|
||||
"2": listingContractPrompt(),
|
||||
"10": listingContractPrompt(),
|
||||
}},
|
||||
}),
|
||||
}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
profiles, err := engine.ListProfiles(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !containsListingProfile(profiles, "configured-profile") {
|
||||
t.Fatalf("profile listing %#v does not include configured profile", profiles)
|
||||
}
|
||||
|
||||
prompts, err := engine.ListPrompts(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := []promptkit.PromptIdentity{
|
||||
{ID: "example-prompt", Version: "10"},
|
||||
{ID: "example-prompt", Version: "2"},
|
||||
}
|
||||
if len(prompts) != len(want) {
|
||||
t.Fatalf("prompt listing = %#v, want %#v", prompts, want)
|
||||
}
|
||||
for i := range want {
|
||||
if prompts[i] != want[i] {
|
||||
t.Fatalf("prompt listing = %#v, want %#v", prompts, want)
|
||||
}
|
||||
}
|
||||
|
||||
profile, err := engine.InspectProfile(context.Background(), "configured-profile")
|
||||
if err != nil || profile == nil || profile.ProfileID != "configured-profile" {
|
||||
t.Fatalf("listed profile could not be inspected: (%+v, %v)", profile, err)
|
||||
}
|
||||
prompt, err := engine.InspectPrompt(context.Background(), prompts[0].ID, prompts[0].Version)
|
||||
if err != nil || prompt == nil || prompt.PromptID != prompts[0].ID || prompt.PromptVersion != prompts[0].Version {
|
||||
t.Fatalf("listed prompt could not be inspected: (%+v, %v)", prompt, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListIdentitiesInvalidEngine(t *testing.T) {
|
||||
var nilEngine *promptkit.Engine
|
||||
var zeroEngine promptkit.Engine
|
||||
for _, engine := range []*promptkit.Engine{nilEngine, &zeroEngine} {
|
||||
if _, err := engine.ListProfiles(context.Background()); !errors.Is(err, promptkit.ErrInvalidConfig) {
|
||||
t.Errorf("ListProfiles on invalid engine: %v", err)
|
||||
}
|
||||
if _, err := engine.ListPrompts(context.Background()); !errors.Is(err, promptkit.ErrInvalidConfig) {
|
||||
t.Errorf("ListPrompts on invalid engine: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestListIdentitiesPreserveContextErrorsWithoutPartialResults(t *testing.T) {
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
|
||||
Source: configkit.Mapping("listing", map[string]any{
|
||||
"profiles": map[string]any{"profile": map[string]any{}},
|
||||
"prompts": map[string]any{"example": map[string]any{"v1": listingContractPrompt()}},
|
||||
}),
|
||||
}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
canceled, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
expired, expire := context.WithDeadline(context.Background(), time.Now().Add(-time.Second))
|
||||
defer expire()
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
ctx context.Context
|
||||
want error
|
||||
}{
|
||||
{name: "canceled", ctx: canceled, want: context.Canceled},
|
||||
{name: "expired", ctx: expired, want: context.DeadlineExceeded},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
profiles, err := engine.ListProfiles(test.ctx)
|
||||
if !errors.Is(err, test.want) || len(profiles) != 0 {
|
||||
t.Errorf("ListProfiles = %#v, %v; want no results and %v", profiles, err, test.want)
|
||||
}
|
||||
prompts, err := engine.ListPrompts(test.ctx)
|
||||
if !errors.Is(err, test.want) || len(prompts) != 0 {
|
||||
t.Errorf("ListPrompts = %#v, %v; want no results and %v", prompts, err, test.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestListIdentitiesUseEffectiveLayeredMembership(t *testing.T) {
|
||||
underlay := configkit.Input{Source: configkit.Mapping("consumer defaults", map[string]any{
|
||||
"profiles": map[string]any{
|
||||
"base/profile": map[string]any{"endpoint": "https://example.test/v1", "model": "base-model"},
|
||||
"child/profile": map[string]any{"extends": "base/profile"},
|
||||
"overlay/profile": map[string]any{"endpoint": "https://example.test/v1", "model": "old-model"},
|
||||
},
|
||||
"prompts": map[string]any{
|
||||
"shared/prompt": map[string]any{
|
||||
"v1": listingContractPrompt(),
|
||||
"v2": listingContractPrompt(),
|
||||
},
|
||||
"underlay-only": map[string]any{"v1": listingContractPrompt()},
|
||||
},
|
||||
})}
|
||||
main := configkit.Input{Source: configkit.Mapping("application overlay", map[string]any{
|
||||
"profiles": map[string]any{
|
||||
"overlay/profile": map[string]any{"model": "effective-model"},
|
||||
},
|
||||
"prompts": map[string]any{
|
||||
"shared/prompt": map[string]any{"v1": listingContractPrompt()},
|
||||
"main-only": map[string]any{"v3": listingContractPrompt()},
|
||||
},
|
||||
})}
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{underlay}}},
|
||||
Main: &main,
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
profiles, err := engine.ListProfiles(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, id := range []string{
|
||||
"base/profile", "child/profile", "overlay/profile",
|
||||
"promptkit-openrouter-standard", "promptkit-openrouter-auto",
|
||||
} {
|
||||
if !containsListingProfile(profiles, id) {
|
||||
t.Errorf("profile listing %#v does not include %q", profiles, id)
|
||||
}
|
||||
}
|
||||
for _, id := range []string{"base/profile", "child/profile", "overlay/profile"} {
|
||||
if got := countListingProfile(profiles, id); got != 1 {
|
||||
t.Errorf("profile %q appears %d times, want 1", id, got)
|
||||
}
|
||||
}
|
||||
|
||||
overlay, err := engine.InspectProfile(context.Background(), "overlay/profile")
|
||||
if err != nil || overlay == nil || overlay.EffectiveModelParams.Model != "effective-model" {
|
||||
t.Fatalf("overlay profile did not resolve to its effective definition: (%+v, %v)", overlay, err)
|
||||
}
|
||||
child, err := engine.InspectProfile(context.Background(), "child/profile")
|
||||
if err != nil || child == nil || child.EffectiveModelParams.Model != "base-model" {
|
||||
t.Fatalf("inherited child profile did not resolve its effective definition: (%+v, %v)", child, err)
|
||||
}
|
||||
|
||||
prompts, err := engine.ListPrompts(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wantPrompts := []promptkit.PromptIdentity{
|
||||
{ID: "main-only", Version: "v3"},
|
||||
{ID: "shared/prompt", Version: "v1"},
|
||||
{ID: "shared/prompt", Version: "v2"},
|
||||
{ID: "underlay-only", Version: "v1"},
|
||||
}
|
||||
if !reflect.DeepEqual(prompts, wantPrompts) {
|
||||
t.Fatalf("effective prompt identities = %#v, want %#v", prompts, wantPrompts)
|
||||
}
|
||||
if got := countListingPrompt(prompts, "shared/prompt", "v1"); got != 1 {
|
||||
t.Fatalf("overlaid prompt identity appears %d times, want 1", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListIdentitiesEmptyEffectiveConfiguration(t *testing.T) {
|
||||
main := configkit.Input{Source: configkit.Mapping("clear effective identities", map[string]any{
|
||||
"profiles": nil,
|
||||
"prompts": nil,
|
||||
})}
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &main}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
profiles, err := engine.ListProfiles(context.Background())
|
||||
if err != nil || len(profiles) != 0 {
|
||||
t.Fatalf("cleared profile listing = %#v, %v", profiles, err)
|
||||
}
|
||||
prompts, err := engine.ListPrompts(context.Background())
|
||||
if err != nil || len(prompts) != 0 {
|
||||
t.Fatalf("cleared prompt listing = %#v, %v", prompts, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListIdentitiesAreOwnedAndSafeForConcurrentUse(t *testing.T) {
|
||||
engine, err := newListingContractEngine(t, map[string]any{
|
||||
"profiles": map[string]any{"profile": map[string]any{"model": "partial-model"}},
|
||||
"prompts": map[string]any{"prompt": map[string]any{"v1": listingContractPrompt()}},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
profiles, err := engine.ListProfiles(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
prompts, err := engine.ListPrompts(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i := range profiles {
|
||||
if profiles[i] == "profile" {
|
||||
profiles[i] = "caller mutation"
|
||||
}
|
||||
}
|
||||
for i := range prompts {
|
||||
if prompts[i].ID == "prompt" {
|
||||
prompts[i].ID = "caller mutation"
|
||||
prompts[i].Version = "caller mutation"
|
||||
}
|
||||
}
|
||||
profiles, err = engine.ListProfiles(context.Background())
|
||||
if err != nil || !containsListingProfile(profiles, "profile") {
|
||||
t.Fatalf("profile result mutation escaped into engine: %#v, %v", profiles, err)
|
||||
}
|
||||
prompts, err = engine.ListPrompts(context.Background())
|
||||
if err != nil || countListingPrompt(prompts, "prompt", "v1") != 1 {
|
||||
t.Fatalf("prompt result mutation escaped into engine: %#v, %v", prompts, err)
|
||||
}
|
||||
|
||||
const workers = 4
|
||||
const iterations = 25
|
||||
start := make(chan struct{})
|
||||
results := make(chan error, workers)
|
||||
var wait sync.WaitGroup
|
||||
wait.Add(workers)
|
||||
for worker := 0; worker < workers; worker++ {
|
||||
go func() {
|
||||
defer wait.Done()
|
||||
<-start
|
||||
for iteration := 0; iteration < iterations; iteration++ {
|
||||
profiles, err := engine.ListProfiles(context.Background())
|
||||
if err != nil {
|
||||
results <- fmt.Errorf("list profiles: %w", err)
|
||||
return
|
||||
}
|
||||
if len(profiles) == 0 {
|
||||
results <- errors.New("profile listing is empty")
|
||||
return
|
||||
}
|
||||
profiles[0] = "worker mutation"
|
||||
|
||||
prompts, err := engine.ListPrompts(context.Background())
|
||||
if err != nil {
|
||||
results <- fmt.Errorf("list prompts: %w", err)
|
||||
return
|
||||
}
|
||||
if len(prompts) == 0 {
|
||||
results <- errors.New("prompt listing is empty")
|
||||
return
|
||||
}
|
||||
prompts[0].ID = "worker mutation"
|
||||
prompts[0].Version = "worker mutation"
|
||||
|
||||
if _, err := engine.InspectProfile(context.Background(), "profile"); err != nil {
|
||||
results <- fmt.Errorf("inspect profile: %w", err)
|
||||
return
|
||||
}
|
||||
if _, err := engine.InspectPrompt(context.Background(), "prompt", "v1"); err != nil {
|
||||
results <- fmt.Errorf("inspect prompt: %w", err)
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
close(start)
|
||||
wait.Wait()
|
||||
close(results)
|
||||
for err := range results {
|
||||
t.Error(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListIdentitiesUseCapturedSourcesWithoutExecutionWork(t *testing.T) {
|
||||
files := fstest.MapFS{
|
||||
"config/listing.yaml": &fstest.MapFile{Data: []byte("profiles:\n captured-partial:\n model: partial-model\nprompts:\n captured-prompt:\n v1:\n messages:\n - role: user\n content: Hello.\n output:\n format: text\n validation_mode: basic\n")},
|
||||
}
|
||||
inputs, err := promptkit.FilesystemInputs("application listing", files, "config")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||
Underlays: []configkit.Underlay{{Inputs: inputs}},
|
||||
}}, promptkit.WithLLMClient(rejectingListingLLMClient{}), promptkit.WithArtifactReader(rejectingListingArtifactReader{}))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
files["config/listing.yaml"].Data = []byte("profiles:\n replacement-profile: {}\nprompts:\n replacement-prompt:\n v2:\n messages:\n - role: user\n content: Changed.\n output:\n format: text\n validation_mode: basic\n")
|
||||
|
||||
profiles, err := engine.ListProfiles(context.Background())
|
||||
if err != nil || !containsListingProfile(profiles, "captured-partial") || containsListingProfile(profiles, "replacement-profile") {
|
||||
t.Fatalf("profile listing did not use captured source: %#v, %v", profiles, err)
|
||||
}
|
||||
prompts, err := engine.ListPrompts(context.Background())
|
||||
if err != nil || countListingPrompt(prompts, "captured-prompt", "v1") != 1 || countListingPrompt(prompts, "replacement-prompt", "v2") != 0 {
|
||||
t.Fatalf("prompt listing did not use captured source: %#v, %v", prompts, err)
|
||||
}
|
||||
}
|
||||
|
||||
func listingContractPrompt() map[string]any {
|
||||
return map[string]any{
|
||||
"messages": []any{map[string]any{"role": "user", "content": "Hello."}},
|
||||
"output": map[string]any{"format": "text", "validation_mode": "basic"},
|
||||
}
|
||||
}
|
||||
|
||||
func containsListingProfile(profiles []string, id string) bool {
|
||||
for _, profile := range profiles {
|
||||
if profile == id {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func countListingProfile(profiles []string, id string) int {
|
||||
count := 0
|
||||
for _, profile := range profiles {
|
||||
if profile == id {
|
||||
count++
|
||||
}
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
func countListingPrompt(prompts []promptkit.PromptIdentity, id, version string) int {
|
||||
count := 0
|
||||
for _, prompt := range prompts {
|
||||
if prompt.ID == id && prompt.Version == version {
|
||||
count++
|
||||
}
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
func newListingContractEngine(t *testing.T, values map[string]any, options ...promptkit.Option) (*promptkit.Engine, error) {
|
||||
t.Helper()
|
||||
input := configkit.Input{Source: configkit.Mapping("listing contract", values)}
|
||||
return promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &input}}, options...)
|
||||
}
|
||||
|
||||
type rejectingListingLLMClient struct{}
|
||||
|
||||
func (rejectingListingLLMClient) Generate(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
return nil, errors.New("unexpected model generation")
|
||||
}
|
||||
|
||||
type rejectingListingArtifactReader struct{}
|
||||
|
||||
func (rejectingListingArtifactReader) Read(context.Context, promptkit.ArtifactRef) (*promptkit.Artifact, error) {
|
||||
return nil, errors.New("unexpected artifact read")
|
||||
}
|
||||
29
releasekit.json
Normal file
29
releasekit.json
Normal file
@@ -0,0 +1,29 @@
|
||||
{
|
||||
"product": "Promptkit",
|
||||
"remote": "origin",
|
||||
"validation": {
|
||||
"go": {
|
||||
"race": true,
|
||||
"full_vet": true
|
||||
},
|
||||
"go_format": true,
|
||||
"markdown": {},
|
||||
"module_distribution": {
|
||||
"workspace_files": true,
|
||||
"local_replacements": true
|
||||
}
|
||||
},
|
||||
"checks": [
|
||||
{
|
||||
"args": ["python3", "./scripts/check-examples.py"],
|
||||
"env": {
|
||||
"GOTOOLCHAIN": "local",
|
||||
"GOWORK": "off",
|
||||
"GOFLAGS": "-mod=readonly",
|
||||
"GOPROXY": "off",
|
||||
"GONOPROXY": "none",
|
||||
"GOSUMDB": "off"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
240
requirements.go
Normal file
240
requirements.go
Normal file
@@ -0,0 +1,240 @@
|
||||
package promptkit
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/usecase"
|
||||
)
|
||||
|
||||
// RequirementInspectionRequest selects a prompt and supplies caller context
|
||||
// for requirement discovery. It requires no artifact bindings or credentials.
|
||||
type RequirementInspectionRequest struct {
|
||||
// PromptID is the exact, case-sensitive configured prompt ID.
|
||||
PromptID string
|
||||
// PromptVersion selects an exact version. An empty value uses the prompt
|
||||
// selection rule that requires exactly one configured version.
|
||||
PromptVersion string
|
||||
// ProfileID selects the exact effective profile identity. If empty, the
|
||||
// prompt's default profile is used; the profile need not be ready for
|
||||
// execution.
|
||||
ProfileID string
|
||||
// Vars supplies string values for caller-variable lookups and simple
|
||||
// conditions. Presence is determined by exact map key, including an empty
|
||||
// value. Inspection reports caller keys and presence, not their values.
|
||||
// A supplied value may nevertheless appear as a resolved input identifier.
|
||||
Vars map[string]string
|
||||
// SessionID, when nonblank after the normal session-ID normalization,
|
||||
// bypasses the prompt's session template during discovery.
|
||||
SessionID string
|
||||
}
|
||||
|
||||
// RequirementKind describes how a selected prompt may depend on an input or
|
||||
// caller variable for the supplied context. It is independent of whether a
|
||||
// caller-variable value is present.
|
||||
type RequirementKind string
|
||||
|
||||
const (
|
||||
// RequirementRequired identifies an input declared required or a dependency
|
||||
// reached by a recognized definite access.
|
||||
RequirementRequired RequirementKind = "required"
|
||||
// RequirementConditional identifies a dependency whose reachability or
|
||||
// identity remains unresolved for the supplied context.
|
||||
RequirementConditional RequirementKind = "conditional"
|
||||
// RequirementOptional identifies a dependency known not to be required,
|
||||
// including a nonrequired declaration, optional lookup, or reference used
|
||||
// only on a known skipped path.
|
||||
RequirementOptional RequirementKind = "optional"
|
||||
)
|
||||
|
||||
// InputRequirement describes a declared or template-referenced prompt input.
|
||||
type InputRequirement struct {
|
||||
// Name is the exact input identifier.
|
||||
Name string
|
||||
// Kind is the strongest classification established for this name.
|
||||
Kind RequirementKind
|
||||
// Declared reports whether the name appears in the effective input
|
||||
// declarations; helper-only references have Declared false.
|
||||
Declared bool
|
||||
// ContentType preserves effective metadata presence: nil means absent or
|
||||
// null, while a non-nil pointer may refer to an empty string.
|
||||
ContentType *string
|
||||
// Description preserves effective metadata presence independently of
|
||||
// Explanation, including an explicitly empty string.
|
||||
Description *string
|
||||
// Explanation is safe human-readable context, such as why a reference is
|
||||
// optional because it occurs only on a known skipped path. It is not a
|
||||
// machine-readable contract.
|
||||
Explanation string
|
||||
}
|
||||
|
||||
// VariableRequirement describes a caller-map key referenced by a prompt.
|
||||
type VariableRequirement struct {
|
||||
// Name is the exact caller-map key, not a template-local name.
|
||||
Name string
|
||||
// Kind is the strongest classification established for this key.
|
||||
Kind RequirementKind
|
||||
// Supplied reports exact-key presence in the request Vars map. An empty
|
||||
// string is supplied; the value itself is not returned.
|
||||
Supplied bool
|
||||
// Explanation is safe human-readable context and is not a machine-readable
|
||||
// contract.
|
||||
Explanation string
|
||||
}
|
||||
|
||||
// RequirementIssueCode identifies a limitation that can affect discovery.
|
||||
type RequirementIssueCode string
|
||||
|
||||
const (
|
||||
// RequirementIssueMissingVariable means an absent caller value prevents
|
||||
// resolving a dependency decision or input name.
|
||||
RequirementIssueMissingVariable RequirementIssueCode = "missing_variable"
|
||||
// RequirementIssueInputContent means unavailable input content affects
|
||||
// dependency discovery.
|
||||
RequirementIssueInputContent RequirementIssueCode = "input_content"
|
||||
// RequirementIssueDynamicInputName means a referenced input name cannot be
|
||||
// determined for the supplied context.
|
||||
RequirementIssueDynamicInputName RequirementIssueCode = "dynamic_input_name"
|
||||
// RequirementIssueUnsupportedConstruct means a dependency-relevant
|
||||
// expression or scope is outside the supported discovery analysis.
|
||||
RequirementIssueUnsupportedConstruct RequirementIssueCode = "unsupported_construct"
|
||||
)
|
||||
|
||||
// RequirementIssue describes one discovery limitation using safe summary
|
||||
// text. Descriptions are not intended as machine-readable details.
|
||||
type RequirementIssue struct {
|
||||
// Code is the machine-readable limitation category.
|
||||
Code RequirementIssueCode
|
||||
// InputName identifies an affected input when one is known; empty means no
|
||||
// particular input was identified.
|
||||
InputName string
|
||||
// VariableName identifies an affected caller key when one is known; empty
|
||||
// means no particular key was identified.
|
||||
VariableName string
|
||||
// Description is a safe human-readable summary, not a string to parse.
|
||||
Description string
|
||||
}
|
||||
|
||||
// RequirementInspection contains discovered dependencies for one selected
|
||||
// prompt context. Input and variable entries are sorted by exact name. Issues
|
||||
// are deduplicated by code and associated names, then sorted by that tuple.
|
||||
// Their descriptions and entry explanations are safe summaries, not
|
||||
// machine-readable details.
|
||||
//
|
||||
// Complete describes the supported analysis's discovery coverage, not whether
|
||||
// requirements are supplied, templates will render, artifacts or schemas are
|
||||
// valid, or execution is ready. A complete result can still contain required
|
||||
// inputs or missing required variables. Valid templates using dependency-
|
||||
// relevant constructs outside the deliberately small recognition scope may
|
||||
// produce useful partial inventories with Complete false; this is not a
|
||||
// template-definition error or an authoring prohibition.
|
||||
type RequirementInspection struct {
|
||||
// PromptID is the selected exact prompt ID.
|
||||
PromptID string
|
||||
// PromptVersion is the selected exact prompt version.
|
||||
PromptVersion string
|
||||
// ProfileID is the selected profile identity; it does not indicate
|
||||
// execution readiness.
|
||||
ProfileID string
|
||||
// Inputs contains declared inputs and recognizable template references,
|
||||
// including entries known to be optional.
|
||||
Inputs []InputRequirement
|
||||
// Variables contains recognizable caller-map keys, including keys used
|
||||
// only on known skipped paths.
|
||||
Variables []VariableRequirement
|
||||
// Complete reports discovery coverage, not requirement satisfaction or
|
||||
// execution readiness.
|
||||
Complete bool
|
||||
// Issues contains deduplicated, structured discovery limitations.
|
||||
Issues []RequirementIssue
|
||||
}
|
||||
|
||||
// InspectRequirements discovers inputs and caller variables for one selected
|
||||
// prompt before its input artifacts exist. It uses exact, case-sensitive
|
||||
// prompt selection and requires a profile ID either in the request or the
|
||||
// selected prompt's default. The selected profile establishes identity only;
|
||||
// it need not be execution-ready. A normalized nonblank SessionID bypasses the
|
||||
// captured session template. Vars are copied before analysis, and returned
|
||||
// slices and metadata pointers belong to the caller.
|
||||
//
|
||||
// Discovery recognizes literal input names, direct caller-map keys, literal-
|
||||
// key root-map lookups, simple caller-derived input names, and if/with
|
||||
// conditions based on direct values, literals, negation, or two-string
|
||||
// equality/inequality. It classifies recognizable references on known skipped
|
||||
// paths as optional. Other dependency-relevant expressions or scopes may leave
|
||||
// a useful partial result with Complete false. The returned inventories do
|
||||
// not reject otherwise valid templates or prohibit their use.
|
||||
//
|
||||
// Results exclude artifact payloads, prompt contents, and unrelated supplied
|
||||
// values. Resolved input identifiers may equal the caller values that supply
|
||||
// them; those identifiers are intentionally included in the inventory.
|
||||
//
|
||||
// This method does not perform artifact, schema, credential, rendering,
|
||||
// validation, capacity, observation, provider, or execution work. Complete
|
||||
// describes dependency discovery, not whether every dependency is supplied or
|
||||
// the prompt is ready to execute. Discovery limitations return a partial
|
||||
// inspection and no error; selection and operational failures return no
|
||||
// inspection. Blank prompt IDs, invalid session IDs, and absent profile
|
||||
// selection match ErrInvalidRequest; an absent profile selection also matches
|
||||
// ErrProfileRequired. Missing prompt/profile IDs match ErrPromptNotFound or
|
||||
// ErrProfileNotFound. Other selection/snapshot failures match ErrPromptLoad or
|
||||
// ErrProfileLoad. When cancellation prevents lookup or analysis, no inspection
|
||||
// is returned and the context error remains discoverable with errors.Is. A nil
|
||||
// Engine matches ErrInvalidConfig.
|
||||
func (e *Engine) InspectRequirements(
|
||||
ctx context.Context,
|
||||
req RequirementInspectionRequest,
|
||||
) (*RequirementInspection, error) {
|
||||
if e == nil || e.internal == nil {
|
||||
return nil, ErrInvalidConfig
|
||||
}
|
||||
inspection, err := e.internal.InspectRequirements(ctx, usecase.RequirementInspectionRequest{
|
||||
PromptID: req.PromptID, PromptVersion: req.PromptVersion, ProfileID: req.ProfileID,
|
||||
Vars: copyStringMap(req.Vars), SessionID: req.SessionID,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, mapPublicError(err)
|
||||
}
|
||||
return fromDomainRequirementInspection(inspection), nil
|
||||
}
|
||||
|
||||
func fromDomainRequirementInspection(inspection *domain.RequirementInspection) *RequirementInspection {
|
||||
if inspection == nil {
|
||||
return nil
|
||||
}
|
||||
result := &RequirementInspection{
|
||||
PromptID: inspection.PromptID, PromptVersion: inspection.PromptVersion,
|
||||
ProfileID: inspection.ProfileID, Complete: inspection.Complete,
|
||||
Inputs: make([]InputRequirement, len(inspection.Inputs)),
|
||||
Variables: make([]VariableRequirement, len(inspection.Variables)),
|
||||
Issues: make([]RequirementIssue, len(inspection.Issues)),
|
||||
}
|
||||
for index, input := range inspection.Inputs {
|
||||
result.Inputs[index] = InputRequirement{
|
||||
Name: input.Name, Kind: RequirementKind(input.Kind), Declared: input.Declared,
|
||||
ContentType: copyRequirementString(input.ContentType), Description: copyRequirementString(input.Description),
|
||||
Explanation: input.Explanation,
|
||||
}
|
||||
}
|
||||
for index, variable := range inspection.Variables {
|
||||
result.Variables[index] = VariableRequirement{
|
||||
Name: variable.Name, Kind: RequirementKind(variable.Kind), Supplied: variable.Supplied,
|
||||
Explanation: variable.Explanation,
|
||||
}
|
||||
}
|
||||
for index, issue := range inspection.Issues {
|
||||
result.Issues[index] = RequirementIssue{
|
||||
Code: RequirementIssueCode(issue.Code), InputName: issue.InputName,
|
||||
VariableName: issue.VariableName, Description: issue.Description,
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func copyRequirementString(value *string) *string {
|
||||
if value == nil {
|
||||
return nil
|
||||
}
|
||||
copy := *value
|
||||
return ©
|
||||
}
|
||||
72
requirements_inspection_benchmark_test.go
Normal file
72
requirements_inspection_benchmark_test.go
Normal file
@@ -0,0 +1,72 @@
|
||||
package promptkit_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
var inspectedRequirements *promptkit.RequirementInspection
|
||||
|
||||
func requirementInspectionExtras(entries int) map[string]any {
|
||||
providers := make(map[string]any, entries)
|
||||
for index := range entries {
|
||||
providers[fmt.Sprintf("provider-%03d", index)] = map[string]any{
|
||||
"enabled": true,
|
||||
"labels": []any{fmt.Sprintf("tier-%d", index%4), fmt.Sprintf("region-%d", index%3)},
|
||||
"limits": map[string]any{"burst": index + 1, "period": "minute"},
|
||||
"routes": []any{map[string]any{"weight": index + 1, "fallback": true}},
|
||||
}
|
||||
}
|
||||
return map[string]any{"provider_options": providers}
|
||||
}
|
||||
|
||||
func BenchmarkRequirementInspection(b *testing.B) {
|
||||
profiles := map[string]any{
|
||||
"small": map[string]any{
|
||||
"model": "inspection-model", "extra_params": requirementInspectionExtras(2),
|
||||
},
|
||||
"large": map[string]any{
|
||||
"model": "inspection-model", "extra_params": requirementInspectionExtras(64),
|
||||
},
|
||||
}
|
||||
definition := contractPrompt("small", "Hello {{.name}} {{input \"transcript\"}}", []any{
|
||||
map[string]any{"name": "transcript", "required": true},
|
||||
}, "", noValidationOutput)
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
|
||||
Source: configkit.Mapping("requirement inspection benchmark", map[string]any{
|
||||
"prompts": oneVersionPrompt("inspection", "1", definition),
|
||||
"profiles": profiles,
|
||||
}),
|
||||
}}})
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
ctx := context.Background()
|
||||
for _, profileID := range []string{"small", "large"} {
|
||||
request := promptkit.RequirementInspectionRequest{
|
||||
PromptID: "inspection", ProfileID: profileID, Vars: map[string]string{"name": "Ada"},
|
||||
}
|
||||
inspection, err := engine.InspectRequirements(ctx, request)
|
||||
if err != nil || inspection.PromptID != "inspection" || inspection.PromptVersion != "1" ||
|
||||
inspection.ProfileID != profileID || !inspection.Complete || len(inspection.Inputs) != 1 ||
|
||||
inspection.Inputs[0].Name != "transcript" || inspection.Inputs[0].Kind != promptkit.RequirementRequired ||
|
||||
len(inspection.Variables) != 1 || inspection.Variables[0].Name != "name" || !inspection.Variables[0].Supplied {
|
||||
b.Fatalf("requirement inspection fixture for %q is invalid: inspection=%+v err=%v", profileID, inspection, err)
|
||||
}
|
||||
selectedRequest := request
|
||||
b.Run(profileID+"_extra", func(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
for b.Loop() {
|
||||
inspection, err := engine.InspectRequirements(ctx, selectedRequest)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
inspectedRequirements = inspection
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
370
requirements_inspection_contract_test.go
Normal file
370
requirements_inspection_contract_test.go
Normal file
@@ -0,0 +1,370 @@
|
||||
package promptkit_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"testing/fstest"
|
||||
|
||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
func requirementsEngine(t *testing.T, prompts, profiles map[string]any, options ...promptkit.Option) *promptkit.Engine {
|
||||
t.Helper()
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
|
||||
Source: configkit.Mapping("requirements contract", map[string]any{
|
||||
"prompts": prompts, "profiles": profiles,
|
||||
}),
|
||||
}}}, options...)
|
||||
if err != nil {
|
||||
t.Fatalf("construct requirements engine: %v", err)
|
||||
}
|
||||
return engine
|
||||
}
|
||||
|
||||
type requirementCountingArtifactReader struct{ calls atomic.Int64 }
|
||||
|
||||
func (reader *requirementCountingArtifactReader) Read(context.Context, promptkit.ArtifactRef) (*promptkit.Artifact, error) {
|
||||
reader.calls.Add(1)
|
||||
return nil, errors.New("artifact reader must not be called")
|
||||
}
|
||||
|
||||
func contractPrompt(defaultProfile string, content string, inputs []any, session string, output map[string]any) map[string]any {
|
||||
definition := map[string]any{
|
||||
"messages": []any{map[string]any{"role": "user", "content": content}},
|
||||
"output": output,
|
||||
}
|
||||
if defaultProfile != "" {
|
||||
definition["default_profile"] = defaultProfile
|
||||
}
|
||||
if inputs != nil {
|
||||
definition["inputs"] = inputs
|
||||
}
|
||||
if session != "" {
|
||||
definition["session_id"] = session
|
||||
}
|
||||
return definition
|
||||
}
|
||||
|
||||
func oneVersionPrompt(id, version string, definition map[string]any) map[string]any {
|
||||
return map[string]any{id: map[string]any{version: definition}}
|
||||
}
|
||||
|
||||
var noValidationOutput = map[string]any{"format": "text", "validation_mode": "none"}
|
||||
|
||||
func TestInspectRequirementsEnumeratesCapturedDependenciesAndInactiveNames(t *testing.T) {
|
||||
var modelCalls atomic.Int64
|
||||
reader := &requirementCountingArtifactReader{}
|
||||
definition := contractPrompt("partial", `TEMPLATE_PRIVATE_MARKER {{if eq .mode "brief"}}{{input "brief"}}{{else}}{{input "long"}}{{end}} {{input "undeclared"}} {{.required}} {{if false}}{{input "inactive"}}{{.inactive_var}}{{end}}`, []any{
|
||||
map[string]any{"name": "transcript", "required": true, "content_type": "audio/wav", "description": "Transcript source."},
|
||||
map[string]any{"name": "optional", "content_type": "text/plain", "description": ""},
|
||||
}, `{{input "session-only"}} {{.session_variable}}`, map[string]any{
|
||||
"format": "json", "validation_mode": "json_schema", "schema_path": "not-loaded.json",
|
||||
})
|
||||
engine := requirementsEngine(t, oneVersionPrompt("requirements", "v1", definition), map[string]any{
|
||||
"partial": map[string]any{"model": "partial-model", "api_key_required": true},
|
||||
}, promptkit.WithArtifactReader(reader), promptkit.WithLLMClient(countingLLMClient{calls: &modelCalls}))
|
||||
inspection, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{
|
||||
PromptID: "requirements", Vars: map[string]string{"mode": "brief", "unrelated": "VARIABLE_PRIVATE_MARKER"},
|
||||
SessionID: "SESSION_PRIVATE_MARKER",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("inspect before artifacts, credentials, and schemas exist: %v", err)
|
||||
}
|
||||
if inspection.PromptID != "requirements" || inspection.PromptVersion != "v1" || inspection.ProfileID != "partial" || !inspection.Complete {
|
||||
t.Fatalf("selected context or completeness is wrong: %+v", inspection)
|
||||
}
|
||||
inputs := make(map[string]promptkit.InputRequirement, len(inspection.Inputs))
|
||||
for _, input := range inspection.Inputs {
|
||||
inputs[input.Name] = input
|
||||
}
|
||||
if len(inputs) != 6 || inputs["transcript"].Kind != promptkit.RequirementRequired || !inputs["transcript"].Declared ||
|
||||
inputs["transcript"].ContentType == nil || *inputs["transcript"].ContentType != "audio/wav" ||
|
||||
inputs["transcript"].Description == nil || *inputs["transcript"].Description != "Transcript source." {
|
||||
t.Fatalf("required declaration metadata or discovered inputs are wrong: %+v", inspection.Inputs)
|
||||
}
|
||||
if inputs["brief"].Kind != promptkit.RequirementRequired || inputs["long"].Kind != promptkit.RequirementOptional || inputs["long"].Explanation == "" ||
|
||||
inputs["inactive"].Kind != promptkit.RequirementOptional || inputs["inactive"].Explanation == "" ||
|
||||
inputs["undeclared"].Kind != promptkit.RequirementRequired || inputs["undeclared"].Declared ||
|
||||
inputs["optional"].Kind != promptkit.RequirementOptional || !inputs["optional"].Declared ||
|
||||
inputs["optional"].Description == nil || *inputs["optional"].Description != "" {
|
||||
t.Fatalf("selected, inactive, or helper-only input classifications are wrong: %+v", inspection.Inputs)
|
||||
}
|
||||
if _, found := inputs["session-only"]; found {
|
||||
t.Fatalf("bypassed session input appeared in inventory: %+v", inspection.Inputs)
|
||||
}
|
||||
variables := make(map[string]promptkit.VariableRequirement, len(inspection.Variables))
|
||||
for _, variable := range inspection.Variables {
|
||||
variables[variable.Name] = variable
|
||||
}
|
||||
if len(variables) != 3 || variables["mode"].Kind != promptkit.RequirementRequired || !variables["mode"].Supplied ||
|
||||
variables["required"].Kind != promptkit.RequirementRequired || variables["required"].Supplied ||
|
||||
variables["inactive_var"].Kind != promptkit.RequirementOptional || variables["inactive_var"].Explanation == "" {
|
||||
t.Fatalf("caller-variable inventory conflated need and presence: %+v", inspection.Variables)
|
||||
}
|
||||
if _, found := variables["unrelated"]; found {
|
||||
t.Fatalf("unreferenced caller key entered the inventory: %+v", inspection.Variables)
|
||||
}
|
||||
formatted := fmt.Sprintf("%+v", inspection)
|
||||
for _, private := range []string{
|
||||
"TEMPLATE_PRIVATE_MARKER", "VARIABLE_PRIVATE_MARKER", "SESSION_PRIVATE_MARKER",
|
||||
} {
|
||||
if strings.Contains(formatted, private) {
|
||||
t.Fatalf("inspection disclosed %q: %s", private, formatted)
|
||||
}
|
||||
}
|
||||
if reader.calls.Load() != 0 || modelCalls.Load() != 0 {
|
||||
t.Fatalf("inspection invoked execution collaborators: artifact reads=%d provider calls=%d", reader.calls.Load(), modelCalls.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestInspectRequirementsSelectionUsesExactVersionsAndProfileIdentity(t *testing.T) {
|
||||
prompts := map[string]any{
|
||||
"multi": map[string]any{
|
||||
"v1": contractPrompt("first", `{{input "one"}}`, nil, "", noValidationOutput),
|
||||
"v2": contractPrompt("second", `{{input "two"}}`, nil, "", noValidationOutput),
|
||||
},
|
||||
"without-default": map[string]any{
|
||||
"v1": contractPrompt("", "plain", nil, "", noValidationOutput),
|
||||
},
|
||||
"direct-required": map[string]any{"v1": contractPrompt("direct", "plain", nil, "", noValidationOutput)},
|
||||
}
|
||||
profiles := map[string]any{
|
||||
"first": map[string]any{"model": "first-model"},
|
||||
"second": map[string]any{"model": "second-model"},
|
||||
"direct": map[string]any{"endpoint": "https://example.test/v1", "model": "direct-model", "api_key_required": true},
|
||||
}
|
||||
engine := requirementsEngine(t, prompts, profiles)
|
||||
if _, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{PromptID: "multi"}); !errors.Is(err, promptkit.ErrPromptLoad) {
|
||||
t.Fatalf("ambiguous omitted version error=%v", err)
|
||||
}
|
||||
first, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{PromptID: "multi", PromptVersion: "v1"})
|
||||
if err != nil || first.PromptVersion != "v1" || first.ProfileID != "first" || len(first.Inputs) != 1 || first.Inputs[0].Name != "one" {
|
||||
t.Fatalf("exact first version/default profile=(%+v, %v)", first, err)
|
||||
}
|
||||
explicit, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{
|
||||
PromptID: "multi", PromptVersion: "v2", ProfileID: "first",
|
||||
})
|
||||
if err != nil || explicit.PromptVersion != "v2" || explicit.ProfileID != "first" || explicit.Inputs[0].Name != "two" {
|
||||
t.Fatalf("exact version/request profile=(%+v, %v)", explicit, err)
|
||||
}
|
||||
if _, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{PromptID: "Multi", PromptVersion: "v1"}); !errors.Is(err, promptkit.ErrPromptNotFound) {
|
||||
t.Fatalf("case-folded prompt ID unexpectedly selected: %v", err)
|
||||
}
|
||||
if _, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{PromptID: "multi", PromptVersion: "V1"}); !errors.Is(err, promptkit.ErrPromptNotFound) {
|
||||
t.Fatalf("case-folded version unexpectedly selected: %v", err)
|
||||
}
|
||||
if _, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{PromptID: "multi", PromptVersion: "v1", ProfileID: "missing"}); !errors.Is(err, promptkit.ErrProfileNotFound) {
|
||||
t.Fatalf("missing profile error=%v", err)
|
||||
}
|
||||
if _, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{PromptID: "without-default"}); !errors.Is(err, promptkit.ErrProfileRequired) || !errors.Is(err, promptkit.ErrInvalidRequest) {
|
||||
t.Fatalf("missing profile selection error=%v", err)
|
||||
}
|
||||
if direct, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{PromptID: "direct-required"}); err != nil || direct.ProfileID != "direct" {
|
||||
t.Fatalf("direct-key-required profile should be inspectable without a key: (%+v, %v)", direct, err)
|
||||
}
|
||||
if _, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{}); !errors.Is(err, promptkit.ErrInvalidRequest) {
|
||||
t.Fatalf("blank prompt error=%v", err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
if _, err := engine.InspectRequirements(ctx, promptkit.RequirementInspectionRequest{PromptID: "multi", PromptVersion: "v1"}); !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("canceled inspection error=%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInspectPromptHashMatchesPreparationForCapturedConfiguration(t *testing.T) {
|
||||
prompts := oneVersionPrompt("hash-equality", "v1", contractPrompt(
|
||||
"profile", "Hello {{.name}}", nil, "", noValidationOutput,
|
||||
))
|
||||
engine := requirementsEngine(t, prompts, map[string]any{
|
||||
"profile": map[string]any{"endpoint": "https://example.test/v1", "model": "inspection-model"},
|
||||
})
|
||||
inspection, err := engine.InspectPrompt(context.Background(), "hash-equality", "v1")
|
||||
if err != nil {
|
||||
t.Fatalf("inspect captured prompt: %v", err)
|
||||
}
|
||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||
PromptID: "hash-equality", PromptVersion: "v1", Vars: map[string]string{"name": "Ada"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("prepare captured prompt: %v", err)
|
||||
}
|
||||
if inspection.PromptHash != prepared.PromptHash {
|
||||
t.Fatalf("inspection hash=%q, preparation hash=%q", inspection.PromptHash, prepared.PromptHash)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInspectRequirementsReportsConditionalAndCompleteOptionalInventories(t *testing.T) {
|
||||
prompts := map[string]any{
|
||||
"conditional": map[string]any{"v1": contractPrompt("profile",
|
||||
`{{if .mode}}{{input "left"}}{{else}}{{input "right"}}{{end}}`, nil, "", noValidationOutput)},
|
||||
"optional": map[string]any{"v1": contractPrompt("profile", `{{index . "lookup"}}`,
|
||||
[]any{map[string]any{"name": "declared-optional"}}, "", noValidationOutput)},
|
||||
"empty": map[string]any{"v1": contractPrompt("profile", "plain", nil, "", noValidationOutput)},
|
||||
}
|
||||
engine := requirementsEngine(t, prompts, map[string]any{"profile": map[string]any{"model": "partial"}})
|
||||
conditional, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{PromptID: "conditional"})
|
||||
if err != nil || conditional.Complete || len(conditional.Issues) != 1 ||
|
||||
conditional.Issues[0].Code != promptkit.RequirementIssueMissingVariable || len(conditional.Inputs) != 2 ||
|
||||
conditional.Inputs[0].Kind != promptkit.RequirementConditional || conditional.Inputs[1].Kind != promptkit.RequirementConditional {
|
||||
t.Fatalf("conditional discovery=(%+v, %v)", conditional, err)
|
||||
}
|
||||
optional, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{PromptID: "optional"})
|
||||
if err != nil || !optional.Complete || len(optional.Issues) != 0 || len(optional.Inputs) != 1 ||
|
||||
optional.Inputs[0].Kind != promptkit.RequirementOptional || len(optional.Variables) != 1 ||
|
||||
optional.Variables[0].Name != "lookup" || optional.Variables[0].Kind != promptkit.RequirementOptional {
|
||||
t.Fatalf("all-optional result should be complete without artifact bindings: (%+v, %v)", optional, err)
|
||||
}
|
||||
empty, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{PromptID: "empty"})
|
||||
if err != nil || !empty.Complete || len(empty.Inputs) != 0 || len(empty.Variables) != 0 {
|
||||
t.Fatalf("empty complete result=(%+v, %v)", empty, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInspectRequirementsBeforePreparationAndWithDirectSessionBypass(t *testing.T) {
|
||||
definition := contractPrompt("profile", `Name={{.name}} Transcript={{input "transcript"}}{{if false}}ignored {{input "skipped"}} {{.inactive}}{{end}}`, []any{
|
||||
map[string]any{"name": "transcript", "required": true},
|
||||
}, `{{.session_var}} {{input "session-only"}}`, noValidationOutput)
|
||||
engine := requirementsEngine(t, oneVersionPrompt("ready", "v1", definition), map[string]any{
|
||||
"profile": map[string]any{"endpoint": "https://example.test/v1", "model": "offline"},
|
||||
})
|
||||
inspection, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{
|
||||
PromptID: "ready", Vars: map[string]string{"name": "Ada"}, SessionID: "direct-session",
|
||||
})
|
||||
if err != nil || inspection == nil || !inspection.Complete || len(inspection.Inputs) != 2 ||
|
||||
inspection.Inputs[0].Name != "skipped" || inspection.Inputs[0].Kind != promptkit.RequirementOptional ||
|
||||
inspection.Inputs[1].Name != "transcript" || inspection.Inputs[1].Kind != promptkit.RequirementRequired ||
|
||||
len(inspection.Variables) != 2 || inspection.Variables[0].Name != "inactive" || inspection.Variables[0].Kind != promptkit.RequirementOptional ||
|
||||
inspection.Variables[1].Name != "name" || inspection.Variables[1].Kind != promptkit.RequirementRequired {
|
||||
t.Fatalf("artifact-free inspection=(%+v, %v)", inspection, err)
|
||||
}
|
||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||
PromptID: "ready", Vars: map[string]string{"name": "Ada"}, SessionID: "direct-session",
|
||||
Inputs: map[string]promptkit.ArtifactRef{"transcript": promptkit.Inline("source transcript")},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("prepare with matching synthetic artifact: %v", err)
|
||||
}
|
||||
if prepared.Messages[0].Content != "Name=Ada Transcript=source transcript" || prepared.SessionID != "direct-session" {
|
||||
t.Fatalf("prepared run did not use the inspected dependencies: %+v", prepared)
|
||||
}
|
||||
if _, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{
|
||||
PromptID: "ready", Vars: map[string]string{"name": "Ada"},
|
||||
}); err != nil {
|
||||
t.Fatalf("inspect without session bypass: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInspectRequirementsUsesCapturedFileBackedTemplates(t *testing.T) {
|
||||
files := fstest.MapFS{
|
||||
"config/prompt.yaml": &fstest.MapFile{Data: []byte(capturedPromptDocument)},
|
||||
"config/body.tmpl": &fstest.MapFile{Data: []byte(`{{input "captured-input"}}`)},
|
||||
}
|
||||
source := &unreadableAfterConstructionFS{FS: files}
|
||||
engine := capturedPromptEngine(t, source)
|
||||
source.closed.Store(true)
|
||||
delete(files, "config/body.tmpl")
|
||||
inspection, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{PromptID: "captured"})
|
||||
if err != nil || inspection == nil || !inspection.Complete || len(inspection.Inputs) != 1 ||
|
||||
inspection.Inputs[0].Name != "captured-input" || source.reads.Load() != 0 {
|
||||
t.Fatalf("captured template inspection=(%+v, %v), post-construction reads=%d", inspection, err, source.reads.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestInspectRequirementsReturnsIndependentValuesAndCanOverlapPreparation(t *testing.T) {
|
||||
definition := contractPrompt("profile", `{{if .decision}}{{input "conditional"}}{{end}} {{input "declared"}}`, []any{
|
||||
map[string]any{"name": "declared", "required": true, "description": "captured description"},
|
||||
}, `{{.session}}`, noValidationOutput)
|
||||
engine := requirementsEngine(t, oneVersionPrompt("owned", "v1", definition), map[string]any{
|
||||
"profile": map[string]any{"endpoint": "https://example.test/v1", "model": "offline"},
|
||||
})
|
||||
request := promptkit.RequirementInspectionRequest{PromptID: "owned"}
|
||||
first, err := engine.InspectRequirements(context.Background(), request)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for index := range first.Inputs {
|
||||
if first.Inputs[index].Name == "declared" {
|
||||
first.Inputs[index].Name = "mutated"
|
||||
if first.Inputs[index].Description != nil {
|
||||
*first.Inputs[index].Description = "mutated description"
|
||||
}
|
||||
}
|
||||
}
|
||||
first.Variables[0].Name = "mutated-variable"
|
||||
first.Issues[0].Description = "mutated issue"
|
||||
second, err := engine.InspectRequirements(context.Background(), request)
|
||||
if err != nil {
|
||||
t.Fatalf("inspect after mutating returned values: %v", err)
|
||||
}
|
||||
var description *string
|
||||
for _, input := range second.Inputs {
|
||||
if input.Name == "declared" {
|
||||
description = input.Description
|
||||
}
|
||||
}
|
||||
if second.Inputs[0].Name == "mutated" || second.Variables[0].Name != "decision" ||
|
||||
description == nil || *description != "captured description" ||
|
||||
len(second.Issues) != 1 || second.Issues[0].Description == "mutated issue" {
|
||||
t.Fatalf("result mutation reached captured state or later result: (%+v, %v)", second, err)
|
||||
}
|
||||
|
||||
const count = 12
|
||||
var wait sync.WaitGroup
|
||||
errs := make(chan error, count*2)
|
||||
for index := 0; index < count; index++ {
|
||||
wait.Add(2)
|
||||
go func() {
|
||||
defer wait.Done()
|
||||
_, err := engine.InspectRequirements(context.Background(), request)
|
||||
errs <- err
|
||||
}()
|
||||
go func(index int) {
|
||||
defer wait.Done()
|
||||
_, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||
PromptID: "owned", SessionID: fmt.Sprintf("session-%d", index), Vars: map[string]string{"decision": "yes"},
|
||||
Inputs: map[string]promptkit.ArtifactRef{
|
||||
"conditional": promptkit.Inline("condition"), "declared": promptkit.Inline("body"),
|
||||
},
|
||||
})
|
||||
errs <- err
|
||||
}(index)
|
||||
}
|
||||
wait.Wait()
|
||||
close(errs)
|
||||
for err := range errs {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestInspectRequirementsDoesNotEchoUnrelatedPrivateValues(t *testing.T) {
|
||||
t.Setenv("PROMPTKIT_REQUIREMENTS_SECRET", "credential-value-marker")
|
||||
definition := contractPrompt("profile", `template-body-marker {{input "required-input"}} {{.known}}`, nil, `{{.session}}`, noValidationOutput)
|
||||
engine := requirementsEngine(t, oneVersionPrompt("privacy", "v1", definition), map[string]any{
|
||||
"profile": map[string]any{"endpoint": "https://example.test/v1", "model": "offline", "api_key_env": "PROMPTKIT_REQUIREMENTS_SECRET"},
|
||||
})
|
||||
inspection, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{
|
||||
PromptID: "privacy", Vars: map[string]string{"known": "known-value-marker", "unrelated": "unrelated-value-marker"},
|
||||
SessionID: "direct-session-marker",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
formatted := fmt.Sprintf("%+v", inspection)
|
||||
for _, marker := range []string{
|
||||
"template-body-marker", "known-value-marker", "unrelated-value-marker",
|
||||
"direct-session-marker", "credential-value-marker", "PROMPTKIT_REQUIREMENTS_SECRET",
|
||||
} {
|
||||
if strings.Contains(formatted, marker) {
|
||||
t.Fatalf("inspection disclosed %q: %s", marker, formatted)
|
||||
}
|
||||
}
|
||||
}
|
||||
15
requirements_test.go
Normal file
15
requirements_test.go
Normal file
@@ -0,0 +1,15 @@
|
||||
package promptkit
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestInspectRequirementsRejectsNilEngine(t *testing.T) {
|
||||
var engine *Engine
|
||||
inspection, err := engine.InspectRequirements(context.Background(), RequirementInspectionRequest{PromptID: "prompt"})
|
||||
if inspection != nil || !errors.Is(err, ErrInvalidConfig) {
|
||||
t.Fatalf("nil engine returned (%+v, %v), want ErrInvalidConfig", inspection, err)
|
||||
}
|
||||
}
|
||||
161
scripts/check-examples.py
Normal file
161
scripts/check-examples.py
Normal file
@@ -0,0 +1,161 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run the maintained consumer examples and check their semantic output."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
REPOSITORY_ROOT = Path(__file__).resolve().parent.parent
|
||||
|
||||
|
||||
def example_environment():
|
||||
environment = os.environ.copy()
|
||||
environment["GOTOOLCHAIN"] = "local"
|
||||
try:
|
||||
host = subprocess.run(
|
||||
["go", "env", "GOHOSTOS", "GOHOSTARCH"],
|
||||
cwd=REPOSITORY_ROOT,
|
||||
env=environment,
|
||||
text=True,
|
||||
capture_output=True,
|
||||
check=False,
|
||||
)
|
||||
except OSError as error:
|
||||
raise RuntimeError(f"could not run go env: {error}") from error
|
||||
|
||||
if host.returncode != 0:
|
||||
raise RuntimeError(
|
||||
"go env GOHOSTOS GOHOSTARCH failed "
|
||||
f"with exit status {host.returncode}: {host.stderr.strip()}"
|
||||
)
|
||||
|
||||
values = host.stdout.splitlines()
|
||||
if len(values) != 2 or not all(values):
|
||||
raise RuntimeError(
|
||||
"go env GOHOSTOS GOHOSTARCH returned unexpected output: "
|
||||
f"{host.stdout!r}"
|
||||
)
|
||||
|
||||
environment["GOOS"], environment["GOARCH"] = values
|
||||
return environment
|
||||
|
||||
|
||||
def run_example(name, environment, failures):
|
||||
command = ["go", "run", f"./examples/go-library/{name}"]
|
||||
try:
|
||||
result = subprocess.run(
|
||||
command,
|
||||
cwd=REPOSITORY_ROOT,
|
||||
env=environment,
|
||||
text=True,
|
||||
capture_output=True,
|
||||
check=False,
|
||||
)
|
||||
except OSError as error:
|
||||
failures.append(f"{name}: could not start {command[0]}: {error}")
|
||||
return None
|
||||
|
||||
if result.returncode != 0:
|
||||
failures.append(f"{name}: command exited with status {result.returncode}")
|
||||
if result.stderr:
|
||||
failures.append(f"{name}: stderr:\n{result.stderr.rstrip()}")
|
||||
if result.stdout:
|
||||
failures.append(f"{name}: stdout:\n{result.stdout.rstrip()}")
|
||||
return None
|
||||
|
||||
try:
|
||||
value = json.loads(result.stdout)
|
||||
except json.JSONDecodeError as error:
|
||||
failures.append(f"{name}: output is not valid JSON: {error}")
|
||||
if result.stdout:
|
||||
failures.append(f"{name}: stdout:\n{result.stdout.rstrip()}")
|
||||
return None
|
||||
|
||||
if not isinstance(value, dict):
|
||||
failures.append(
|
||||
f"{name}: expected a JSON object, got {type(value).__name__}"
|
||||
)
|
||||
return None
|
||||
|
||||
return value
|
||||
|
||||
|
||||
def expect_equal(example_name, value, key, expected, failures):
|
||||
actual = value.get(key)
|
||||
matches = actual is expected if isinstance(expected, bool) else actual == expected
|
||||
if not matches:
|
||||
failures.append(
|
||||
f"{example_name}: expected {key}={expected!r}, got {actual!r}"
|
||||
)
|
||||
|
||||
|
||||
def expect_positive(example_name, value, key, failures):
|
||||
actual = value.get(key)
|
||||
try:
|
||||
matches = actual > 0
|
||||
except TypeError:
|
||||
matches = False
|
||||
if not matches:
|
||||
failures.append(f"{example_name}: expected {key} to be positive, got {actual!r}")
|
||||
|
||||
|
||||
def main():
|
||||
try:
|
||||
environment = example_environment()
|
||||
except RuntimeError as error:
|
||||
print(error, file=sys.stderr)
|
||||
return 1
|
||||
|
||||
failures = []
|
||||
prepared = run_example("prepare", environment, failures)
|
||||
result = run_example("run", environment, failures)
|
||||
observed = run_example("observe", environment, failures)
|
||||
|
||||
if prepared is not None:
|
||||
expect_equal("prepare", prepared, "prompt_id", "example.prepare", failures)
|
||||
expect_equal(
|
||||
"prepare", prepared, "selected_profile", "offline-example", failures
|
||||
)
|
||||
expect_equal("prepare", prepared, "model", "offline-model", failures)
|
||||
expect_positive("prepare", prepared, "message_count", failures)
|
||||
|
||||
if result is not None:
|
||||
expect_equal(
|
||||
"run", result, "output", "Ada finished the migration review.", failures
|
||||
)
|
||||
expect_equal("run", result, "is_valid", True, failures)
|
||||
expect_equal("run", result, "validation_status", "passed", failures)
|
||||
expect_equal("run", result, "model", "offline-model", failures)
|
||||
expect_equal("run", result, "total_tokens", 18, failures)
|
||||
|
||||
if observed is not None:
|
||||
expect_equal("observe", observed, "proposal_attempts", 2, failures)
|
||||
expect_equal(
|
||||
"observe", observed, "proposal_transient_retries", 1, failures
|
||||
)
|
||||
expect_equal(
|
||||
"observe", observed, "proposal_usage_complete", False, failures
|
||||
)
|
||||
expect_equal("observe", observed, "assessment_attempts", 1, failures)
|
||||
expect_equal(
|
||||
"observe", observed, "assessment_transient_retries", 0, failures
|
||||
)
|
||||
expect_equal(
|
||||
"observe", observed, "assessment_outcome", "accepted", failures
|
||||
)
|
||||
|
||||
if failures:
|
||||
print("Consumer example checks failed:", file=sys.stderr)
|
||||
for failure in failures:
|
||||
print(f"- {failure}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
print("Consumer example checks passed: prepare, run, observe")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,152 +0,0 @@
|
||||
#!/bin/sh
|
||||
# Run local checks without publishing or changing files.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
go test -race -vet=off ./...
|
||||
go vet ./...
|
||||
python3 - <<'PY'
|
||||
import json
|
||||
import subprocess
|
||||
|
||||
def example(name):
|
||||
return json.loads(subprocess.check_output(
|
||||
["go", "run", "./examples/go-library/" + name], text=True))
|
||||
|
||||
prepared = example("prepare")
|
||||
assert prepared["prompt_id"] == "example.prepare", prepared
|
||||
assert prepared["selected_profile"] == "offline-example", prepared
|
||||
assert prepared["model"] == "offline-model", prepared
|
||||
assert prepared["message_count"] > 0, prepared
|
||||
result = example("run")
|
||||
assert result["output"] == "Ada finished the migration review.", result
|
||||
assert result["is_valid"] is True, result
|
||||
assert result["validation_status"] == "passed", result
|
||||
assert result["model"] == "offline-model", result
|
||||
assert result["total_tokens"] == 18, result
|
||||
observed = example("observe")
|
||||
assert observed["proposal_attempts"] == 2, observed
|
||||
assert observed["proposal_transient_retries"] == 1, observed
|
||||
assert observed["proposal_usage_complete"] is False, observed
|
||||
assert observed["assessment_attempts"] == 1, observed
|
||||
assert observed["assessment_transient_retries"] == 0, observed
|
||||
assert observed["assessment_outcome"] == "accepted", observed
|
||||
PY
|
||||
|
||||
unformatted=$(
|
||||
git ls-files '*.go' |
|
||||
while IFS= read -r go_file
|
||||
do
|
||||
gofmt -l "$go_file"
|
||||
done
|
||||
)
|
||||
test -z "$unformatted"
|
||||
|
||||
python3 - <<'PY'
|
||||
from pathlib import Path
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from urllib.parse import unquote
|
||||
|
||||
root = Path.cwd().resolve()
|
||||
markdown_files = [
|
||||
root / name
|
||||
for name in subprocess.check_output(
|
||||
["git", "ls-files", "*.md"], text=True
|
||||
).splitlines()
|
||||
]
|
||||
link_pattern = re.compile(r"!?\[[^]]*\]\(([^)]+)\)")
|
||||
heading_pattern = re.compile(r"^#{1,6}\s+(.+?)\s*#*\s*$")
|
||||
scheme_pattern = re.compile(r"^[a-z][a-z0-9+.-]*:", re.IGNORECASE)
|
||||
|
||||
|
||||
def markdown_lines(path):
|
||||
in_fence = False
|
||||
fence = ""
|
||||
for line in path.read_text(encoding="utf-8").splitlines():
|
||||
stripped = line.lstrip()
|
||||
marker = stripped[:3]
|
||||
if marker in {"```", "~~~"}:
|
||||
if not in_fence:
|
||||
in_fence = True
|
||||
fence = marker
|
||||
elif marker == fence:
|
||||
in_fence = False
|
||||
fence = ""
|
||||
continue
|
||||
if not in_fence:
|
||||
yield line
|
||||
|
||||
|
||||
anchor_cache = {}
|
||||
|
||||
|
||||
def anchors(path):
|
||||
if path in anchor_cache:
|
||||
return anchor_cache[path]
|
||||
found = set()
|
||||
counts = {}
|
||||
for line in markdown_lines(path):
|
||||
match = heading_pattern.match(line)
|
||||
if not match:
|
||||
continue
|
||||
heading = re.sub(r"<[^>]+>", "", match.group(1)).replace("`", "")
|
||||
base = re.sub(r"[^\w\- ]", "", heading.lower()).replace(" ", "-")
|
||||
count = counts.get(base, 0)
|
||||
counts[base] = count + 1
|
||||
found.add(base if count == 0 else f"{base}-{count}")
|
||||
anchor_cache[path] = found
|
||||
return found
|
||||
|
||||
|
||||
failures = []
|
||||
for source in markdown_files:
|
||||
text = "\n".join(markdown_lines(source))
|
||||
for match in link_pattern.finditer(text):
|
||||
target = match.group(1).strip()
|
||||
if target.startswith("<") and target.endswith(">"):
|
||||
target = target[1:-1]
|
||||
if scheme_pattern.match(target) or target.startswith("//"):
|
||||
continue
|
||||
path_text, separator, fragment = target.partition("#")
|
||||
destination = source if not path_text else source.parent / unquote(path_text)
|
||||
try:
|
||||
destination = destination.resolve()
|
||||
destination.relative_to(root)
|
||||
except ValueError:
|
||||
failures.append(f"{source.relative_to(root)}: escapes repository: {target}")
|
||||
continue
|
||||
if not destination.exists():
|
||||
failures.append(f"{source.relative_to(root)}: missing target: {target}")
|
||||
continue
|
||||
if separator and destination.suffix.lower() == ".md":
|
||||
fragment = unquote(fragment).lower()
|
||||
if fragment not in anchors(destination):
|
||||
failures.append(f"{source.relative_to(root)}: missing anchor: {target}")
|
||||
|
||||
if failures:
|
||||
print("\n".join(failures), file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
PY
|
||||
|
||||
case "$(go env GOWORK)" in
|
||||
''|off) ;;
|
||||
*) printf '%s\n' 'an active Go workspace is not allowed' >&2; exit 1 ;;
|
||||
esac
|
||||
test -z "$(git ls-files go.work go.work.sum)"
|
||||
test ! -e vendor
|
||||
if grep -Eq '^[[:space:]]*replace([[:space:]]|\()' go.mod
|
||||
then
|
||||
printf '%s\n' 'go.mod contains a replacement' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
git diff --check
|
||||
git diff --cached --check
|
||||
credential_pattern='-----BEGIN ([A-Z0-9]+ )?PRIV''ATE KEY-----|AKI''A[0-9A-Z]{16}|gh[pousr]_[A-Za-z0-9]{36,}|sk-[A-Za-z0-9]{32,}'
|
||||
if git grep -nEI -e "$credential_pattern" -- .
|
||||
then
|
||||
printf '%s\n' 'possible credential found' >&2
|
||||
exit 1
|
||||
fi
|
||||
@@ -1,57 +1,5 @@
|
||||
#!/bin/sh
|
||||
# Validate one committed candidate and publish only its selected annotated tag.
|
||||
set -eu
|
||||
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
release_version=${1:?usage: scripts/release.sh vMAJOR.MINOR.PATCH}
|
||||
printf '%s\n' "$release_version" |
|
||||
grep -Eq '^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$'
|
||||
release_commit=$(git rev-parse --verify 'HEAD^{commit}')
|
||||
|
||||
check_candidate() {
|
||||
test "$(git branch --show-current)" = main
|
||||
test -z "$(git status --porcelain)"
|
||||
test "$(git rev-parse HEAD)" = "$release_commit"
|
||||
git fetch origin main
|
||||
test "$(git rev-parse refs/remotes/origin/main)" = "$release_commit"
|
||||
if git show-ref --verify --quiet "refs/tags/$release_version"; then
|
||||
printf '%s\n' "local tag already exists: $release_version" >&2
|
||||
exit 1
|
||||
fi
|
||||
remote_refs=$(git ls-remote --tags origin "refs/tags/$release_version" "refs/tags/$release_version^{}")
|
||||
test -z "$remote_refs"
|
||||
}
|
||||
|
||||
check_candidate
|
||||
python3 - "$release_version" <<'PY'
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
version = sys.argv[1]
|
||||
note = Path("docs/releases") / (version + ".md")
|
||||
lines = note.read_text(encoding="utf-8").splitlines()
|
||||
assert lines and lines[0] == "# Promptkit " + version, f"wrong release title: {note}"
|
||||
headings = set()
|
||||
fence = None
|
||||
for line in lines:
|
||||
marker = line.lstrip()[:3]
|
||||
if marker in {"```", "~~~"}:
|
||||
if fence is None:
|
||||
fence = marker
|
||||
elif fence == marker:
|
||||
fence = None
|
||||
elif fence is None:
|
||||
headings.add(line)
|
||||
for section in ("Summary", "Compatibility", "Upgrade", "Changes"):
|
||||
assert "## " + section in headings, f"missing {section}: {note}"
|
||||
PY
|
||||
|
||||
./scripts/release-check.sh
|
||||
check_candidate
|
||||
release_note=$(git show "$release_commit:docs/releases/$release_version.md")
|
||||
printf '%s\n\nValidated commit: %s\n' "$release_note" "$release_commit" |
|
||||
git tag --annotate --cleanup=verbatim "$release_version" --file - "$release_commit"
|
||||
test "$(git cat-file -t "refs/tags/$release_version")" = tag
|
||||
test "$(git rev-parse "refs/tags/$release_version^{commit}")" = "$release_commit"
|
||||
git push origin "refs/tags/$release_version:refs/tags/$release_version"
|
||||
printf '%s\n' "Released $release_version. No CI or module-availability wait is required."
|
||||
exec ./scripts/releasekit.sh publish "$@"
|
||||
|
||||
18
scripts/releasekit.sh
Executable file
18
scripts/releasekit.sh
Executable file
@@ -0,0 +1,18 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
export GOTOOLCHAIN=local
|
||||
export GOWORK=off
|
||||
export GOFLAGS=-mod=readonly
|
||||
|
||||
GOOS=$(go env GOHOSTOS)
|
||||
GOARCH=$(go env GOHOSTARCH)
|
||||
export GOOS GOARCH
|
||||
|
||||
mkdir -p .releasekit/bin
|
||||
go build -mod=readonly -o .releasekit/bin/releasekit \
|
||||
"gitea.maximumdirect.net/eric/ReleaseKit/cmd/releasekit"
|
||||
|
||||
exec ./.releasekit/bin/releasekit "$@"
|
||||
476
scripts/test-release.py
Normal file
476
scripts/test-release.py
Normal file
@@ -0,0 +1,476 @@
|
||||
"""Exercise ReleaseKit publication wiring in disposable repositories."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import textwrap
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
REPOSITORY_ROOT = Path(__file__).resolve().parent.parent
|
||||
RELEASE_VERSION = "v0.0.1"
|
||||
FAILED_RELEASE_VERSION = "v0.0.2"
|
||||
RELEASEKIT_MODULE = "gitea.maximumdirect.net/eric/ReleaseKit"
|
||||
|
||||
|
||||
def execute(command, cwd, environment):
|
||||
try:
|
||||
return subprocess.run(
|
||||
command,
|
||||
cwd=cwd,
|
||||
env=environment,
|
||||
text=True,
|
||||
capture_output=True,
|
||||
check=False,
|
||||
)
|
||||
except OSError as error:
|
||||
raise RuntimeError(f"could not run {command[0]}: {error}") from error
|
||||
|
||||
|
||||
def run(command, cwd, environment):
|
||||
result = execute(command, cwd, environment)
|
||||
if result.returncode != 0:
|
||||
details = [
|
||||
f"command failed with status {result.returncode}: {' '.join(command)}"
|
||||
]
|
||||
if result.stdout:
|
||||
details.append(f"stdout:\n{result.stdout.rstrip()}")
|
||||
if result.stderr:
|
||||
details.append(f"stderr:\n{result.stderr.rstrip()}")
|
||||
raise RuntimeError("\n".join(details))
|
||||
return result.stdout
|
||||
|
||||
|
||||
def write_file(path, contents):
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(textwrap.dedent(contents).lstrip("\n"), encoding="utf-8")
|
||||
|
||||
|
||||
def repo_git(repository, environment, *arguments):
|
||||
return run(
|
||||
["git", "-C", str(repository), *arguments],
|
||||
cwd=repository.parent,
|
||||
environment=environment,
|
||||
)
|
||||
|
||||
|
||||
def remote_git(remote, temporary_root, environment, *arguments):
|
||||
return run(
|
||||
["git", "--git-dir", str(remote), *arguments],
|
||||
cwd=temporary_root,
|
||||
environment=environment,
|
||||
)
|
||||
|
||||
|
||||
def fixture_sources(repository, environment):
|
||||
write_file(
|
||||
repository / "fixture.go",
|
||||
"""
|
||||
package fixture
|
||||
|
||||
func candidateValue() string { return "ready" }
|
||||
""",
|
||||
)
|
||||
write_file(
|
||||
repository / "fixture_test.go",
|
||||
f"""
|
||||
package fixture
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"runtime"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestCandidateSuite(t *testing.T) {{
|
||||
if candidateValue() != "ready" {{
|
||||
t.Fatal("candidate package is not ready")
|
||||
}}
|
||||
marker := os.Getenv("FIXTURE_SUITE_MARKER")
|
||||
if marker == "" {{
|
||||
t.Fatal("FIXTURE_SUITE_MARKER is not set")
|
||||
}}
|
||||
record := struct {{
|
||||
GoVersion string `json:"go_version"`
|
||||
Race bool `json:"race"`
|
||||
}}{{GoVersion: runtime.Version(), Race: raceDetectorEnabled}}
|
||||
data, err := json.Marshal(record)
|
||||
if err != nil {{
|
||||
t.Fatal(err)
|
||||
}}
|
||||
file, err := os.OpenFile(marker, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600)
|
||||
if err != nil {{
|
||||
t.Fatal(err)
|
||||
}}
|
||||
if _, err := file.Write(append(data, '\\n')); err != nil {{
|
||||
file.Close()
|
||||
t.Fatal(err)
|
||||
}}
|
||||
if err := file.Close(); err != nil {{
|
||||
t.Fatal(err)
|
||||
}}
|
||||
}}
|
||||
""",
|
||||
)
|
||||
write_file(
|
||||
repository / "race_enabled_test.go",
|
||||
"""
|
||||
//go:build race
|
||||
|
||||
package fixture
|
||||
|
||||
const raceDetectorEnabled = true
|
||||
""",
|
||||
)
|
||||
write_file(
|
||||
repository / "race_disabled_test.go",
|
||||
"""
|
||||
//go:build !race
|
||||
|
||||
package fixture
|
||||
|
||||
const raceDetectorEnabled = false
|
||||
""",
|
||||
)
|
||||
go_files = [
|
||||
str(repository / filename)
|
||||
for filename in (
|
||||
"fixture.go",
|
||||
"fixture_test.go",
|
||||
"race_enabled_test.go",
|
||||
"race_disabled_test.go",
|
||||
)
|
||||
]
|
||||
run(["gofmt", "-w", *go_files], cwd=repository, environment=environment)
|
||||
|
||||
|
||||
def prepare_repository(repository, remote, environment, module_bytes, version):
|
||||
run(
|
||||
["git", "init", "--bare", "--initial-branch=main", str(remote)],
|
||||
cwd=repository.parent,
|
||||
environment=environment,
|
||||
)
|
||||
run(
|
||||
["git", "init", "--initial-branch=main", str(repository)],
|
||||
cwd=repository.parent,
|
||||
environment=environment,
|
||||
)
|
||||
repo_git(repository, environment, "config", "user.name", "ReleaseKit Fixture")
|
||||
repo_git(repository, environment, "config", "user.email", "fixture@example.invalid")
|
||||
repo_git(repository, environment, "remote", "add", "origin", str(remote))
|
||||
|
||||
for name, contents in module_bytes.items():
|
||||
(repository / name).write_bytes(contents)
|
||||
|
||||
write_file(
|
||||
repository / ".gitignore",
|
||||
"""
|
||||
.releasekit/
|
||||
""",
|
||||
)
|
||||
write_file(
|
||||
repository / "releasekit.json",
|
||||
(REPOSITORY_ROOT / "releasekit.json").read_text(encoding="utf-8"),
|
||||
)
|
||||
|
||||
scripts = repository / "scripts"
|
||||
scripts.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(REPOSITORY_ROOT / "scripts/release.sh", scripts / "release.sh")
|
||||
shutil.copy2(REPOSITORY_ROOT / "scripts/releasekit.sh", scripts / "releasekit.sh")
|
||||
write_file(
|
||||
scripts / "check-examples.py",
|
||||
"""
|
||||
import os
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
marker = Path(os.environ["FIXTURE_EXAMPLE_MARKER"])
|
||||
with marker.open("a", encoding="utf-8") as stream:
|
||||
stream.write("called\\n")
|
||||
if os.environ.get("FIXTURE_EXAMPLE_FAILURE") == "1":
|
||||
print("fixture example check rejected candidate", file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
""",
|
||||
)
|
||||
write_file(
|
||||
repository / f"docs/releases/{version}.md",
|
||||
f"""
|
||||
# Promptkit {version}
|
||||
|
||||
Disposable publication fixture.
|
||||
""",
|
||||
)
|
||||
fixture_sources(repository, environment)
|
||||
|
||||
repo_git(repository, environment, "add", "-A")
|
||||
run(
|
||||
["git", "-C", str(repository), "commit", "-m", "Add disposable release fixture"],
|
||||
cwd=repository.parent,
|
||||
environment=environment,
|
||||
)
|
||||
repo_git(repository, environment, "push", "origin", "main")
|
||||
|
||||
|
||||
def verify_publication(repository, remote, temporary_root, environment, module_bytes, selected_go):
|
||||
selected_releasekit = run(
|
||||
["go", "list", "-mod=readonly", "-m", "-f", "{{.Version}}", RELEASEKIT_MODULE],
|
||||
cwd=repository,
|
||||
environment=environment,
|
||||
).strip()
|
||||
publication = repository / "scripts/release.sh"
|
||||
run(
|
||||
[str(publication), RELEASE_VERSION],
|
||||
cwd=temporary_root,
|
||||
environment=environment,
|
||||
)
|
||||
|
||||
suite_lines = Path(environment["FIXTURE_SUITE_MARKER"]).read_text(
|
||||
encoding="utf-8"
|
||||
).splitlines()
|
||||
if len(suite_lines) != 1:
|
||||
raise RuntimeError(f"expected one suite marker, found {len(suite_lines)}")
|
||||
suite_record = json.loads(suite_lines[0])
|
||||
if suite_record != {"go_version": selected_go, "race": True}:
|
||||
raise RuntimeError(f"unexpected suite marker: {suite_record!r}")
|
||||
|
||||
example_lines = Path(environment["FIXTURE_EXAMPLE_MARKER"]).read_text(
|
||||
encoding="utf-8"
|
||||
).splitlines()
|
||||
if example_lines != ["called"]:
|
||||
raise RuntimeError(f"expected one additional-check marker, got {example_lines!r}")
|
||||
|
||||
binary = repository / ".releasekit/bin/releasekit"
|
||||
if not binary.is_file():
|
||||
raise RuntimeError("ReleaseKit bootstrap did not leave its ignored executable")
|
||||
build_info = run(
|
||||
["go", "version", "-m", str(binary)],
|
||||
cwd=temporary_root,
|
||||
environment=environment,
|
||||
)
|
||||
first_line = build_info.splitlines()[0]
|
||||
if selected_go not in first_line:
|
||||
raise RuntimeError(f"tool was built with {first_line!r}, expected {selected_go}")
|
||||
expected_module = f"\tmod\t{RELEASEKIT_MODULE}\t{selected_releasekit}\t"
|
||||
if expected_module not in build_info:
|
||||
raise RuntimeError(
|
||||
f"bootstrap executable does not record selected ReleaseKit {selected_releasekit}"
|
||||
)
|
||||
|
||||
candidate = repo_git(repository, environment, "rev-parse", "HEAD").strip()
|
||||
if repo_git(repository, environment, "rev-parse", "main").strip() != candidate:
|
||||
raise RuntimeError("fixture main moved away from the published candidate")
|
||||
if remote_git(remote, temporary_root, environment, "rev-parse", "refs/heads/main").strip() != candidate:
|
||||
raise RuntimeError("local origin main does not identify the fixture candidate")
|
||||
|
||||
tag_ref = f"refs/tags/{RELEASE_VERSION}"
|
||||
if repo_git(repository, environment, "cat-file", "-t", tag_ref).strip() != "tag":
|
||||
raise RuntimeError("local release ref is not an annotated tag")
|
||||
local_tag = repo_git(repository, environment, "rev-parse", tag_ref).strip()
|
||||
local_tag_commit = repo_git(
|
||||
repository, environment, "rev-parse", f"{tag_ref}^{{}}"
|
||||
).strip()
|
||||
if local_tag_commit != candidate:
|
||||
raise RuntimeError("local annotated tag does not identify the fixture candidate")
|
||||
|
||||
remote_tag = remote_git(remote, temporary_root, environment, "rev-parse", tag_ref).strip()
|
||||
remote_tag_commit = remote_git(
|
||||
remote, temporary_root, environment, "rev-parse", f"{tag_ref}^{{}}"
|
||||
).strip()
|
||||
if remote_tag != local_tag or remote_tag_commit != candidate:
|
||||
raise RuntimeError("remote annotated tag does not identify the fixture candidate")
|
||||
if remote_git(
|
||||
remote, temporary_root, environment, "cat-file", "-t", tag_ref
|
||||
).strip() != "tag":
|
||||
raise RuntimeError("remote release ref is not an annotated tag")
|
||||
|
||||
tag_object = repo_git(repository, environment, "cat-file", "tag", tag_ref)
|
||||
separator = tag_object.find("\n\n")
|
||||
if separator < 0:
|
||||
raise RuntimeError("annotated tag has no message")
|
||||
tag_message = tag_object[separator + 2 :]
|
||||
note = (repository / f"docs/releases/{RELEASE_VERSION}.md").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
if tag_message != note:
|
||||
raise RuntimeError("annotated tag message does not match the committed release note")
|
||||
|
||||
for name, contents in module_bytes.items():
|
||||
if (repository / name).read_bytes() != contents:
|
||||
raise RuntimeError(f"publication changed {name}")
|
||||
|
||||
status = repo_git(
|
||||
repository, environment, "status", "--porcelain", "--untracked-files=all"
|
||||
)
|
||||
if status:
|
||||
raise RuntimeError(f"fixture checkout is not clean after publication:\n{status}")
|
||||
repo_git(repository, environment, "check-ignore", "-q", ".releasekit/bin/releasekit")
|
||||
|
||||
|
||||
def verify_validation_failure(
|
||||
repository, remote, temporary_root, environment, module_bytes, selected_go
|
||||
):
|
||||
scenario_environment = environment.copy()
|
||||
suite_marker = temporary_root / "failure-suite.jsonl"
|
||||
example_marker = temporary_root / "failure-example.log"
|
||||
scenario_environment.update(
|
||||
{
|
||||
"FIXTURE_SUITE_MARKER": str(suite_marker),
|
||||
"FIXTURE_EXAMPLE_MARKER": str(example_marker),
|
||||
"FIXTURE_EXAMPLE_FAILURE": "1",
|
||||
}
|
||||
)
|
||||
prepare_repository(
|
||||
repository,
|
||||
remote,
|
||||
scenario_environment,
|
||||
module_bytes,
|
||||
FAILED_RELEASE_VERSION,
|
||||
)
|
||||
|
||||
result = execute(
|
||||
[str(repository / "scripts/release.sh"), FAILED_RELEASE_VERSION],
|
||||
cwd=temporary_root,
|
||||
environment=scenario_environment,
|
||||
)
|
||||
if result.returncode != 1:
|
||||
raise RuntimeError(
|
||||
"validation failure returned status "
|
||||
f"{result.returncode}, want 1; stdout:\n{result.stdout}\nstderr:\n{result.stderr}"
|
||||
)
|
||||
if "fixture example check rejected candidate" not in result.stderr:
|
||||
raise RuntimeError(f"failure diagnostic omitted useful context:\n{result.stderr}")
|
||||
|
||||
suite_lines = suite_marker.read_text(encoding="utf-8").splitlines()
|
||||
if len(suite_lines) != 1:
|
||||
raise RuntimeError(f"expected one failure-case suite marker, found {len(suite_lines)}")
|
||||
suite_record = json.loads(suite_lines[0])
|
||||
if suite_record != {"go_version": selected_go, "race": True}:
|
||||
raise RuntimeError(f"unexpected failure-case suite marker: {suite_record!r}")
|
||||
if example_marker.read_text(encoding="utf-8").splitlines() != ["called"]:
|
||||
raise RuntimeError("expected one additional-check invocation before failure")
|
||||
|
||||
tag_ref = f"refs/tags/{FAILED_RELEASE_VERSION}"
|
||||
for description, command in (
|
||||
(
|
||||
"local",
|
||||
["git", "-C", str(repository), "show-ref", "--verify", "--quiet", tag_ref],
|
||||
),
|
||||
(
|
||||
"remote",
|
||||
["git", "--git-dir", str(remote), "show-ref", "--verify", "--quiet", tag_ref],
|
||||
),
|
||||
):
|
||||
result = execute(command, cwd=temporary_root, environment=scenario_environment)
|
||||
if result.returncode == 0:
|
||||
raise RuntimeError(f"validation failure created the {description} tag {tag_ref}")
|
||||
if result.returncode != 1:
|
||||
raise RuntimeError(
|
||||
f"could not verify {description} tag absence: "
|
||||
f"status {result.returncode}: {result.stderr.strip()}"
|
||||
)
|
||||
|
||||
for name, contents in module_bytes.items():
|
||||
if (repository / name).read_bytes() != contents:
|
||||
raise RuntimeError(f"failed publication changed {name}")
|
||||
status = repo_git(
|
||||
repository,
|
||||
scenario_environment,
|
||||
"status",
|
||||
"--porcelain",
|
||||
"--untracked-files=all",
|
||||
)
|
||||
if status:
|
||||
raise RuntimeError(f"failure fixture checkout is not clean:\n{status}")
|
||||
|
||||
|
||||
def verify_usage_classification(binary, temporary_root, environment):
|
||||
result = execute(
|
||||
[str(binary), "unknown"], cwd=temporary_root, environment=environment
|
||||
)
|
||||
if result.returncode != 2:
|
||||
raise RuntimeError(
|
||||
f"unsupported command returned status {result.returncode}, want 2; "
|
||||
f"stdout:\n{result.stdout}\nstderr:\n{result.stderr}"
|
||||
)
|
||||
if result.stdout:
|
||||
raise RuntimeError(f"unsupported command wrote to stdout: {result.stdout!r}")
|
||||
if 'unknown command "unknown"' not in result.stderr or "usage: releasekit" not in result.stderr:
|
||||
raise RuntimeError(f"unsupported command diagnostic was not useful:\n{result.stderr}")
|
||||
|
||||
|
||||
def main():
|
||||
environment = os.environ.copy()
|
||||
environment.update(
|
||||
{
|
||||
"GOTOOLCHAIN": "local",
|
||||
"GOWORK": "off",
|
||||
"GOPROXY": "off",
|
||||
"GONOPROXY": "none",
|
||||
"GOSUMDB": "off",
|
||||
}
|
||||
)
|
||||
selected_go = run(
|
||||
["go", "env", "GOVERSION"],
|
||||
cwd=REPOSITORY_ROOT,
|
||||
environment=environment,
|
||||
).strip()
|
||||
if not selected_go:
|
||||
print("go env GOVERSION returned no version", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
ignored_rule = ".releasekit/"
|
||||
ignore_lines = (REPOSITORY_ROOT / ".gitignore").read_text(
|
||||
encoding="utf-8"
|
||||
).splitlines()
|
||||
if ignored_rule not in ignore_lines:
|
||||
print(".gitignore does not contain the ReleaseKit output rule", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
module_bytes = {
|
||||
name: (REPOSITORY_ROOT / name).read_bytes() for name in ("go.mod", "go.sum")
|
||||
}
|
||||
with tempfile.TemporaryDirectory(prefix="promptkit-release-fixture-") as directory:
|
||||
temporary_root = Path(directory)
|
||||
repository = temporary_root / "checkout"
|
||||
remote = temporary_root / "origin.git"
|
||||
suite_marker = temporary_root / "suite.jsonl"
|
||||
example_marker = temporary_root / "example.log"
|
||||
environment["FIXTURE_SUITE_MARKER"] = str(suite_marker)
|
||||
environment["FIXTURE_EXAMPLE_MARKER"] = str(example_marker)
|
||||
try:
|
||||
prepare_repository(
|
||||
repository, remote, environment, module_bytes, RELEASE_VERSION
|
||||
)
|
||||
verify_publication(
|
||||
repository,
|
||||
remote,
|
||||
temporary_root,
|
||||
environment,
|
||||
module_bytes,
|
||||
selected_go,
|
||||
)
|
||||
verify_usage_classification(
|
||||
repository / ".releasekit/bin/releasekit", temporary_root, environment
|
||||
)
|
||||
verify_validation_failure(
|
||||
temporary_root / "failure-checkout",
|
||||
temporary_root / "failure-origin.git",
|
||||
temporary_root,
|
||||
environment,
|
||||
module_bytes,
|
||||
selected_go,
|
||||
)
|
||||
except (RuntimeError, OSError, json.JSONDecodeError) as error:
|
||||
print(error, file=sys.stderr)
|
||||
return 1
|
||||
|
||||
print("ReleaseKit publication success and failure fixtures passed")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user