Compare commits
74 Commits
62a4a20529
...
v0.13.0
| Author | SHA1 | Date | |
|---|---|---|---|
| 18ca21fa45 | |||
| e32fdc22f4 | |||
| b02b802427 | |||
| 301192b974 | |||
| f2aeb5eb30 | |||
| a27ee8f7da | |||
| 4d1895c223 | |||
| d7fe70520a | |||
| 7325a34c07 | |||
| 03c44923c8 | |||
| dc0adf5751 | |||
| eae21797f7 | |||
| ea6c7d5f0f | |||
| 389ef26db3 | |||
| 8a5c0de162 | |||
| 80aecb646d | |||
| b7b4000aab | |||
| 11ecb923ba | |||
| f46d5a4591 | |||
| 743e78d514 | |||
| 3743221aa5 | |||
| a8863d49d0 | |||
| c5f733af45 | |||
| c09d2c6e20 | |||
| 6c5dec761c | |||
| 74b91fae99 | |||
| b7ff37118c | |||
| ef012ea1e5 | |||
| 04d5d695e0 | |||
| 78bff91395 | |||
| c63ca5352f | |||
| 3f157150e4 | |||
| f1e1a45a05 | |||
| 996295d5c1 | |||
| 58a5bbe790 | |||
| dd89be3b03 | |||
| ed7c27324d | |||
| 05fe13d80b | |||
| 0eb009ad5a | |||
| 3fde1f6cbb | |||
| 481e2d494c | |||
| eb830bb613 | |||
| dd6d4591ae | |||
| de6443f3ae | |||
| 5a63b649f9 | |||
| c7e22c0dbe | |||
| e69a847cf2 | |||
| 21815ad7c0 | |||
| a051edd82d | |||
| ee0ad99e17 | |||
| bc5a2da0f2 | |||
| 089e824a92 | |||
| 93e8874813 | |||
| cc0b6183bf | |||
| bf472dad8a | |||
| a6e0ffea8f | |||
| e459c7d54b | |||
| 71f6683fda | |||
| 7827ca043b | |||
| b595dca8c7 | |||
| d5f34642d7 | |||
| b1866946d1 | |||
| 05082c6fdf | |||
| 51b072965c | |||
| d5940856be | |||
| 0d163f8165 | |||
| 1889915a81 | |||
| 3ca01ae0b0 | |||
| 6e283d16c3 | |||
| 41b2021d8c | |||
| 3add7024db | |||
| a148a2b1f4 | |||
| 7310ef34a4 | |||
| 4b8169da28 |
4
.gitignore
vendored
4
.gitignore
vendored
@@ -22,6 +22,9 @@
|
|||||||
go.work
|
go.work
|
||||||
go.work.sum
|
go.work.sum
|
||||||
|
|
||||||
|
# Local ReleaseKit executable
|
||||||
|
.releasekit/
|
||||||
|
|
||||||
# env file
|
# env file
|
||||||
.env
|
.env
|
||||||
|
|
||||||
@@ -67,4 +70,3 @@ Icon
|
|||||||
.AppleDesktop
|
.AppleDesktop
|
||||||
Network Trash Folder
|
Network Trash Folder
|
||||||
Temporary Items
|
Temporary Items
|
||||||
.apdisk
|
|
||||||
|
|||||||
@@ -37,6 +37,10 @@ boundary and constraints that framework work must preserve.
|
|||||||
|
|
||||||
## Release Guidance
|
## Release Guidance
|
||||||
|
|
||||||
|
Consumers upgrading to `v0.12.0` should read the
|
||||||
|
[v0.12.0 release notes](docs/releases/v0.12.0.md) for the intentional ConfigKit
|
||||||
|
configuration and public construction API migration.
|
||||||
|
|
||||||
The [v0.11.2 release notes](docs/releases/v0.11.2.md) cover policy alignment
|
The [v0.11.2 release notes](docs/releases/v0.11.2.md) cover policy alignment
|
||||||
and copied-policy attribution updates.
|
and copied-policy attribution updates.
|
||||||
|
|
||||||
|
|||||||
237
assembled_workflow_benchmark_test.go
Normal file
237
assembled_workflow_benchmark_test.go
Normal file
@@ -0,0 +1,237 @@
|
|||||||
|
package promptkit_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"testing/fstest"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/ConfigKit"
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
assembledPreparedSink *promptkit.PreparedRun
|
||||||
|
assembledResultSink *promptkit.RunResult
|
||||||
|
assembledDetailsSink promptkit.PreparedRun
|
||||||
|
assembledHandleSink *promptkit.PreparedExecution
|
||||||
|
)
|
||||||
|
|
||||||
|
type assembledWorkflowClient struct{ content string }
|
||||||
|
|
||||||
|
func (client assembledWorkflowClient) Generate(
|
||||||
|
context.Context,
|
||||||
|
promptkit.GenerateRequest,
|
||||||
|
) (*promptkit.GenerateResponse, error) {
|
||||||
|
return &promptkit.GenerateResponse{Content: client.content}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type assembledWorkflowFixture struct {
|
||||||
|
engine *promptkit.Engine
|
||||||
|
request promptkit.RunRequest
|
||||||
|
expectedMessage string
|
||||||
|
response string
|
||||||
|
}
|
||||||
|
|
||||||
|
func assembledWorkflowSchema(b *testing.B) []byte {
|
||||||
|
b.Helper()
|
||||||
|
properties := make(map[string]any, 256)
|
||||||
|
for index := range 256 {
|
||||||
|
properties[fmt.Sprintf("property-%03d", index)] = map[string]any{"type": "string"}
|
||||||
|
}
|
||||||
|
data, err := json.Marshal(map[string]any{
|
||||||
|
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||||
|
"type": "object",
|
||||||
|
"properties": properties,
|
||||||
|
"additionalProperties": false,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
b.Fatal(err)
|
||||||
|
}
|
||||||
|
return data
|
||||||
|
}
|
||||||
|
|
||||||
|
func assembledWorkflowExtras(entries int) map[string]any {
|
||||||
|
providers := make(map[string]any, entries)
|
||||||
|
for index := range entries {
|
||||||
|
providers[fmt.Sprintf("provider-%03d", index)] = map[string]any{
|
||||||
|
"enabled": true,
|
||||||
|
"labels": []any{fmt.Sprintf("tier-%d", index%4), fmt.Sprintf("region-%d", index%3)},
|
||||||
|
"limits": map[string]any{"burst": index + 1, "period": "minute"},
|
||||||
|
"routes": []any{map[string]any{"weight": index + 1, "fallback": true}},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return map[string]any{"provider_options": providers}
|
||||||
|
}
|
||||||
|
|
||||||
|
func newAssembledWorkflowFixture(b *testing.B, name string) assembledWorkflowFixture {
|
||||||
|
b.Helper()
|
||||||
|
message := "Hello {{.name}}"
|
||||||
|
response := "ok"
|
||||||
|
output := map[string]any{"format": "text", "validation_mode": "none"}
|
||||||
|
profile := map[string]any{"endpoint": "http://example.test/v1", "model": "benchmark-model"}
|
||||||
|
options := []promptkit.Option{promptkit.WithLLMClient(assembledWorkflowClient{content: response})}
|
||||||
|
var expectedMessage string
|
||||||
|
|
||||||
|
switch name {
|
||||||
|
case "small":
|
||||||
|
expectedMessage = "Hello Ada"
|
||||||
|
case "large_message":
|
||||||
|
prefix := strings.Repeat("x", 256*1024)
|
||||||
|
message = prefix + "{{.name}}"
|
||||||
|
expectedMessage = prefix + "Ada"
|
||||||
|
case "mutable_payload":
|
||||||
|
message = `{"subject":"{{.name}}"}`
|
||||||
|
expectedMessage = `{"subject":"Ada"}`
|
||||||
|
response = "{}"
|
||||||
|
profile["extra_params"] = assembledWorkflowExtras(64)
|
||||||
|
output = map[string]any{
|
||||||
|
"format": "json", "validation_mode": "json_schema", "schema_path": "workflow-schema.json",
|
||||||
|
}
|
||||||
|
options[0] = promptkit.WithLLMClient(assembledWorkflowClient{content: response})
|
||||||
|
options = append(options, promptkit.WithSchemaFS(fstest.MapFS{
|
||||||
|
"workflow-schema.json": &fstest.MapFile{Data: assembledWorkflowSchema(b)},
|
||||||
|
}, "."))
|
||||||
|
default:
|
||||||
|
b.Fatalf("unknown assembled workflow fixture %q", name)
|
||||||
|
}
|
||||||
|
|
||||||
|
config := map[string]any{
|
||||||
|
"profiles": map[string]any{"profile": profile},
|
||||||
|
"prompts": map[string]any{"workflow": map[string]any{"v1": map[string]any{
|
||||||
|
"default_profile": "profile",
|
||||||
|
"messages": []any{map[string]any{"role": "user", "content": message}},
|
||||||
|
"output": output,
|
||||||
|
}}},
|
||||||
|
}
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
|
||||||
|
Source: configkit.Mapping("assembled workflow benchmark", config),
|
||||||
|
}}}, options...)
|
||||||
|
if err != nil {
|
||||||
|
b.Fatal(err)
|
||||||
|
}
|
||||||
|
request := promptkit.RunRequest{
|
||||||
|
PromptID: "workflow", PromptVersion: "v1", Vars: map[string]string{"name": "Ada"},
|
||||||
|
}
|
||||||
|
return assembledWorkflowFixture{
|
||||||
|
engine: engine, request: request, expectedMessage: expectedMessage, response: response,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func verifyAssembledWorkflowFixture(b *testing.B, fixture assembledWorkflowFixture) {
|
||||||
|
b.Helper()
|
||||||
|
ctx := context.Background()
|
||||||
|
prepared, err := fixture.engine.Prepare(ctx, fixture.request)
|
||||||
|
if err != nil {
|
||||||
|
b.Fatal(err)
|
||||||
|
}
|
||||||
|
if prepared.PromptID != "workflow" || prepared.PromptVersion != "v1" || len(prepared.Messages) != 1 ||
|
||||||
|
prepared.Messages[0].Content != fixture.expectedMessage {
|
||||||
|
b.Fatalf("prepared fixture output is invalid: %+v", prepared)
|
||||||
|
}
|
||||||
|
handle, err := fixture.engine.PrepareExecution(ctx, fixture.request)
|
||||||
|
if err != nil {
|
||||||
|
b.Fatal(err)
|
||||||
|
}
|
||||||
|
details := handle.Details()
|
||||||
|
if details.PromptID != "workflow" || len(details.Messages) != 1 || details.Messages[0].Content != fixture.expectedMessage {
|
||||||
|
handle.Discard()
|
||||||
|
b.Fatalf("prepared execution details are invalid: %+v", details)
|
||||||
|
}
|
||||||
|
handle.Discard()
|
||||||
|
result, err := fixture.engine.Run(ctx, fixture.request)
|
||||||
|
if err != nil {
|
||||||
|
b.Fatal(err)
|
||||||
|
}
|
||||||
|
if result.RawOutput != fixture.response {
|
||||||
|
b.Fatalf("run output=%q, want %q", result.RawOutput, fixture.response)
|
||||||
|
}
|
||||||
|
if fixture.response == "{}" && result.Validation.Status != promptkit.ValidationPassed {
|
||||||
|
b.Fatalf("structured response validation=%+v, want passed", result.Validation)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func BenchmarkEnginePreparation(b *testing.B) {
|
||||||
|
for _, name := range []string{"small", "large_message", "mutable_payload"} {
|
||||||
|
b.Run(name, func(b *testing.B) {
|
||||||
|
fixture := newAssembledWorkflowFixture(b, name)
|
||||||
|
verifyAssembledWorkflowFixture(b, fixture)
|
||||||
|
ctx := context.Background()
|
||||||
|
b.Run("prepare", func(b *testing.B) {
|
||||||
|
b.ReportAllocs()
|
||||||
|
b.ResetTimer()
|
||||||
|
for b.Loop() {
|
||||||
|
prepared, err := fixture.engine.Prepare(ctx, fixture.request)
|
||||||
|
if err != nil {
|
||||||
|
b.Fatal(err)
|
||||||
|
}
|
||||||
|
assembledPreparedSink = prepared
|
||||||
|
}
|
||||||
|
b.StopTimer()
|
||||||
|
assembledPreparedSink = nil
|
||||||
|
})
|
||||||
|
b.Run("prepared_execution_creation_and_discard", func(b *testing.B) {
|
||||||
|
b.ReportAllocs()
|
||||||
|
b.ResetTimer()
|
||||||
|
for b.Loop() {
|
||||||
|
handle, err := fixture.engine.PrepareExecution(ctx, fixture.request)
|
||||||
|
if err != nil {
|
||||||
|
b.Fatal(err)
|
||||||
|
}
|
||||||
|
handle.Discard()
|
||||||
|
assembledHandleSink = handle
|
||||||
|
}
|
||||||
|
b.StopTimer()
|
||||||
|
assembledHandleSink = nil
|
||||||
|
})
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func BenchmarkEngineRun(b *testing.B) {
|
||||||
|
for _, name := range []string{"small", "large_message", "mutable_payload"} {
|
||||||
|
b.Run(name, func(b *testing.B) {
|
||||||
|
fixture := newAssembledWorkflowFixture(b, name)
|
||||||
|
verifyAssembledWorkflowFixture(b, fixture)
|
||||||
|
ctx := context.Background()
|
||||||
|
b.ReportAllocs()
|
||||||
|
b.ResetTimer()
|
||||||
|
for b.Loop() {
|
||||||
|
result, err := fixture.engine.Run(ctx, fixture.request)
|
||||||
|
if err != nil {
|
||||||
|
b.Fatal(err)
|
||||||
|
}
|
||||||
|
assembledResultSink = result
|
||||||
|
}
|
||||||
|
b.StopTimer()
|
||||||
|
assembledResultSink = nil
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func BenchmarkPreparedExecutionDetails(b *testing.B) {
|
||||||
|
for _, name := range []string{"small", "large_message", "mutable_payload"} {
|
||||||
|
b.Run(name, func(b *testing.B) {
|
||||||
|
fixture := newAssembledWorkflowFixture(b, name)
|
||||||
|
verifyAssembledWorkflowFixture(b, fixture)
|
||||||
|
handle, err := fixture.engine.PrepareExecution(context.Background(), fixture.request)
|
||||||
|
if err != nil {
|
||||||
|
b.Fatal(err)
|
||||||
|
}
|
||||||
|
defer handle.Discard()
|
||||||
|
initial := handle.Details()
|
||||||
|
if len(initial.Messages) != 1 || initial.Messages[0].Content != fixture.expectedMessage {
|
||||||
|
b.Fatalf("prepared details fixture is invalid: %+v", initial)
|
||||||
|
}
|
||||||
|
b.ReportAllocs()
|
||||||
|
b.ResetTimer()
|
||||||
|
for b.Loop() {
|
||||||
|
assembledDetailsSink = handle.Details()
|
||||||
|
}
|
||||||
|
b.StopTimer()
|
||||||
|
assembledDetailsSink = promptkit.PreparedRun{}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
25
doc.go
25
doc.go
@@ -4,15 +4,16 @@
|
|||||||
// Applications construct an [Engine] with [NewEngine], select ConfigKit
|
// Applications construct an [Engine] with [NewEngine], select ConfigKit
|
||||||
// sources for prompts, profiles, and backends, and
|
// sources for prompts, profiles, and backends, and
|
||||||
// call [Engine.InspectConfiguration], [Engine.InspectPrompt],
|
// call [Engine.InspectConfiguration], [Engine.InspectPrompt],
|
||||||
// [Engine.InspectProfile], [Engine.Prepare],
|
// [Engine.InspectRequirements], [Engine.InspectProfile], [Engine.ListProfiles],
|
||||||
// [Engine.PrepareExecution], [Engine.Run], or [Engine.RunPrepared]. Concrete
|
// [Engine.ListPrompts], [Engine.Prepare], [Engine.PrepareExecution],
|
||||||
// registries, repositories, validators, and the built-in OpenAI-compatible
|
// [Engine.Run], or [Engine.RunPrepared]. Concrete registries, repositories,
|
||||||
// client remain internal implementation details.
|
// validators, and the built-in OpenAI-compatible client remain internal
|
||||||
|
// implementation details.
|
||||||
//
|
//
|
||||||
// # Concurrency and ownership
|
// # Concurrency and ownership
|
||||||
//
|
//
|
||||||
// An Engine supports concurrent InspectConfiguration, InspectPrompt,
|
// An Engine supports concurrent InspectConfiguration, InspectPrompt,
|
||||||
// InspectProfile, Prepare,
|
// InspectRequirements, InspectProfile, ListProfiles, ListPrompts, Prepare,
|
||||||
// PrepareExecution, Run, and RunPrepared calls. Engine-local backend policies
|
// PrepareExecution, Run, and RunPrepared calls. Engine-local backend policies
|
||||||
// bound admitted Run and RunPrepared calls and model generations where
|
// bound admitted Run and RunPrepared calls and model generations where
|
||||||
// configured, while different backend pools and unlimited backends continue
|
// configured, while different backend pools and unlimited backends continue
|
||||||
@@ -20,11 +21,12 @@
|
|||||||
// still receive concurrent calls and must be safe for that use.
|
// still receive concurrent calls and must be safe for that use.
|
||||||
//
|
//
|
||||||
// ConfigKit captures the selected configuration when NewEngine constructs an
|
// ConfigKit captures the selected configuration when NewEngine constructs an
|
||||||
// Engine. Prepare,
|
// Engine. Prepare, PrepareExecution, and Run copy request maps, slices, pointer
|
||||||
// PrepareExecution, and Run copy request maps, slices, pointer values, and
|
// values, and JSON-compatible extra parameters before using them. InspectPrompt,
|
||||||
// JSON-compatible extra parameters before using them. InspectPrompt and
|
// InspectRequirements, and InspectProfile return copied inspection values.
|
||||||
// InspectProfile return copied inspection values. Returned values and values
|
// ListProfiles and ListPrompts return caller-owned identity slices.
|
||||||
// passed to extension interfaces are likewise isolated from engine state.
|
// Returned values and values passed to extension interfaces are likewise
|
||||||
|
// isolated from engine state.
|
||||||
// Callers own those copies and may mutate them after the call that supplied or
|
// Callers own those copies and may mutate them after the call that supplied or
|
||||||
// returned them. [CapacityError] values are caller-owned and may be mutated
|
// returned them. [CapacityError] values are caller-owned and may be mutated
|
||||||
// without affecting engine state or another error. Immutable [GenerationError]
|
// without affecting engine state or another error. Immutable [GenerationError]
|
||||||
@@ -58,7 +60,8 @@
|
|||||||
// [ConfigurationSources], [ConfigurationInspection], [BackendConfig],
|
// [ConfigurationSources], [ConfigurationInspection], [BackendConfig],
|
||||||
// [ProfileConfig], [RunRequest],
|
// [ProfileConfig], [RunRequest],
|
||||||
// [ArtifactRef], [ExecutionTargetOverride], [ProfileInspection],
|
// [ArtifactRef], [ExecutionTargetOverride], [ProfileInspection],
|
||||||
// [PromptInputDefinition], [PromptInspection], [PreparedExecution],
|
// [PromptIdentity], [PromptInputDefinition], [PromptInspection],
|
||||||
|
// [RequirementInspectionRequest], [RequirementInspection], [PreparedExecution],
|
||||||
// [CapacityError], [GenerationError], [RetryExhaustedError], and
|
// [CapacityError], [GenerationError], [RetryExhaustedError], and
|
||||||
// [ExecutionEvent], do not have stable JSON
|
// [ExecutionEvent], do not have stable JSON
|
||||||
// representations. Direct API keys are nevertheless excluded from JSON for
|
// representations. Direct API keys are nevertheless excluded from JSON for
|
||||||
|
|||||||
@@ -19,6 +19,20 @@ The following Go fragments are illustrative and omit surrounding package,
|
|||||||
import, and error-handling code. Use the maintained examples for complete
|
import, and error-handling code. Use the maintained examples for complete
|
||||||
programs.
|
programs.
|
||||||
|
|
||||||
|
## Go Compatibility
|
||||||
|
|
||||||
|
The `go` directive in the selected release's [module metadata](../../go.mod)
|
||||||
|
declares its minimum supported consumer Go version. Maintainers build and
|
||||||
|
validate releases with the locally selected Go patch release; the minimum
|
||||||
|
does not simply track that patch version.
|
||||||
|
|
||||||
|
Successful release validation on the selected patch is sufficient to presume
|
||||||
|
support across its Go series, subject to the declared minimum and concrete
|
||||||
|
evidence of incompatibility. Specific compatibility concerns are investigated
|
||||||
|
as they arise. Minor releases may raise the minimum; patch releases preserve
|
||||||
|
it. Release notes explain increases and any required consumer action. See the
|
||||||
|
[release policy](../policy/release.md#consumer-minimum-and-release-toolchain).
|
||||||
|
|
||||||
## Construct An Engine
|
## Construct An Engine
|
||||||
|
|
||||||
Create an engine with [`NewEngine`](../../engine.go). Select each directory
|
Create an engine with [`NewEngine`](../../engine.go). Select each directory
|
||||||
@@ -104,6 +118,43 @@ engines and prepared handles retain their captured definitions; schema and
|
|||||||
request artifact resources still follow their runtime lifecycle. Inspection
|
request artifact resources still follow their runtime lifecycle. Inspection
|
||||||
does not watch or reload sources automatically.
|
does not watch or reload sources automatically.
|
||||||
|
|
||||||
|
## Discover Available Profiles And Prompts
|
||||||
|
|
||||||
|
Use [`Engine.ListProfiles`](../../listing.go) and
|
||||||
|
[`Engine.ListPrompts`](../../listing.go) to discover exact identities from the
|
||||||
|
engine's final captured configuration, then inspect a selected prompt:
|
||||||
|
|
||||||
|
```go
|
||||||
|
profiles, err := engine.ListProfiles(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if len(profiles) > 0 {
|
||||||
|
_, err := engine.InspectProfile(ctx, profiles[0])
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
prompts, err := engine.ListPrompts(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if len(prompts) > 0 {
|
||||||
|
selected := prompts[0]
|
||||||
|
inspection, err := engine.InspectPrompt(ctx, selected.ID, selected.Version)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
_ = inspection
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
These lists describe effective membership, not ConfigKit contribution history,
|
||||||
|
and do not promise that a profile is ready to execute. Use inspection for
|
||||||
|
details and preparation to resolve request resources. The linked method GoDoc
|
||||||
|
owns ordering, ownership, concurrency, and error behavior; identifiers may be
|
||||||
|
sensitive application data.
|
||||||
|
|
||||||
## Inspect The Captured Configuration
|
## Inspect The Captured Configuration
|
||||||
|
|
||||||
Use [`Engine.InspectConfiguration`](../../configuration_inspection.go) for an
|
Use [`Engine.InspectConfiguration`](../../configuration_inspection.go) for an
|
||||||
@@ -154,6 +205,50 @@ rendered messages, and use prepared execution when that work must remain tied
|
|||||||
to later execution. The method's [GoDoc](../../engine.go) owns exact fields,
|
to later execution. The method's [GoDoc](../../engine.go) owns exact fields,
|
||||||
hash, ownership, and error semantics.
|
hash, ownership, and error semantics.
|
||||||
|
|
||||||
|
## Inspect Requirements Before Preparation
|
||||||
|
|
||||||
|
Use [`Engine.InspectRequirements`](../../requirements.go) when the application
|
||||||
|
needs the prompt's recognizable input and caller-variable dependencies for a
|
||||||
|
particular context before it has input artifacts. Unlike `InspectPrompt`, it
|
||||||
|
also walks captured template syntax and selects a profile identity; unlike
|
||||||
|
`Prepare`, it does not load artifacts or render messages.
|
||||||
|
|
||||||
|
```go
|
||||||
|
requirements, err := engine.InspectRequirements(ctx, promptkit.RequirementInspectionRequest{
|
||||||
|
PromptID: "meeting.summary",
|
||||||
|
Vars: map[string]string{"audience": "project leads"},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, input := range requirements.Inputs {
|
||||||
|
if input.Kind == promptkit.RequirementRequired {
|
||||||
|
// Plan to bind this input before preparation.
|
||||||
|
}
|
||||||
|
if input.Kind == promptkit.RequirementOptional && input.Explanation != "" {
|
||||||
|
// A recognizable reference may occur only on a known skipped path.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, variable := range requirements.Variables {
|
||||||
|
if variable.Kind == promptkit.RequirementRequired && !variable.Supplied {
|
||||||
|
// The key is required even though this inspection did not receive it.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !requirements.Complete {
|
||||||
|
// Preserve the partial inventory and account for its discovery issues.
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Optional entries can include recognizable inputs and variables found only in
|
||||||
|
inactive branches. A missing required variable is reported separately from
|
||||||
|
discovery completeness: missing a value used only for direct text substitution
|
||||||
|
does not by itself make the inventory incomplete. Conversely, an unresolved
|
||||||
|
dependency-bearing branch can make discovery incomplete while retaining known
|
||||||
|
conditional entries. Do not treat an empty partial inventory as proof that no
|
||||||
|
input bindings are needed. See the method's [GoDoc](../../requirements.go) for
|
||||||
|
the exact classifications, supported syntax, ownership, and error contract.
|
||||||
|
|
||||||
## Prepare Without Model Execution
|
## Prepare Without Model Execution
|
||||||
|
|
||||||
[`Engine.Prepare`](../../engine.go) resolves the selected prompt and profile,
|
[`Engine.Prepare`](../../engine.go) resolves the selected prompt and profile,
|
||||||
|
|||||||
@@ -59,37 +59,43 @@ exist.
|
|||||||
## Maintainer Validation
|
## Maintainer Validation
|
||||||
|
|
||||||
For ordinary work, select validation proportionate to the change. Do not repeat
|
For ordinary work, select validation proportionate to the change. Do not repeat
|
||||||
successful checks for unchanged content merely to create a commit. The full
|
successful checks for unchanged content merely to create a commit. During
|
||||||
release-candidate checks have one executable entry point from the repository root:
|
development, run relevant Go tests and vet, format changed Go files, and execute
|
||||||
|
the maintained examples when they are affected:
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
./scripts/release-check.sh
|
go test ./...
|
||||||
|
go vet ./...
|
||||||
|
gofmt -w path/to/changed.go
|
||||||
|
python3 scripts/check-examples.py
|
||||||
|
git diff --check
|
||||||
```
|
```
|
||||||
|
|
||||||
The [script](../scripts/release-check.sh) owns the executable checks:
|
Release candidates use `./scripts/release.sh VERSION`, which runs the selected
|
||||||
race-enabled tests, vet, the three offline consumer examples with semantic output
|
profile checks and publishes the tag. The [release procedure](release.md)
|
||||||
assertions, tracked Go formatting, local Markdown links, module/workspace
|
describes its requirements and recovery. For an optional diagnostic of a
|
||||||
hygiene, whitespace, and the credential-pattern scan. It does not publish,
|
committed candidate without publication, run `./scripts/releasekit.sh check
|
||||||
change files, validate release-note structure, or retire roadmaps. It requires Go,
|
VERSION`; this is not a required pre-publication step.
|
||||||
Git, a POSIX shell, and Python 3. Tests and examples require no provider
|
|
||||||
credentials or live inference calls.
|
|
||||||
|
|
||||||
The script can run on intended uncommitted changes during development.
|
The standalone `python3 scripts/check-examples.py` command checks semantic
|
||||||
Formatting and documentation checks use tracked files; stage intended new
|
expectations for the three offline consumer examples. The focused
|
||||||
files before final validation so those checks include them. The release
|
`python3 scripts/test-release.py` command exercises the release tooling against
|
||||||
procedure separately enforces a clean, committed candidate.
|
disposable local repositories; use it when changing that tooling, not as a
|
||||||
|
recurring release gate. Tests and examples require no provider credentials or
|
||||||
|
live inference calls.
|
||||||
|
|
||||||
|
Ordinary validation can run on intended uncommitted work. Release validation
|
||||||
|
uses committed candidate contents and the selected local Go installation.
|
||||||
|
|
||||||
### Change Review
|
### Change Review
|
||||||
|
|
||||||
Inspect `git status --short --untracked-files=all` and the complete diff before
|
Inspect `git status --short --untracked-files=all` and the complete diff before
|
||||||
accepting a change. Reject unintended files, credentials, private data,
|
accepting a change. Reject unintended files, credentials, private data,
|
||||||
generated binaries, test output, downloaded assets, and template residue.
|
generated binaries, test output, downloaded assets, and template residue.
|
||||||
The automated credential scan does not replace review of the actual change.
|
|
||||||
|
|
||||||
For releases, follow the [release procedure](release.md). Its publication
|
For releases, follow the [release procedure](release.md). Do not repeat its
|
||||||
command runs the full local checks once on the final committed candidate and
|
candidate checks as a separate sign-off. Roadmap retirement remains preparation
|
||||||
automates release-note structure and Git checks. Do not repeat ordinary change
|
housekeeping.
|
||||||
review as a release sign-off. Roadmap retirement remains preparation housekeeping.
|
|
||||||
|
|
||||||
Follow the [agent commit policy](policy/agents.md#git-commit-policy) for local
|
Follow the [agent commit policy](policy/agents.md#git-commit-policy) for local
|
||||||
commits. Ordinary commits may leave unrelated work in the workspace; preserve
|
commits. Ordinary commits may leave unrelated work in the workspace; preserve
|
||||||
|
|||||||
@@ -51,7 +51,8 @@ message files on ordinary runs; construct a new engine to observe edits.
|
|||||||
Schema and request artifact reads still happen during preparation. Use
|
Schema and request artifact reads still happen during preparation. Use
|
||||||
`InspectConfiguration` for an explicit effective-configuration or provenance
|
`InspectConfiguration` for an explicit effective-configuration or provenance
|
||||||
report, `InspectProfile` for backend-completed profile settings, and preparation
|
report, `InspectProfile` for backend-completed profile settings, and preparation
|
||||||
for request-specific rendered details.
|
for request-specific rendered details. For identity discovery before inspection,
|
||||||
|
see the consumer guide's [profile and prompt listing](consumers/pkg-promptkit.md#discover-available-profiles-and-prompts).
|
||||||
|
|
||||||
## Prompt Definitions
|
## Prompt Definitions
|
||||||
|
|
||||||
@@ -124,6 +125,12 @@ Optional `session_id` uses the same template variables and input helper. Its
|
|||||||
rendered value is trimmed, omitted if empty, and limited to 256 Unicode code
|
rendered value is trimmed, omitted if empty, and limited to 256 Unicode code
|
||||||
points. A nonblank direct request session ID bypasses the template.
|
points. A nonblank direct request session ID bypasses the template.
|
||||||
|
|
||||||
|
`InspectRequirements` can enumerate recognizable dependencies from captured
|
||||||
|
templates before artifact content is available. Its conservative syntax
|
||||||
|
coverage does not change template validity or rendering; see the [consumer
|
||||||
|
guide](consumers/pkg-promptkit.md#inspect-requirements-before-preparation) and
|
||||||
|
the method's [GoDoc](../requirements.go) for that inspection boundary.
|
||||||
|
|
||||||
### Cache Control
|
### Cache Control
|
||||||
|
|
||||||
Optional `cache_control` has required `type: ephemeral` and optional `ttl`,
|
Optional `cache_control` has required `type: ephemeral` and optional `ttl`,
|
||||||
|
|||||||
@@ -78,9 +78,8 @@ The client conditionally includes:
|
|||||||
|
|
||||||
The engine resolves backend, profile, and request extra-parameter maps by
|
The engine resolves backend, profile, and request extra-parameter maps by
|
||||||
whole-map replacement rather than key merging. The resulting effective map is
|
whole-map replacement rather than key merging. The resulting effective map is
|
||||||
then merged directly into the top-level body after JSON serialization is
|
merged into the top-level body. Empty keys, values that cannot be encoded, and
|
||||||
verified. Empty keys and collisions with these reserved fields are rejected
|
collisions with these reserved fields are rejected before any provider call:
|
||||||
before any provider call:
|
|
||||||
|
|
||||||
- `model`
|
- `model`
|
||||||
- `session_id`
|
- `session_id`
|
||||||
|
|||||||
@@ -34,9 +34,13 @@ resolved request target may supply the endpoint. Generation preparation then:
|
|||||||
1. validates shared execution-setting invariants and the final selected base
|
1. validates shared execution-setting invariants and the final selected base
|
||||||
endpoint;
|
endpoint;
|
||||||
2. maps the internal request into the OpenAI-compatible chat payload;
|
2. maps the internal request into the OpenAI-compatible chat payload;
|
||||||
3. validates and merges extra parameters;
|
3. checks extra-parameter keys against empty and reserved names, then merges
|
||||||
|
their values into the payload;
|
||||||
4. composes `/chat/completions` through parsed URL path operations;
|
4. composes `/chat/completions` through parsed URL path operations;
|
||||||
5. resolves authentication and freezes the URL, body, and optional credential.
|
5. encodes the complete body once on success. If encoding fails, extra values
|
||||||
|
are checked individually to retain invalid-request classification; otherwise
|
||||||
|
the complete-request failure remains a request failure;
|
||||||
|
6. resolves authentication and freezes the URL, body, and optional credential.
|
||||||
|
|
||||||
Each execution of that prepared request creates a fresh HTTP request, body
|
Each execution of that prepared request creates a fresh HTTP request, body
|
||||||
reader, and attempt timeout context. It closes the response body and returns
|
reader, and attempt timeout context. It closes the response body and returns
|
||||||
|
|||||||
@@ -11,28 +11,28 @@ contributor workflow and validation.
|
|||||||
|
|
||||||
| Component | Implemented responsibility | References |
|
| Component | Implemented responsibility | References |
|
||||||
| --- | --- | --- |
|
| --- | --- | --- |
|
||||||
| Root `promptkit` package | Provides the supported engine facade, ConfigKit source helpers, profile and backend mapping conveniences, injection options, public requests and results, explicit inspection, observation values, opaque prepared-execution handles, value conversion, redacted formatting, and typed capacity and generation error mapping. | [Package GoDoc](../../doc.go), [observations](../../observations.go), [prepared execution](../../prepared_execution.go), [backend API](../../backends.go), [engine facade](../../engine.go) |
|
| Root `promptkit` package | Provides the supported engine facade, ConfigKit source helpers, profile and backend mapping conveniences, injection options, public requests and results, prompt/profile/requirement inspection and identity listing, observation values, opaque prepared-execution handles, value conversion, redacted formatting, and typed capacity and generation error mapping. | [Package GoDoc](../../doc.go), [observations](../../observations.go), [prepared execution](../../prepared_execution.go), [backend API](../../backends.go), [engine facade](../../engine.go), [identity listing](../../listing.go), [requirement facade](../../requirements.go) |
|
||||||
| `examples/go-library/prepare` | Demonstrates offline preparation and explicit configuration inspection with a prompt file, programmatic profile, and inline input. It is not a public library package. | [Example program](../../examples/go-library/prepare/main.go) |
|
| `examples/go-library/prepare` | Demonstrates offline preparation and explicit configuration inspection with a prompt file, programmatic profile, and inline input. It is not a public library package. | [Example program](../../examples/go-library/prepare/main.go) |
|
||||||
| `examples/go-library/run` | Demonstrates an offline downstream consumer using a prompt file, programmatic profile, inline input, an injected deterministic model client, and `Run`. It is not a public library package. | [Example program](../../examples/go-library/run/main.go) |
|
| `examples/go-library/run` | Demonstrates an offline downstream consumer using a prompt file, programmatic profile, inline input, an injected deterministic model client, and `Run`. It is not a public library package. | [Example program](../../examples/go-library/run/main.go) |
|
||||||
| `examples/go-library/observe` | Demonstrates offline built-in HTTP retry and synchronous metadata observation across structured proposal and assessment runs using embedded sources, two profiles on one finite backend, and an application decision. It is not a public library package. | [Example program](../../examples/go-library/observe/main.go) |
|
| `examples/go-library/observe` | Demonstrates offline built-in HTTP retry and synchronous metadata observation across structured proposal and assessment runs using embedded sources, two profiles on one finite backend, and an application decision. It is not a public library package. | [Example program](../../examples/go-library/observe/main.go) |
|
||||||
| `internal/backend` | Holds a defensive, immutable lookup and capacity-policy index for final backend definitions validated by the configuration adapter. | [Backend registry](../../internal/backend/registry.go) |
|
| `internal/backend` | Holds a defensive, immutable lookup and capacity-policy index for final backend definitions validated by the configuration adapter. | [Backend registry](../../internal/backend/registry.go) |
|
||||||
| `internal/catalog` | Selects released keyed catalog assets and PromptKit-owned profile definitions as ConfigKit underlays; effective validation belongs to the configuration loader. | [Catalog adapter](../../internal/catalog/catalog.go), [internal sources](sources.md#profiles-and-built-ins) |
|
| `internal/catalog` | Selects released keyed catalog assets and PromptKit-owned profile definitions as ConfigKit underlays; effective validation belongs to the configuration loader. | [Catalog adapter](../../internal/catalog/catalog.go), [internal sources](sources.md#profiles-and-built-ins) |
|
||||||
| `internal/configuration` | Loads keyed backend, profile, and prompt configuration through ConfigKit; records effective field presence, validates final values, captures indexed prompt definitions and referenced message content, constructs backend and presence-aware profile snapshots, and retains an explicit read-only inspection view. | [Typed loader](../../internal/configuration/load.go), [internal sources](sources.md#typed-configuration-loading) |
|
| `internal/configuration` | Loads keyed backend, profile, and prompt configuration through ConfigKit; records effective field presence, validates final values, captures indexed prompt definitions, referenced message content, compiled templates, input metadata snapshots, sorted identity-only profile and prompt inventories, backend and presence-aware profile snapshots, and an explicit read-only inspection view. | [Typed loader](../../internal/configuration/load.go), [identity listing](../../internal/configuration/listing.go), [internal sources](sources.md#typed-configuration-loading) |
|
||||||
| `internal/capacity` | Owns engine-local bounded execution admission and FIFO model-generation permits for limited backend IDs, including cancellation-safe waiter removal and client wrapping. | [Internal capacity management](capacity.md) |
|
| `internal/capacity` | Owns engine-local bounded execution admission and FIFO model-generation permits for limited backend IDs, including cancellation-safe waiter removal and client wrapping. | [Internal capacity management](capacity.md) |
|
||||||
| `internal/domain` | Defines internal framework values for requests, artifacts, prompt definitions, profiles, execution targets, rendering, generation, and validation, and owns source-neutral invariants for shared execution settings, OpenAI-compatible base endpoints, session identifiers, and output contracts. Source parsing, required fields, other source-specific normalization, defaulting, and boundary-specific error classification remain with their callers. | [Domain declarations](../../internal/domain/domain.go), [endpoint invariant](../../internal/domain/endpoint.go) |
|
| `internal/domain` | Defines internal framework values for requests, artifacts, prompt definitions, requirement inspection, profiles, execution targets, rendering, generation, and validation, and owns source-neutral invariants for shared execution settings, OpenAI-compatible base endpoints, session identifiers, and output contracts. Source parsing, required fields, other source-specific normalization, defaulting, and boundary-specific error classification remain with their callers. | [Domain declarations](../../internal/domain/domain.go), [requirement values](../../internal/domain/requirements.go), [endpoint invariant](../../internal/domain/endpoint.go) |
|
||||||
| `internal/engine` | Assembles selected catalog and caller configuration into one snapshot, then constructs capacity, rendering, validation, generation, repair, and the use-case runner for each engine. | [Engine assembly](../../internal/engine/engine.go), [internal sources](sources.md#profiles-and-built-ins), [internal runner](runner.md) |
|
| `internal/engine` | Assembles selected catalog and caller configuration into one snapshot, then constructs capacity, rendering, validation, generation, repair, and the use-case runner for each engine. It delegates identity listing directly to the captured configuration inventory rather than the runner. | [Engine assembly](../../internal/engine/engine.go), [internal sources](sources.md#profiles-and-built-ins), [internal runner](runner.md) |
|
||||||
| `internal/generation` | Coordinates frozen built-in HTTP attempts and one-shot injected calls through active backend permits, including bounded status retries, cancellable backoff, and attempt observations. | [Internal generation coordination](generation.md) |
|
| `internal/generation` | Coordinates frozen built-in HTTP attempts and one-shot injected calls through active backend permits, including bounded status retries, cancellable backoff, and attempt observations. | [Internal generation coordination](generation.md) |
|
||||||
| `internal/defaults` | Defines application-neutral framework constants and constructs the default execution target. It contains no CLI, server, or inbound HTTP limits. | [Framework defaults](../../internal/defaults/defaults.go) |
|
| `internal/defaults` | Defines application-neutral framework constants and constructs the default execution target. It contains no CLI, server, or inbound HTTP limits. | [Framework defaults](../../internal/defaults/defaults.go) |
|
||||||
| `internal/filecatalog` | Provides deterministic YAML discovery and path helpers for operating-system filesystems and `fs.FS` sources. | [File catalog](../../internal/filecatalog/catalog.go) |
|
| `internal/filecatalog` | Provides deterministic YAML discovery and path helpers for operating-system filesystems and `fs.FS` sources. | [File catalog](../../internal/filecatalog/catalog.go) |
|
||||||
| `internal/jsonvalue` | Validates and deeply copies bounded JSON-compatible extra-parameter and prepared-schema trees while preserving supported concrete value types and rejecting cycles or excessive depth and work. | [JSON values](../../internal/jsonvalue/jsonvalue.go) |
|
| `internal/jsonvalue` | Validates and deeply copies bounded JSON-compatible extra-parameter and prepared-schema trees while preserving supported concrete value types and rejecting cycles or excessive depth and work. | [JSON values](../../internal/jsonvalue/jsonvalue.go) |
|
||||||
| `internal/promptdef` | Provides a narrow captured-prompt lookup contract, not-found identity, and execution-ready definition hash. | [Prompt lookup](../../internal/promptdef/repository.go), [framework formats](../formats.md) |
|
| `internal/promptdef` | Provides a narrow captured-prompt lookup contract, not-found identity, and execution-ready definition hash. | [Prompt lookup](../../internal/promptdef/repository.go), [framework formats](../formats.md) |
|
||||||
| `internal/profile` | Provides a narrow captured-profile lookup contract and not-found identity. | [Profile lookup](../../internal/profile/repository.go), [framework formats](../formats.md) |
|
| `internal/profile` | Provides a narrow captured-profile lookup contract and not-found identity. | [Profile lookup](../../internal/profile/repository.go), [framework formats](../formats.md) |
|
||||||
| `internal/prompt` | Renders prompt messages from Go templates with artifact, variable, session, and cache-control data. | [Go-template renderer](../../internal/prompt/go_renderer.go) |
|
| `internal/prompt` | Renders prompt messages from Go templates with artifact, variable, session, and cache-control data, and conservatively enumerates dependencies from captured template syntax. | [Go-template renderer](../../internal/prompt/go_renderer.go), [requirement discovery](../../internal/prompt/requirements.go) |
|
||||||
| `internal/artifact` | Resolves ordinary inline and unrestricted caller-selected file references into copied artifacts with metadata and hashes. | [Internal sources and validation](sources.md) |
|
| `internal/artifact` | Resolves ordinary inline and unrestricted caller-selected file references into copied artifacts with metadata and hashes. | [Internal sources and validation](sources.md) |
|
||||||
| `internal/validate` | Validates basic, JSON, and JSON Schema output using operating-system filesystem or `fs.FS` schema sources and creates operation-local validation plans with canonical contained schema resources. | [Framework formats](../formats.md#schemas), [internal sources and validation](sources.md) |
|
| `internal/validate` | Validates basic, JSON, and JSON Schema output using operating-system filesystem or `fs.FS` schema sources and creates operation-local validation plans with canonical contained schema resources. | [Framework formats](../formats.md#schemas), [internal sources and validation](sources.md) |
|
||||||
| `internal/llm` | Defines the internal generation boundary and implements outbound OpenAI-compatible chat requests from resolved execution targets, including bounded structured non-success response decoding, successful-response decoding, authentication, deadline handling, and ownership of the OpenAI-compatible reserved request-field policy. | [Internal model client](llm.md) |
|
| `internal/llm` | Defines the internal generation boundary and implements outbound OpenAI-compatible chat requests from resolved execution targets, including bounded structured non-success response decoding, successful-response decoding, authentication, deadline handling, and ownership of the OpenAI-compatible reserved request-field policy. | [Internal model client](llm.md) |
|
||||||
| `internal/observation` | Owns invocation-local run IDs, presence-aware usage and phase accounting, and synchronous metadata event delivery for ordinary and claimed prepared runs and generation attempts. | [Internal execution observation](observation.md) |
|
| `internal/observation` | Owns invocation-local run IDs, presence-aware usage and phase accounting, and synchronous metadata event delivery for ordinary and claimed prepared runs and generation attempts. | [Internal execution observation](observation.md) |
|
||||||
| `internal/usecase` | Selects captured prompts and profiles, applies request overrides and completion checks, and coordinates ordinary and single-claim prepared execution across rendering, artifact loading, operation-local validation plans, generation, capacity, and bounded repair. | [Internal runner](runner.md), [prepared-execution implementation](../../internal/usecase/prepared_execution.go) |
|
| `internal/usecase` | Selects captured prompts and profiles, enumerates requirements from captured prompt syntax, applies request overrides and completion checks, and coordinates ordinary and single-claim prepared execution across rendering, artifact loading, operation-local validation plans, generation, capacity, and bounded repair. | [Internal runner](runner.md), [requirement inspection](../../internal/usecase/requirements_inspection.go), [prepared-execution implementation](../../internal/usecase/prepared_execution.go) |
|
||||||
|
|
||||||
The internal engine assembles these internal components without exposing their
|
The internal engine assembles these internal components without exposing their
|
||||||
representations. Consumers depend only on the root facade.
|
representations. Consumers depend only on the root facade.
|
||||||
|
|||||||
@@ -32,16 +32,29 @@ The runner has no durable run or session store.
|
|||||||
|
|
||||||
## Shared Prompt Selection
|
## Shared Prompt Selection
|
||||||
|
|
||||||
The runner uses one prompt-selection and hashing boundary for ordinary
|
Ordinary preparation selects the full prompt definition and construction-time
|
||||||
preparation and exact prompt inspection. Preparation retains its early
|
hash. It retains its early request-ID check before direct-session
|
||||||
request-ID check before direct-session normalization; both operations select
|
normalization. Referenced message content and static template syntax were
|
||||||
one captured definition and its construction-time hash. Referenced message
|
captured when the engine was built.
|
||||||
content and static template syntax were captured when the engine was built.
|
|
||||||
|
|
||||||
Inspection stops after that structural lookup. It does not render templates or
|
`InspectPrompt` selects captured metadata through an optional repository
|
||||||
touch profile, artifact, schema, renderer, validator, admission, or model
|
capability, falling back to the full definition and existing hash resolution
|
||||||
collaborators. The root [`Engine.InspectPrompt`](../../engine.go) GoDoc owns
|
for internal repositories without it. It returns owned metadata without
|
||||||
the public operation's exact contract.
|
rendering templates or touching profile, artifact, schema, renderer, validator,
|
||||||
|
admission, or model collaborators. The root
|
||||||
|
[`Engine.InspectPrompt`](../../engine.go) GoDoc owns the public operation's
|
||||||
|
exact contract.
|
||||||
|
|
||||||
|
`InspectRequirements` uses the same metadata selection to resolve the
|
||||||
|
request-profile/default identity, then obtains the exact captured requirement
|
||||||
|
snapshot by the selected prompt ID and version. The analyzer receives only
|
||||||
|
those identity fields rather than an execution definition. It analyzes the
|
||||||
|
snapshot with the supplied caller variables and normalized direct-session
|
||||||
|
bypass; it does not resolve target readiness or enter the preparation
|
||||||
|
pipeline. Missing snapshot capability is a prompt-load failure rather than a
|
||||||
|
source fallback. The root
|
||||||
|
[`Engine.InspectRequirements`](../../requirements.go) GoDoc owns the public
|
||||||
|
operation's exact contract.
|
||||||
|
|
||||||
## Shared Profile Selection
|
## Shared Profile Selection
|
||||||
|
|
||||||
@@ -92,8 +105,11 @@ one execution-precedence and error-ordering implementation while allowing a
|
|||||||
full backend pool to reject work before expensive schema, artifact, and
|
full backend pool to reject work before expensive schema, artifact, and
|
||||||
rendering operations. `Prepare` discards the plan after returning its public
|
rendering operations. `Prepare` discards the plan after returning its public
|
||||||
metadata. `Run` retains the plan through initial and repaired-output validation
|
metadata. `Run` retains the plan through initial and repaired-output validation
|
||||||
and discards it when the operation ends. Prepared execution stores the same
|
and discards it when the operation ends. Prepared execution transfers the
|
||||||
kind of plan only in its private payload.
|
completed preparation's owned message, input-hash, and copied-schema values
|
||||||
|
into its private payload, while deep-copying mutable provider extras and
|
||||||
|
creating an independent details snapshot. Claim completion or discard clears
|
||||||
|
the execution payload without removing the details snapshot.
|
||||||
|
|
||||||
Pointer-based numeric overrides preserve an explicit zero. Invalid negative or
|
Pointer-based numeric overrides preserve an explicit zero. Invalid negative or
|
||||||
out-of-range values fail as invalid requests. Endpoint overrides do not change
|
out-of-range values fail as invalid requests. Endpoint overrides do not change
|
||||||
|
|||||||
@@ -21,6 +21,11 @@ profile inheritance. It retains ConfigKit's result privately for explicit
|
|||||||
inspection, not for per-run resolution. The runtime/domain packages do not
|
inspection, not for per-run resolution. The runtime/domain packages do not
|
||||||
depend on ConfigKit.
|
depend on ConfigKit.
|
||||||
|
|
||||||
|
After prompt materialization and effective-profile snapshot construction, the
|
||||||
|
adapter captures lexically ordered, identity-only inventories from the final
|
||||||
|
prompt and indexed-profile maps. Its listing methods return caller-owned copies
|
||||||
|
of those inventories without consulting ConfigKit or copying definitions.
|
||||||
|
|
||||||
The root `DirectoryInputs` and `FilesystemInputs` helpers discover sorted YAML
|
The root `DirectoryInputs` and `FilesystemInputs` helpers discover sorted YAML
|
||||||
files under an explicit root, provide resource context, and leave imports
|
files under an explicit root, provide resource context, and leave imports
|
||||||
disabled. Each returned slice belongs to one strict-composition group; file
|
disabled. Each returned slice belongs to one strict-composition group; file
|
||||||
@@ -35,21 +40,68 @@ The configuration adapter indexes each prompt by exact ID and version,
|
|||||||
materializes file-backed message content using the exact effective field's
|
materializes file-backed message content using the exact effective field's
|
||||||
origin and resource context, and checks static message/session template syntax
|
origin and resource context, and checks static message/session template syntax
|
||||||
at construction. It stores captured content, compiled templates, declared
|
at construction. It stores captured content, compiled templates, declared
|
||||||
input metadata, and a definition hash. It closes content-file handles before
|
input metadata, and a definition hash. Alongside the unchanged hashed
|
||||||
|
definition, it retains content-type and description presence for each declared
|
||||||
|
input and provides an internal requirement snapshot with those declarations
|
||||||
|
and the captured parses. Metadata values come from the final normalized
|
||||||
|
declaration; absent and null fields remain distinguishable from explicit
|
||||||
|
strings, including empty strings. It closes content-file handles before
|
||||||
returning and retains no source capability for public inspection.
|
returning and retains no source capability for public inspection.
|
||||||
|
|
||||||
Role and cache-control normalization is applied to execution-owned message
|
Role and cache-control normalization is applied to execution-owned message
|
||||||
copies. The retained ConfigKit configuration keeps effective input message
|
copies. The retained ConfigKit configuration keeps effective input message
|
||||||
metadata and its resource origins: recording a sequence-wide normalization in
|
metadata and its resource origins: recording a sequence-wide normalization in
|
||||||
ConfigKit v0.5.0 would replace origins of unchanged content-file fields.
|
ConfigKit v0.6.0 would replace origins of unchanged content-file fields.
|
||||||
|
|
||||||
The runner selects from this index for preparation and `InspectPrompt`.
|
The runner selects full definitions from this index for preparation and uses
|
||||||
Neither path rescans configuration or reopens message files. `InspectPrompt`
|
captured prompt metadata for `InspectPrompt` and `InspectRequirements`; the
|
||||||
|
internal requirement snapshot lookup selects by exact prompt ID and version.
|
||||||
|
These paths do not rescan configuration or reopen message files. `InspectPrompt`
|
||||||
returns declared metadata without rendering or resolving profile, input, or
|
returns declared metadata without rendering or resolving profile, input, or
|
||||||
schema resources. Runtime rendering still checks request variables and input
|
schema resources, and its public shape and prompt hashes do not include the
|
||||||
helpers. The [prompt adapter tests](../../internal/configuration/prompts_test.go)
|
captured presence state. Runtime rendering still checks request variables and
|
||||||
|
input helpers. The [prompt adapter tests](../../internal/configuration/prompts_test.go)
|
||||||
cover resource origins, context containment, capture, and selection.
|
cover resource origins, context containment, capture, and selection.
|
||||||
|
|
||||||
|
The configuration adapter shares exact ID/version selection across its
|
||||||
|
internal prompt lookups. The execution snapshot lookup returns an owned prompt
|
||||||
|
definition and cached hash; metadata inspection returns owned input metadata
|
||||||
|
without copying execution templates. Profile selection likewise has an
|
||||||
|
identity-only lookup for consumers that do not need the execution target,
|
||||||
|
while the full profile lookup continues to return an owned target and deep copy
|
||||||
|
of provider extras. These lookups use captured indexes and do not query
|
||||||
|
ConfigKit again.
|
||||||
|
|
||||||
|
The `internal/prompt` requirement walker uses the captured parse trees and
|
||||||
|
declaration metadata without executing templates or reading artifacts. It
|
||||||
|
collects literal `input` names, direct root caller keys, and literal-key
|
||||||
|
`index` lookups. For `if` and `with`, it classifies branches when their
|
||||||
|
conditions are simple supported values: direct values, literals, negation, or
|
||||||
|
two-string equality/inequality. It uses Go template truthiness, so a supplied
|
||||||
|
nonempty string such as `false` is true. Missing decision values and
|
||||||
|
content-dependent conditions leave dependency-bearing branches conditional
|
||||||
|
and report incomplete discovery; unresolved conditions with no branch
|
||||||
|
dependencies do not by themselves make discovery incomplete. Recognizable
|
||||||
|
references on known skipped paths remain optional inventory entries with an
|
||||||
|
inactive-use explanation, unless another applicable use establishes a
|
||||||
|
stronger classification. Skipped expressions are scanned only for
|
||||||
|
syntactically recognizable names and are not evaluated. A direct caller value
|
||||||
|
or literal-key lookup can also supply an input name, including through simple
|
||||||
|
parentheses or a direct pipeline to `input`; names are used exactly as
|
||||||
|
supplied. Unresolved names and unsupported dependency-bearing expressions
|
||||||
|
produce structured limitations while retaining recognizable references.
|
||||||
|
Where an unresolved name or scope could affect existing optional entries, those
|
||||||
|
entries are reported as conditional rather than inactive-only or definitively
|
||||||
|
optional. Direct root-map `with` forms retain caller-map identity. Known scalar
|
||||||
|
values remain separate from unknown scopes; accesses through untracked local
|
||||||
|
maps report incomplete discovery rather than silently losing caller keys.
|
||||||
|
Unsupported name expressions reuse value analysis's conservative syntax scan
|
||||||
|
instead of recursively rescanning the same subtree.
|
||||||
|
Invoked named templates are scanned conservatively, and
|
||||||
|
dependency-bearing range scopes make discovery incomplete. Uncalled definitions
|
||||||
|
and bypassed session templates are not traversed. This internal walker does not
|
||||||
|
change rendering behavior.
|
||||||
|
|
||||||
## Profiles And Built-Ins
|
## Profiles And Built-Ins
|
||||||
|
|
||||||
`internal/catalog` selects the released keyed OpenRouter and Rakestrawhome
|
`internal/catalog` selects the released keyed OpenRouter and Rakestrawhome
|
||||||
@@ -66,10 +118,16 @@ snapshots with backend defaults and presence/suppression state. A profile can
|
|||||||
remain partial until runtime overrides complete it. `InspectProfile` selects
|
remain partial until runtime overrides complete it. `InspectProfile` selects
|
||||||
that captured target without prompt, artifact, schema, credential lookup, or
|
that captured target without prompt, artifact, schema, credential lookup, or
|
||||||
provider work. Preparation applies runtime overrides and then checks target
|
provider work. Preparation applies runtime overrides and then checks target
|
||||||
completeness. The [target tests](../../internal/configuration/targets_test.go)
|
completeness. Identity-only profile selection can identify a partial captured
|
||||||
cover effective target interpretation, while the [catalog tests](../../internal/catalog/catalog_test.go)
|
profile without copying its target or requiring readiness. The
|
||||||
|
[target tests](../../internal/configuration/targets_test.go) cover effective
|
||||||
|
target interpretation, while the [catalog tests](../../internal/catalog/catalog_test.go)
|
||||||
cover selected released assets.
|
cover selected released assets.
|
||||||
|
|
||||||
|
The internal engine delegates listing through a narrow identity-inventory
|
||||||
|
interface directly to the configuration snapshot. The execution runner has no
|
||||||
|
listing responsibility.
|
||||||
|
|
||||||
## Ordinary Artifacts
|
## Ordinary Artifacts
|
||||||
|
|
||||||
`internal/artifact` accepts explicitly typed inline references even when their
|
`internal/artifact` accepts explicitly typed inline references even when their
|
||||||
@@ -79,10 +137,15 @@ and after opening the file. It copies content into an artifact, records
|
|||||||
metadata and an opaque content-equality value, and applies a content-type
|
metadata and an opaque content-equality value, and applies a content-type
|
||||||
fallback.
|
fallback.
|
||||||
|
|
||||||
Regular files are read synchronously in bounded chunks. Cancellation is
|
Regular files are read synchronously in bounded chunks. The opened file's size
|
||||||
checked before opening, before and after every read, and before publishing the
|
is only an initial-capacity hint, capped at 1 MiB; reads continue to EOF, and
|
||||||
artifact, so a canceled read never publishes partial content. The ordinary
|
the returned body determines artifact size and hash. If an overreported hint
|
||||||
reader does not detach file reads into background goroutines.
|
leaves unused capacity greater than the actual body length, the completed body
|
||||||
|
is copied to release the excess backing storage, including when the body is
|
||||||
|
empty. Cancellation is checked before opening, before and after every read,
|
||||||
|
and before publishing the artifact, so a
|
||||||
|
canceled read never publishes partial content. The ordinary reader does not
|
||||||
|
detach file reads into background goroutines.
|
||||||
|
|
||||||
This ordinary reader does not implement an inbound HTTP security boundary. In
|
This ordinary reader does not implement an inbound HTTP security boundary. In
|
||||||
particular, it does not constrain files to an application root or impose an
|
particular, it does not constrain files to an application root or impose an
|
||||||
@@ -106,7 +169,9 @@ between artifact conversion chunks, around each message, and before publishing
|
|||||||
the complete prompt. It cannot interrupt template work already in progress and
|
the complete prompt. It cannot interrupt template work already in progress and
|
||||||
never publishes a partial prompt after observing cancellation. It validates and
|
never publishes a partial prompt after observing cancellation. It validates and
|
||||||
canonicalizes message roles before carrying roles, session IDs, and cache
|
canonicalizes message roles before carrying roles, session IDs, and cache
|
||||||
control into the rendered prompt. The
|
control into the rendered prompt. Message and session templates execute into
|
||||||
|
per-call string builders, so their rendered strings do not require a final
|
||||||
|
buffer-to-string copy. The
|
||||||
[renderer tests](../../internal/prompt/renderer_test.go) own rendering behavior.
|
[renderer tests](../../internal/prompt/renderer_test.go) own rendering behavior.
|
||||||
|
|
||||||
## Schemas And Output Validation
|
## Schemas And Output Validation
|
||||||
@@ -114,7 +179,8 @@ control into the rendered prompt. The
|
|||||||
`internal/validate` provides validators backed by an operating-system
|
`internal/validate` provides validators backed by an operating-system
|
||||||
filesystem or an `fs.FS`. Invalid generated content is returned as a validation
|
filesystem or an `fs.FS`. Invalid generated content is returned as a validation
|
||||||
result; inability to load, register, or compile a schema is an operational
|
result; inability to load, register, or compile a schema is an operational
|
||||||
error.
|
error. Basic validation checks body emptiness on the byte slice directly
|
||||||
|
instead of first making a response-sized string copy.
|
||||||
|
|
||||||
Every preparation operation creates one operation-local validation plan. None,
|
Every preparation operation creates one operation-local validation plan. None,
|
||||||
basic, and JSON modes retain the effective output contract without source
|
basic, and JSON modes retain the effective output contract without source
|
||||||
|
|||||||
@@ -6,29 +6,37 @@ Material: policy documents under `docs/policy/` and the Policy Reading Guide in
|
|||||||
`docs/development.md`, adapted for this repository; root `AGENTS.md`, copied
|
`docs/development.md`, adapted for this repository; root `AGENTS.md`, copied
|
||||||
without modification from upstream `agents/AGENTS.md`.
|
without modification from upstream `agents/AGENTS.md`.
|
||||||
|
|
||||||
|
The release policy uses upstream commit
|
||||||
|
`e833faddf120c789a9b45348edaf75720de16348`; the remaining material uses
|
||||||
|
`d84a38ebab26f048cafbe59a2246700c35d29843`.
|
||||||
|
|
||||||
Copyright (C) 2026 Eric Rakestraw
|
Copyright (C) 2026 Eric Rakestraw
|
||||||
|
|
||||||
SPDX-License-Identifier: CC-BY-SA-4.0
|
SPDX-License-Identifier: CC-BY-SA-4.0
|
||||||
|
|
||||||
License and notices: [CC BY-SA 4.0](licenses/CC-BY-SA-4.0.txt) |
|
License and notices: [CC BY-SA 4.0](licenses/CC-BY-SA-4.0.txt) |
|
||||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/COPYRIGHT) |
|
[COPYRIGHT](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/COPYRIGHT) |
|
||||||
[LICENSE](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/LICENSE).
|
[LICENSE](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/LICENSE) |
|
||||||
|
[COPYRIGHT](https://gitea.maximumdirect.net/eric/policy/src/commit/e833faddf120c789a9b45348edaf75720de16348/COPYRIGHT) |
|
||||||
|
[LICENSE](https://gitea.maximumdirect.net/eric/policy/src/commit/e833faddf120c789a9b45348edaf75720de16348/LICENSE).
|
||||||
|
|
||||||
[Source](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843).
|
[Source](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843).
|
||||||
|
|
||||||
|
[Source](https://gitea.maximumdirect.net/eric/policy/src/commit/e833faddf120c789a9b45348edaf75720de16348).
|
||||||
|
|
||||||
## ConfigKit
|
## ConfigKit
|
||||||
|
|
||||||
Selected module: `gitea.maximumdirect.net/eric/ConfigKit` at `v0.5.0`.
|
Selected module: `gitea.maximumdirect.net/eric/ConfigKit` at `v0.6.0`.
|
||||||
|
|
||||||
Copyright (C) 2026 Eric Rakestraw
|
Copyright (C) 2026 Eric Rakestraw
|
||||||
|
|
||||||
SPDX-License-Identifier: LGPL-3.0-or-later
|
SPDX-License-Identifier: LGPL-3.0-or-later
|
||||||
|
|
||||||
License and notices: [LGPL 3.0](../../LICENSE) |
|
License and notices: [LGPL 3.0](../../LICENSE) |
|
||||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/ConfigKit/src/commit/dc09959cca4e105f49ec316c6d4ba585bd2340bd/COPYRIGHT) |
|
[COPYRIGHT](https://gitea.maximumdirect.net/eric/ConfigKit/src/commit/1a885934c28733d9a5dca261fd6da0e2a4279285/COPYRIGHT) |
|
||||||
[LICENSE](https://gitea.maximumdirect.net/eric/ConfigKit/src/commit/dc09959cca4e105f49ec316c6d4ba585bd2340bd/LICENSE).
|
[LICENSE](https://gitea.maximumdirect.net/eric/ConfigKit/src/commit/1a885934c28733d9a5dca261fd6da0e2a4279285/LICENSE).
|
||||||
|
|
||||||
[Source](https://gitea.maximumdirect.net/eric/ConfigKit/src/commit/dc09959cca4e105f49ec316c6d4ba585bd2340bd).
|
[Source](https://gitea.maximumdirect.net/eric/ConfigKit/src/commit/1a885934c28733d9a5dca261fd6da0e2a4279285).
|
||||||
|
|
||||||
## Promptkit OpenRouter Catalog v3
|
## Promptkit OpenRouter Catalog v3
|
||||||
|
|
||||||
|
|||||||
@@ -1,10 +1,10 @@
|
|||||||
# Release Policy
|
# Release Policy
|
||||||
|
|
||||||
**Base Policy Version:** 1.4
|
**Base Policy Version:** 1.6
|
||||||
|
|
||||||
**Last Updated**: 2026-09-27
|
**Last Updated**: 2026-10-08
|
||||||
|
|
||||||
**Upstream Source:** [release.md](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/docs/policy/release.md)
|
**Upstream Source:** [release.md](https://gitea.maximumdirect.net/eric/policy/src/commit/e833faddf120c789a9b45348edaf75720de16348/docs/policy/release.md)
|
||||||
|
|
||||||
## Purpose
|
## Purpose
|
||||||
|
|
||||||
@@ -12,6 +12,9 @@ This policy defines a proportionate workflow for validating and publishing an
|
|||||||
intended source revision under a fresh version. Releases should require little
|
intended source revision under a fresh version. Releases should require little
|
||||||
routine intervention, avoid repeated work, and preserve published contents.
|
routine intervention, avoid repeated work, and preserve published contents.
|
||||||
|
|
||||||
|
Adopt stronger verification or reproducibility requirements through an explicit
|
||||||
|
project-specific decision justified by the intended audience and support needs.
|
||||||
|
|
||||||
The repository-specific commands belong in `docs/release.md`. Historical release
|
The repository-specific commands belong in `docs/release.md`. Historical release
|
||||||
summaries belong in `docs/releases/`. The procedure must agree with the adopted
|
summaries belong in `docs/releases/`. The procedure must agree with the adopted
|
||||||
policy and link to it rather than introduce additional release requirements.
|
policy and link to it rather than introduce additional release requirements.
|
||||||
@@ -59,7 +62,8 @@ requires a deliberate update to the policy and procedure where needed.
|
|||||||
For releases at or after `v1.0.0`:
|
For releases at or after `v1.0.0`:
|
||||||
|
|
||||||
- a major release may make intentional incompatible changes;
|
- a major release may make intentional incompatible changes;
|
||||||
- a minor release may add backward-compatible behavior; and
|
- a minor release may add backward-compatible behavior or raise the Go consumer
|
||||||
|
minimum as described below; and
|
||||||
- a patch release may make backward-compatible fixes.
|
- a patch release may make backward-compatible fixes.
|
||||||
|
|
||||||
Before `v1.0.0`, a minor release may change a documented public, operational,
|
Before `v1.0.0`, a minor release may change a documented public, operational,
|
||||||
@@ -71,14 +75,44 @@ Additional compatibility promises and deliberate exceptions belong in the
|
|||||||
adopted policy or canonical contract documentation. Release notes explain
|
adopted policy or canonical contract documentation. Release notes explain
|
||||||
changes; they do not redefine current compatibility contracts.
|
changes; they do not redefine current compatibility contracts.
|
||||||
|
|
||||||
## Release Documentation
|
## Consumer Minimum And Release Toolchain
|
||||||
|
|
||||||
|
Distinguish the minimum toolchain supported for consumers from the toolchain
|
||||||
|
maintainers use to build and validate a release. Record the consumer minimum in
|
||||||
|
the language's standard metadata and explain its support meaning in canonical
|
||||||
|
documentation. Define the selected release toolchain and its setup commands in
|
||||||
|
`docs/release.md`. Toolchain selection and upgrade cadence are repository-specific
|
||||||
|
choices.
|
||||||
|
|
||||||
|
For Go repositories, normally declare the initial stable release of the selected
|
||||||
|
series as the consumer minimum, such as `go 1.26.0`. Use a higher patch minimum
|
||||||
|
when a dependency or an identified correctness fix requires it, and document
|
||||||
|
the reason.
|
||||||
|
|
||||||
|
For release-policy purposes, successful build and release validation on the
|
||||||
|
selected Go patch release are sufficient to presume support across that Go
|
||||||
|
series, subject to the declared consumer minimum and concrete evidence of
|
||||||
|
incompatibility. Investigate specific compatibility concerns as they arise.
|
||||||
|
|
||||||
|
Use the selected release toolchain for candidate validation and release builds,
|
||||||
|
including CI jobs performing those tasks. A newer validation patch version alone
|
||||||
|
leaves the consumer minimum unchanged.
|
||||||
|
|
||||||
|
A minor release may raise the Go consumer minimum, including from one Go series
|
||||||
|
to the next, both before and after `v1.0.0`. This is an explicit exception to the
|
||||||
|
backward-compatibility requirement for minor releases. Patch releases preserve
|
||||||
|
the consumer minimum. Explain an increase's impact and required consumer action
|
||||||
|
in the release note. Other incompatible changes remain subject to the versioning
|
||||||
|
rules above.
|
||||||
|
|
||||||
|
## Release Documentation And Preparation
|
||||||
|
|
||||||
Every new release must include `docs/releases/<version>.md` in the tagged
|
Every new release must include `docs/releases/<version>.md` in the tagged
|
||||||
source tree. Use the complete version tag as the filename and begin with a
|
source tree. Use the complete version tag as the filename and begin with a
|
||||||
title containing the product name and that exact version.
|
title containing the product name and that exact version.
|
||||||
|
|
||||||
Include a short summary and any relevant compatibility impact or upgrade
|
Include a short summary and any relevant compatibility impact or upgrade
|
||||||
instructions. Use the Promptkit section structure below and keep the note
|
instructions. Choose sections suited to the release and keep the note
|
||||||
proportionate to the change.
|
proportionate to the change.
|
||||||
|
|
||||||
Keep exact current contracts in their canonical documents and update affected
|
Keep exact current contracts in their canonical documents and update affected
|
||||||
@@ -186,7 +220,8 @@ release without replacing its contents or reusing its version.
|
|||||||
Maintain `docs/release.md` as a short, executable maintainer runbook. It must
|
Maintain `docs/release.md` as a short, executable maintainer runbook. It must
|
||||||
identify:
|
identify:
|
||||||
|
|
||||||
1. prerequisites, primary branch, and authoritative remote;
|
1. prerequisites, selected release toolchain, primary branch, and authoritative
|
||||||
|
remote;
|
||||||
2. version and release-note preparation;
|
2. version and release-note preparation;
|
||||||
3. one publication command with its selected local checks and automated guards;
|
3. one publication command with its selected local checks and automated guards;
|
||||||
4. declared deliverables, required build targets, and automation responsibilities;
|
4. declared deliverables, required build targets, and automation responsibilities;
|
||||||
@@ -210,20 +245,39 @@ annotated stable SemVer tags. It has no binaries, hosted download pages, or
|
|||||||
asynchronous deliverables. The source commit is pushed through main before the
|
asynchronous deliverables. The source commit is pushed through main before the
|
||||||
tag. The [release procedure](../release.md) owns commands and recovery.
|
tag. The [release procedure](../release.md) owns commands and recovery.
|
||||||
|
|
||||||
The existing publication script requires Summary, Compatibility, Upgrade, and
|
Release preparation uses the latest Go installation available in the development
|
||||||
Changes headings beneath a title of `# Promptkit vMAJOR.MINOR.PATCH`. State
|
environment to build the maintainer release executable and validate the
|
||||||
“No action required” or “None” where appropriate. This retained local structure
|
candidate, including the consumer examples. Keep that same local toolchain
|
||||||
matches the automated release-note guard; changing it is separate work.
|
active throughout those operations.
|
||||||
|
|
||||||
The [publication script](../../scripts/release.sh) runs the
|
For a minor release, advance Promptkit's consumer minimum to the initial stable
|
||||||
[local checks](../../scripts/release-check.sh) once for the final committed
|
release of that Go series, retaining a higher patch minimum when required by a
|
||||||
candidate: race-enabled tests, vet, three offline consumer examples, tracked
|
dependency or an identified correctness fix. Patch releases preserve the
|
||||||
Go formatting, Markdown links, module/workspace hygiene, whitespace, and a
|
existing consumer minimum. Commit any module update before validating the final
|
||||||
credential-pattern scan. These protect concurrency, assembled library use,
|
candidate. The [release procedure](../release.md#prepare-the-candidate) owns
|
||||||
source distribution, and documentation integrity. A separate ordinary test run
|
setup commands, and the consumer guide explains
|
||||||
or standalone build is not an additional release gate. Candidate, tag, and
|
[Go compatibility](../consumers/pkg-promptkit.md#go-compatibility).
|
||||||
synchronized-main checks remain automated. No CI or module-availability wait
|
|
||||||
is required. Licensing footer URLs are preparation work, not a network gate.
|
Use a title containing `Promptkit` and the exact version, followed by a short
|
||||||
|
summary and any relevant compatibility impact or upgrade instructions. The
|
||||||
|
note is committed with the candidate and supplies the annotated tag message.
|
||||||
|
No fixed set of section headings is required.
|
||||||
|
|
||||||
|
The [publication command](../../scripts/release.sh) delegates to the released
|
||||||
|
ReleaseKit tool selected in `go.mod`. The committed
|
||||||
|
[profile](../../releasekit.json) selects the race-enabled Go suite, full vet,
|
||||||
|
committed Go formatting, local Markdown links, workspace and filesystem
|
||||||
|
replacement inspection, and the offline semantic checks for the three
|
||||||
|
maintained consumer examples. These are the selected checks for the final
|
||||||
|
candidate; a separate ordinary test run or standalone build is not an
|
||||||
|
additional release gate. The upstream tool owns candidate, remote, and tag
|
||||||
|
publication checks. It returns native status 0 for confirmed publication, 1
|
||||||
|
for a known validation or publication failure, 2 for invalid input or profile,
|
||||||
|
and 3 when the push outcome is unknown. For status 3, resolve the exact
|
||||||
|
captured publication endpoint before retrying; the repository-specific
|
||||||
|
[procedure](../release.md#failure-recovery) links to the upstream recovery
|
||||||
|
instructions. Do not wait for CI or Go module availability. Licensing footer
|
||||||
|
URLs are preparation work, not a network gate.
|
||||||
|
|
||||||
## Catalog Profile Compatibility
|
## Catalog Profile Compatibility
|
||||||
|
|
||||||
|
|||||||
@@ -58,17 +58,24 @@ and repository hygiene.
|
|||||||
Introducing hosted CI later would supplement, not silently redefine, this
|
Introducing hosted CI later would supplement, not silently redefine, this
|
||||||
documented validation model.
|
documented validation model.
|
||||||
|
|
||||||
Full release-candidate validation includes race-enabled package tests, explicit
|
The selected ReleaseKit profile runs the race-enabled package suite and full
|
||||||
static analysis, and execution with automated semantic checks of the three maintained
|
vet, checks committed Go formatting, local Markdown links, and module
|
||||||
offline consumer examples. The test run builds the packages; the examples
|
distribution hygiene, and executes semantic assertions for the three
|
||||||
also exercise ordinary compilation. Separate ordinary tests and a standalone
|
maintained offline consumer examples. ReleaseKit owns this shared validation
|
||||||
build are not additional release gates. Ordinary tests remain useful during
|
machinery and publication workflow. The test run builds the packages; the
|
||||||
development. The preparation example protects assembled preparation and
|
examples also exercise ordinary compilation. Separate ordinary tests and a
|
||||||
|
standalone build are not additional release gates. Ordinary tests remain useful
|
||||||
|
during development. The preparation example protects assembled preparation and
|
||||||
inspection behavior. The execution example separately protects assembled
|
inspection behavior. The execution example separately protects assembled
|
||||||
`Run`, injected-client, validation, usage, and result behavior. The observation
|
`Run`, injected-client, validation, usage, and result behavior. The observation
|
||||||
example protects assembled built-in retries, structured proposal and assessment
|
example protects assembled built-in retries, structured proposal and assessment
|
||||||
runs, and application-owned interpretation of metadata.
|
runs, and application-owned interpretation of metadata.
|
||||||
|
|
||||||
|
The standalone `python3 scripts/test-release.py` command exercises Promptkit's
|
||||||
|
ReleaseKit wiring against disposable local repositories. It is focused
|
||||||
|
development validation when release tooling changes, not part of the default
|
||||||
|
Go test suite or a recurring release check.
|
||||||
|
|
||||||
Tests in the default suite must be deterministic, offline, and independent of
|
Tests in the default suite must be deterministic, offline, and independent of
|
||||||
real credentials. They must not invoke paid APIs, use live network
|
real credentials. They must not invoke paid APIs, use live network
|
||||||
dependencies, or depend on mutable external services. Tests that require live
|
dependencies, or depend on mutable external services. Tests that require live
|
||||||
|
|||||||
157
docs/release.md
157
docs/release.md
@@ -1,64 +1,101 @@
|
|||||||
# Release Procedure
|
# Release Procedure
|
||||||
|
|
||||||
This runbook implements the [release policy](policy/release.md). A successful
|
This runbook implements the [release policy](policy/release.md). Promptkit
|
||||||
push of the selected annotated tag to origin completes the release. Never wait
|
publishes source-only Go module tags from `main` to `origin`. A successful push
|
||||||
for CI to finish or succeed.
|
of the selected annotated tag, after its source commit is available on `main`,
|
||||||
|
completes the release. Do not wait for CI or Go module availability.
|
||||||
|
|
||||||
## Prepare The Candidate
|
## Prepare The Candidate
|
||||||
|
|
||||||
Required tools are Git, Go with race-detector support, POSIX shell, and Python
|
Required tools are Git, Go with race-detector support, POSIX shell, and Python
|
||||||
3. Publication requires push access to origin. Work from the repository root.
|
3. Work from the repository root with push access to `origin`.
|
||||||
|
|
||||||
Select an unused version and create `docs/releases/<version>.md` using the
|
Use the latest Go installation available locally for the release tool, the
|
||||||
[required structure](policy/release.md#release-documentation). Describe
|
candidate validation, and the maintained examples. Make it the active `go`
|
||||||
compatibility impact and consumer actions and update affected canonical docs.
|
command and record its version:
|
||||||
For catalog updates, record selected catalog versions and any consumer action,
|
|
||||||
linking to the catalog policy rather than repeating its contracts.
|
|
||||||
|
|
||||||
Follow the [OSS policy](policy/oss.md) when preparing licensing and attribution
|
```sh
|
||||||
changes. Each new note must end with the following navigation, replacing
|
export GOTOOLCHAIN=local
|
||||||
`vMAJOR.MINOR.PATCH` with the selected release tag:
|
go version
|
||||||
|
|
||||||
```markdown
|
|
||||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit/src/tag/vMAJOR.MINOR.PATCH/COPYRIGHT) | [LICENSE](https://gitea.maximumdirect.net/eric/promptkit/src/tag/vMAJOR.MINOR.PATCH/LICENSE) | [ATTRIBUTION](https://gitea.maximumdirect.net/eric/promptkit/src/tag/vMAJOR.MINOR.PATCH/docs/oss/attribution.md)
|
|
||||||
```
|
```
|
||||||
|
|
||||||
Check these file paths against the candidate; a future tag need not resolve
|
For a minor release, advance Promptkit's consumer minimum to the initial stable
|
||||||
remotely during preparation. The annotated tag derives from this same note,
|
release of that Go series before committing the candidate:
|
||||||
including its footer. Promptkit publishes no separate hosted release body or
|
|
||||||
download page. If such a channel is deliberately introduced, reuse the note
|
|
||||||
and preserve its footer. No extra network checks, artifacts, or approval gates
|
|
||||||
are introduced, and published notes are not retrofitted.
|
|
||||||
|
|
||||||
Retire completed roadmaps, preserving planned or deferred work and repairing
|
```sh
|
||||||
links. This is housekeeping, not a separate approval gate. Review changes
|
release_go_version=$(go env GOVERSION)
|
||||||
through the ordinary development workflow, commit them, and push main to
|
release_go_series=${release_go_version#go}
|
||||||
origin. No repeated release-specific diff review is required.
|
release_go_series=${release_go_series%.*}
|
||||||
|
go mod edit "-go=${release_go_series}.0"
|
||||||
|
```
|
||||||
|
|
||||||
|
For example, Go 1.26.8 normally results in `go 1.26.0`. Keep a higher patch
|
||||||
|
minimum if a dependency or identified correctness fix requires it. For a patch
|
||||||
|
release, leave the consumer minimum unchanged; using a newer validation
|
||||||
|
toolchain does not itself raise that minimum. If the candidate needs a higher
|
||||||
|
minimum, prepare a minor release instead. See the
|
||||||
|
[consumer-minimum policy](policy/release.md#consumer-minimum-and-release-toolchain).
|
||||||
|
|
||||||
|
Prepare module caches before validation; this may update `go.sum` and belongs
|
||||||
|
before the candidate commit:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
GOTOOLCHAIN=local go mod download
|
||||||
|
```
|
||||||
|
|
||||||
|
Choose an unused stable version and create `docs/releases/<version>.md` with a
|
||||||
|
title containing `Promptkit` and that exact version, a short summary, and any
|
||||||
|
relevant compatibility impact or upgrade instructions. No fixed section
|
||||||
|
headings are required. Describe a raised Go minimum and its compatibility
|
||||||
|
impact under the [versioning policy](policy/release.md#versioning-and-compatibility).
|
||||||
|
End each new note with the licensing footer below, using the selected version:
|
||||||
|
|
||||||
|
```markdown
|
||||||
|
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit/src/tag/vX.Y.Z/COPYRIGHT) | [LICENSE](https://gitea.maximumdirect.net/eric/promptkit/src/tag/vX.Y.Z/LICENSE) | [ATTRIBUTION](https://gitea.maximumdirect.net/eric/promptkit/src/tag/vX.Y.Z/docs/oss/attribution.md)
|
||||||
|
```
|
||||||
|
|
||||||
|
Check the footer paths against the candidate; the future tag need not resolve
|
||||||
|
remotely during preparation. Follow the [OSS policy](policy/oss.md) for these
|
||||||
|
links. Promptkit publishes no binaries, hosted download page, or other
|
||||||
|
asynchronous deliverables.
|
||||||
|
|
||||||
|
Retire completed roadmaps while preserving deferred work and repairing incoming
|
||||||
|
links. Review the intended changes through the ordinary development workflow,
|
||||||
|
commit the candidate on `main`, and push `main` to `origin` before publication.
|
||||||
|
Preparation may update module metadata; the release bootstrap and validation
|
||||||
|
preserve the committed candidate.
|
||||||
|
|
||||||
## Validate And Publish
|
## Validate And Publish
|
||||||
|
|
||||||
Run the following with the chosen unused version (replace the placeholder):
|
For an optional local diagnostic of a committed candidate, run:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
./scripts/releasekit.sh check vMAJOR.MINOR.PATCH
|
||||||
|
```
|
||||||
|
|
||||||
|
This is not a required step before publication. The upstream
|
||||||
|
[candidate-check contract](https://gitea.maximumdirect.net/eric/ReleaseKit/src/tag/v0.3.0/docs/consumers/check.md)
|
||||||
|
describes its requirements.
|
||||||
|
|
||||||
|
Run the single publication command with the chosen version:
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
./scripts/release.sh vMAJOR.MINOR.PATCH
|
./scripts/release.sh vMAJOR.MINOR.PATCH
|
||||||
```
|
```
|
||||||
|
|
||||||
The [publication script](../scripts/release.sh) checks a clean, synchronized
|
The familiar wrapper delegates to the native ReleaseKit executable selected in
|
||||||
main candidate, the version and unused tag, and the release note's title and
|
`go.mod`. The committed [profile](../releasekit.json) selects the race-enabled
|
||||||
required headings. It runs the [local checks](../scripts/release-check.sh)
|
Go suite, full vet, committed Go formatting, local Markdown links, workspace
|
||||||
once, rechecks candidate identity and publication state, generates the
|
and filesystem replacement inspection, and semantic checks for the three
|
||||||
annotation from the committed note plus the validated commit, and pushes only
|
maintained offline consumer examples. ReleaseKit also verifies the committed
|
||||||
the selected annotated tag.
|
candidate and publication state, then publishes only the selected tag. Its
|
||||||
|
checks run once for this candidate; do not repeat them as a separate sign-off.
|
||||||
|
|
||||||
There is no separate tag-message authoring, example-output inspection,
|
The native command returns status 0 for confirmed publication, 1 for a known
|
||||||
metadata inspection, or human sign-off. Compatibility and release-note content
|
validation or publication failure, 2 for invalid input or profile, and 3 when
|
||||||
remain ordinary editorial responsibilities. Development validation is useful
|
the push outcome remains unknown. Successful publication needs no remote
|
||||||
but does not require a second full run before invoking the publication script.
|
read-back, CI wait, or installation check. Provider probes and inference calls
|
||||||
|
are not part of validation.
|
||||||
The script stops successfully after the tag push. CI completion, CI success,
|
|
||||||
remote read-back, and Go module availability are not release completion gates.
|
|
||||||
Do not wait or poll for them. Provider probes and inference calls are never
|
|
||||||
part of release validation.
|
|
||||||
|
|
||||||
## Catalog Profile Compatibility
|
## Catalog Profile Compatibility
|
||||||
|
|
||||||
@@ -67,28 +104,32 @@ catalog compatibility boundary. This heading preserves older incoming links.
|
|||||||
|
|
||||||
## Failure Recovery
|
## Failure Recovery
|
||||||
|
|
||||||
If local checks fail, fix the candidate, commit and push changes, then rerun
|
For status 1, use the diagnostic to distinguish a validation failure from a
|
||||||
the publication command. It never automatically removes a tag.
|
known publication conflict. Correct and commit candidate defects, push `main`,
|
||||||
|
and rerun publication with an unused version when appropriate. Status 2 means
|
||||||
|
the invocation or profile must be corrected.
|
||||||
|
|
||||||
If a push fails or its result is ambiguous, set `RELEASE_VERSION` to the
|
For status 3, do not retry or remove the local tag until the remote outcome is
|
||||||
attempted version and inspect origin before retrying:
|
resolved. ReleaseKit reports the attempted version, tag object, and recovery
|
||||||
|
state, including the captured publication endpoint when available. Use that
|
||||||
|
endpoint—not merely the current `origin` URL—to query the exact tag; follow the
|
||||||
|
upstream
|
||||||
|
[failure-recovery procedure](https://gitea.maximumdirect.net/eric/ReleaseKit/src/tag/v0.3.0/docs/release.md#resolve-failures):
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
git ls-remote --tags origin "refs/tags/$RELEASE_VERSION" "refs/tags/$RELEASE_VERSION^{}"
|
releasekit_endpoint='COPY_CAPTURED_ENDPOINT'
|
||||||
git rev-parse "refs/tags/$RELEASE_VERSION" "refs/tags/$RELEASE_VERSION^{}"
|
releasekit_version='vX.Y.Z'
|
||||||
|
test "$(git ls-remote --get-url -- "$releasekit_endpoint")" = "$releasekit_endpoint" &&
|
||||||
|
git ls-remote --refs -- "$releasekit_endpoint" "refs/tags/$releasekit_version"
|
||||||
```
|
```
|
||||||
|
|
||||||
Matching remote tag and peeled commit values confirm publication, even if the
|
Compare the returned object ID with the reported tag object. An exact match
|
||||||
push command reported an error. If the remote query fails, publication status
|
confirms publication. A missing ref confirms absence only when the endpoint
|
||||||
is still unknown. Do not delete or recreate the local tag until absence on
|
identity check and query both succeed. If either check fails, the outcome
|
||||||
origin is confirmed. Once confirmed unpublished, an unsuccessful attempt's
|
remains unknown; correct the transport configuration and resolve it before
|
||||||
local tag can be removed and the publication command rerun. Never overwrite
|
retrying. Only after authoritative absence is confirmed may an unpublished
|
||||||
a published tag.
|
local tag be removed before correcting the candidate or retrying. Never move,
|
||||||
|
overwrite, or delete a published tag.
|
||||||
CI or module-resolution problems may be investigated separately; temporary
|
|
||||||
service, network, or authentication failures do not require another release.
|
|
||||||
Only an actual defect requiring changes to published contents warrants a new
|
|
||||||
appropriate version. See the [recovery policy](policy/release.md#immutability-and-failure-recovery).
|
|
||||||
|
|
||||||
## Policy Changes
|
## Policy Changes
|
||||||
|
|
||||||
|
|||||||
74
docs/releases/v0.12.0.md
Normal file
74
docs/releases/v0.12.0.md
Normal file
@@ -0,0 +1,74 @@
|
|||||||
|
# Promptkit v0.12.0
|
||||||
|
|
||||||
|
This release replaces Promptkit's configuration loader with ConfigKit and
|
||||||
|
migrates maintained catalogs to keyed `/v3` assets. It intentionally changes
|
||||||
|
the public construction API and configuration format.
|
||||||
|
|
||||||
|
## Summary
|
||||||
|
|
||||||
|
Backends, profiles, and versioned prompts now share one layered configuration
|
||||||
|
snapshot built when `NewEngine` is called. ConfigKit supplies composition,
|
||||||
|
inheritance, presence, and source history; Promptkit validates the final
|
||||||
|
effective definitions and captures prompt content and templates. Consumers can
|
||||||
|
inspect the captured configuration explicitly without triggering a reload.
|
||||||
|
|
||||||
|
## Compatibility
|
||||||
|
|
||||||
|
This is an incompatible minor release under Promptkit's pre-1.0
|
||||||
|
[versioning policy](../policy/release.md#versioning-and-compatibility). The
|
||||||
|
former source options, `Config.PromptDir`, `Config.ProfileDir`, and the old
|
||||||
|
profile/backend convenience values have been removed. Configuration documents
|
||||||
|
now use literal keys beneath `backends`, `profiles`, and `prompts`; embedded
|
||||||
|
`id`/`version` fields and `base_profile` are no longer accepted. Source groups
|
||||||
|
compose strictly within a group and overlay in the supplied order. Explicit
|
||||||
|
zero, false, empty, and null values can now differ from omission; maps overlay
|
||||||
|
recursively, and an empty map does not clear an inherited map.
|
||||||
|
|
||||||
|
All final definitions, including unused ones, are validated during engine
|
||||||
|
construction. Prompt text and static templates are captured then; an existing
|
||||||
|
engine does not automatically reload changed configuration or message files.
|
||||||
|
Request artifacts, schemas, credentials, and provider operations retain their
|
||||||
|
runtime boundaries. See the [format reference](../formats.md) for the current
|
||||||
|
configuration contract and the [consumer migration guide](../consumers/pkg-promptkit.md#migrate-from-the-previous-configuration-api)
|
||||||
|
for API and source-selection changes.
|
||||||
|
|
||||||
|
Maintained OpenRouter and Rakestrawhome catalogs are selected as `/v3` modules,
|
||||||
|
both at `v3.20260930.0`. Their asset format changed; consumers that read catalog
|
||||||
|
assets directly must adopt the keyed format. The maintained Promptkit profile
|
||||||
|
IDs remain available under the [catalog compatibility policy](../policy/release.md#catalog-profile-compatibility),
|
||||||
|
which does not promise fixed profile data.
|
||||||
|
|
||||||
|
## Upgrade
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go get gitea.maximumdirect.net/eric/promptkit@v0.12.0
|
||||||
|
```
|
||||||
|
|
||||||
|
Replace removed source and backend registration options with
|
||||||
|
`Config.Sources`, explicitly ordered ConfigKit underlays, and optionally a
|
||||||
|
highest-priority main input. Use `DirectoryInputs` or `FilesystemInputs` for
|
||||||
|
selected directories, `ProfileSource` and `BackendSource` for typed
|
||||||
|
programmatic values, or `configkit.Mapping` for explicit null and combined
|
||||||
|
documents. Move profile `base_profile` to `extends` and embedded identities to
|
||||||
|
mapping keys. Review zero/false/null values and any assumptions that replacing
|
||||||
|
a model or backend discards unrelated explicit settings. Rebuild the engine
|
||||||
|
when definitions or referenced message files change. The
|
||||||
|
[consumer guide](../consumers/pkg-promptkit.md#migrate-from-the-previous-configuration-api)
|
||||||
|
provides the migration details and maintained examples.
|
||||||
|
|
||||||
|
## Changes
|
||||||
|
|
||||||
|
- Add ConfigKit v0.5.0 as the configuration boundary and consume released
|
||||||
|
OpenRouter and Rakestrawhome `/v3` catalog modules.
|
||||||
|
- Add keyed backend, profile, and prompt definitions with final-value domain
|
||||||
|
validation, explicit presence/null behavior, and source-relative content
|
||||||
|
capture.
|
||||||
|
- Add `ConfigurationSources`, directory discovery helpers, programmatic
|
||||||
|
profile/backend source builders, and read-only configuration inspection.
|
||||||
|
- Preserve execution workflows while rendering captured templates, applying
|
||||||
|
backend defaults after inheritance, and honoring explicit suppression of
|
||||||
|
provider settings.
|
||||||
|
- Remove superseded loader, catalog preflight, and inheritance machinery;
|
||||||
|
update examples, format guidance, and internal documentation.
|
||||||
|
|
||||||
|
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.12.0/COPYRIGHT) | [LICENSE](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.12.0/LICENSE) | [ATTRIBUTION](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.12.0/docs/oss/attribution.md)
|
||||||
69
docs/releases/v0.13.0.md
Normal file
69
docs/releases/v0.13.0.md
Normal file
@@ -0,0 +1,69 @@
|
|||||||
|
# Promptkit v0.13.0
|
||||||
|
|
||||||
|
This release adds prompt requirement discovery and profile/prompt listing,
|
||||||
|
reduces avoidable configuration and execution allocations, and standardizes
|
||||||
|
maintainer publication through ReleaseKit. The minimum supported Go version
|
||||||
|
increases to 1.26.0.
|
||||||
|
|
||||||
|
## Discover Requirements And Available Definitions
|
||||||
|
|
||||||
|
- `Engine.InspectRequirements` reports recognizable input and caller-variable
|
||||||
|
dependencies for a selected prompt and supplied caller context before input
|
||||||
|
artifacts exist. It classifies dependencies as required, conditional, or
|
||||||
|
optional and reports structured limitations when discovery is incomplete.
|
||||||
|
Discovery coverage is not proof that requirements are satisfied or execution
|
||||||
|
will succeed. See the [requirement inspection guide](../consumers/pkg-promptkit.md#inspect-requirements-before-preparation)
|
||||||
|
and [GoDoc](../../requirements.go).
|
||||||
|
- `Engine.ListProfiles` returns effective profile IDs, and `Engine.ListPrompts`
|
||||||
|
returns exact ID/version pairs as `PromptIdentity` values. Lists come from the
|
||||||
|
engine's captured configuration, are lexically ordered and caller-owned, and
|
||||||
|
include consumer definitions and maintained profiles that remain effective.
|
||||||
|
Listing performs no provider or execution work. See the
|
||||||
|
[discovery guide](../consumers/pkg-promptkit.md#discover-available-profiles-and-prompts)
|
||||||
|
and [GoDoc](../../listing.go).
|
||||||
|
|
||||||
|
## Efficiency And Tooling
|
||||||
|
|
||||||
|
- Reduce response-sized copies in basic validation and rendered-message output,
|
||||||
|
and encode valid provider extra parameters only as part of the complete
|
||||||
|
request rather than serializing them twice.
|
||||||
|
- Use bounded initial file-size reservations without limiting accepted content,
|
||||||
|
releasing excess backing storage for small or empty completed artifacts.
|
||||||
|
- Reuse owned preparation values when creating prepared-execution snapshots,
|
||||||
|
while preserving independent inspection details and caller-owned data.
|
||||||
|
- Use lightweight captured profile identities and prompt metadata for inspection
|
||||||
|
instead of copying execution targets and definitions unnecessarily.
|
||||||
|
- Upgrade ConfigKit to v0.6.0 for more efficient loading, resolution, metadata,
|
||||||
|
and inspection processing. Configuration semantics remain unchanged; these
|
||||||
|
improvements primarily benefit construction and explicit operator inspection,
|
||||||
|
not upstream inference latency.
|
||||||
|
- Adopt module-selected ReleaseKit v0.3.0 for maintainer validation and tag
|
||||||
|
publication. The [release procedure](../release.md) owns the current commands
|
||||||
|
and recovery workflow; successful tag publication remains the completion
|
||||||
|
point, without waiting for CI.
|
||||||
|
|
||||||
|
These changes preserve existing execution and ownership contracts. Benefits
|
||||||
|
depend on workload; this release introduces no fixed performance guarantee.
|
||||||
|
|
||||||
|
## Compatibility And Upgrade
|
||||||
|
|
||||||
|
Consumers upgrading from v0.12.0 need **Go 1.26.0 or newer**, rather than the
|
||||||
|
previous 1.25.5 minimum. Update application build environments and any CI images
|
||||||
|
still using Go 1.25 before upgrading:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go get gitea.maximumdirect.net/eric/promptkit@v0.13.0
|
||||||
|
```
|
||||||
|
|
||||||
|
No application-code or configuration migration is required from v0.12.0 beyond
|
||||||
|
the Go toolchain update. The new discovery methods are optional additions;
|
||||||
|
existing construction, inspection, preparation, and execution workflows remain
|
||||||
|
available. See [Go compatibility](../consumers/pkg-promptkit.md#go-compatibility)
|
||||||
|
for the distinction between the consumer minimum and maintainer toolchain.
|
||||||
|
|
||||||
|
Maintainers should follow the current release runbook rather than the retired
|
||||||
|
`scripts/release-check.sh`. The publication entry point remains
|
||||||
|
`scripts/release.sh`; ReleaseKit now owns its shared validation and publication
|
||||||
|
mechanics.
|
||||||
|
|
||||||
|
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.13.0/COPYRIGHT) | [LICENSE](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.13.0/LICENSE) | [ATTRIBUTION](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.13.0/docs/oss/attribution.md)
|
||||||
@@ -1,505 +0,0 @@
|
|||||||
# ConfigKit Adoption Assessment
|
|
||||||
|
|
||||||
Status: assessment aligned to released ConfigKit v0.5.0. Upstream capability
|
|
||||||
prerequisites are available; PromptKit integration and migration decisions remain
|
|
||||||
for the feature roadmap and implementation plan. No runtime changes are part of
|
|
||||||
this assessment.
|
|
||||||
|
|
||||||
## Goal And Recommendation
|
|
||||||
|
|
||||||
Use ConfigKit to standardize PromptKit's configuration semantics and simplify
|
|
||||||
its loading machinery, improving ConfigKit where a reusable capability is
|
|
||||||
justified. Preserve substantive consumer capabilities, not historical loading
|
|
||||||
sequences, file syntax, or Go signatures. Document intentional breaking changes
|
|
||||||
and their migration paths.
|
|
||||||
|
|
||||||
Recommend adoption through an eager, immutable definition-loading boundary.
|
|
||||||
ConfigKit should own YAML interpretation, source-tier combination, strict typed
|
|
||||||
decoding, sibling inheritance, presence, and source diagnostics. PromptKit should
|
|
||||||
own definition selection, LLM-specific rules, and execution. Do not retain the lazy loading
|
|
||||||
architecture merely to minimize migration, or introduce a second generic
|
|
||||||
configuration framework inside PromptKit.
|
|
||||||
|
|
||||||
The expected benefits are consistent configuration semantics for applications
|
|
||||||
using both libraries and removal of repeated discovery, partial metadata parsing,
|
|
||||||
lazy error arbitration, and repeated profile-chain resolution. Some of those
|
|
||||||
savings come from the loading redesign rather than ConfigKit itself. ConfigKit
|
|
||||||
adds ongoing value by owning the generic decoding behavior and its tests.
|
|
||||||
|
|
||||||
## Released ConfigKit Capabilities
|
|
||||||
|
|
||||||
ConfigKit v0.5.0 satisfies the two requested prerequisites and also implements
|
|
||||||
sibling inheritance. Adopt the released public contracts rather than waiting for
|
|
||||||
the earlier proposals to be implemented:
|
|
||||||
|
|
||||||
1. **Filesystem sources:** `NewFilesystem(label, fsys).File(name)` borrows an
|
|
||||||
`fs.FS` with reusable namespace identity. Imports and resource contexts retain
|
|
||||||
that namespace. `Result.SourceContext` and `Resolution.SourceContext` expose
|
|
||||||
passive `ResourceContext` values with `FS` and `BaseDir`.
|
|
||||||
2. **Source-backed underlays:** `LoadLayered` accepts a main `Input`
|
|
||||||
and ordered `Underlay` groups. Each group composes strictly; groups overlay
|
|
||||||
recursively from lowest to highest priority. `Input{Source: NoSource()}` omits
|
|
||||||
the main input. `Mapping(label, map[string]any)` snapshots supported built-in
|
|
||||||
scalar/list/map values with explicit presence; it does not ingest arbitrary
|
|
||||||
typed structs. Generic programmatic input is not represented as CLI overrides.
|
|
||||||
3. **Sibling inheritance:** the reserved `extends` key names one immediate sibling
|
|
||||||
mapping. Relationships resolve after precedence assembly and before typed
|
|
||||||
decoding, with inherited origins, history, and resource context preserved.
|
|
||||||
This is built-in behavior, not the opt-in/registered-collection API originally
|
|
||||||
suggested by PromptKit.
|
|
||||||
|
|
||||||
Adopt ConfigKit's recursive combination model rather than preserving PromptKit's
|
|
||||||
whole-profile shadowing or atomic provider-map replacement across source tiers.
|
|
||||||
No hidden replacement or inheritance pass should run after ConfigKit loading.
|
|
||||||
Use released sibling inheritance for profiles, leaving domain validation in
|
|
||||||
PromptKit. This assessment does not authorize code changes in either repository.
|
|
||||||
|
|
||||||
## Evidence And Scope
|
|
||||||
|
|
||||||
PromptKit's runtime baseline remains v0.11.2; this assessment has since received
|
|
||||||
documentation-only revisions. Updated upstream review on 2026-09-30:
|
|
||||||
|
|
||||||
- PromptKit: `12d6d80024e80c552d122a076c22429bf6f2f6c2` (v0.11.2).
|
|
||||||
- ConfigKit: `dc09959cca4e105f49ec316c6d4ba585bd2340bd` (v0.5.0), verified
|
|
||||||
against the published tag. Its clean checkout passed `go test ./...` during
|
|
||||||
the release review. This replaces the earlier v0.4.2 and uncommitted-roadmap
|
|
||||||
comparison as the upstream basis for adoption.
|
|
||||||
|
|
||||||
PromptKit evidence: [consumer workflows](../consumers/pkg-promptkit.md),
|
|
||||||
[file formats](../formats.md), [source implementation](../internal/sources.md),
|
|
||||||
[runner](../internal/runner.md), and the implementation owners identified below.
|
|
||||||
ConfigKit evidence at that release: `docs/config.md`, `docs/releases/v0.5.0.md`,
|
|
||||||
`layered.go`, `source.go`, `resource_context.go`, and ADR 0009 in the sibling
|
|
||||||
checkout. That checkout is review context, not a build dependency or required
|
|
||||||
directory layout. The released configuration contract owns exact upstream behavior;
|
|
||||||
the summaries here identify its consequences for the proposed PromptKit design.
|
|
||||||
|
|
||||||
This is a source-and-contract assessment, not a performance benchmark or full
|
|
||||||
correctness audit. It does not claim measured allocation, latency, or line-count
|
|
||||||
savings. Current code and canonical contracts remain authoritative until an
|
|
||||||
approved implementation changes them.
|
|
||||||
|
|
||||||
## 1. Capability Inventory
|
|
||||||
|
|
||||||
| Preserve the ability to… | Replaceable mechanics and representation |
|
|
||||||
| --- | --- |
|
|
||||||
| Define named, versioned prompts, inspect their declared inputs, and select a version explicitly | Directory scans on each operation, YAML metadata prepasses, filename layout, constructor signatures |
|
|
||||||
| Use inline and file-backed message templates, variables, input helpers, session templates, roles, and cache control | Timing of definition/content reads and private repository interfaces |
|
|
||||||
| Configure named profiles, alias/inherit another profile, and override selected settings | Zero-as-absence fields, per-call chain traversal, raw YAML decoding types |
|
|
||||||
| Supply embedded application defaults, operator definitions, maintained catalogs, and programmatic profiles | Repository-wrapper stacks and exact source-option API |
|
|
||||||
| Register custom/local backends, use built-ins or endpoint-only profiles, and control backend capacity | Configuration DTO shapes and registry assembly order |
|
|
||||||
| Prepare, inspect, execute, or freeze an execution for later single-claim execution | Source reloading during ordinary preparation; exact preparation error ordering |
|
|
||||||
| Override per-run settings, credentials, reasoning, session, and output contract; append rendered messages | Public value signatures if presence or clearer configuration separation requires revision |
|
|
||||||
| Supply inline/file artifacts and inject artifact readers, model clients, and HTTP clients | Assembly APIs; runtime collaborators need not be YAML-decodable |
|
|
||||||
| Validate output, use provider-native structured output, bounded repair, transport retries, cancellation, and observation | Internal wiring, not the substantive execution behaviors |
|
|
||||||
| Receive results, usage, identities, hashes, validation outcomes, and inspectable failures | Configuration error types and when definition failures appear |
|
|
||||||
|
|
||||||
Preserve independent profile and prompt inspection as useful operations. They
|
|
||||||
need not retain the old guarantee that engine construction has never examined
|
|
||||||
other definitions. Preserve custom backend identity and capacity behavior,
|
|
||||||
optional upstream credentials, and the distinction between failed generated
|
|
||||||
content and operational errors. This project does not authorize silently
|
|
||||||
dropping execution features to simplify the configuration layer.
|
|
||||||
|
|
||||||
Add explicit configuration inspection for consumers implementing operator-facing
|
|
||||||
CLI commands or debug output. This is a new capability motivated by this adoption,
|
|
||||||
not an assertion that the current prompt/profile inspection methods already
|
|
||||||
provide configuration provenance.
|
|
||||||
|
|
||||||
## 2. Responsibility Mapping
|
|
||||||
|
|
||||||
| Concern | Recommended owner and disposition |
|
|
||||||
| --- | --- |
|
|
||||||
| YAML grammar, scalar interpretation, one-document enforcement, unknown fields, typed ranges, null legality | Delegate to ConfigKit; remove corresponding PromptKit parser logic and redundant parser tests |
|
|
||||||
| Field presence, source coordinates, and configuration provenance | Delegate to ConfigKit; retain its inspection state with the loaded snapshot for explicit operator queries, separate from execution-ready values |
|
|
||||||
| Operator-facing configuration inspection | Provide a PromptKit consumer API reusing ConfigKit inspection facilities; applications own CLI commands, presentation, and disclosure policy |
|
|
||||||
| File/directory/embedded source selection and definition ID indexing | Retain a small PromptKit collection loader; ConfigKit loads documents, not a prompt catalog |
|
|
||||||
| Configuration filesystem sources, relative imports, and generic layer origins | Delegate to the prerequisite ConfigKit APIs; do not maintain parallel reader-based loading machinery |
|
|
||||||
| Definition discovery and relative message-content references | Retain PromptKit's domain resource adapters, using ConfigKit source context and standard `io/fs` facilities where sufficient |
|
|
||||||
| Profile-source priority and combination | PromptKit selects tiers; ConfigKit owns strict within-tier composition and recursive cross-tier overlay; eliminate repository fallback and whole-definition shadowing |
|
|
||||||
| Named profile inheritance | Delegate sibling graph resolution, combination, and derivation to ConfigKit's `extends`; remove the custom profile resolver |
|
|
||||||
| Backend selection and domain normalization | Retain PromptKit domain rules; use recorded application resolution for configuration transformations not already handled by structural inheritance |
|
|
||||||
| LLM setting bounds, role/session normalization, output rules, reserved provider fields | Retain shared PromptKit domain owners; ConfigKit validation callbacks may invoke them |
|
|
||||||
| Definition defaults | Use ConfigKit defaults where they do not erase inheritance presence; apply effective execution defaults after inheritance |
|
|
||||||
| Programmatic configuration | Prefer labeled mapping input with explicit presence; evaluate whether typed PromptKit convenience constructors justify conversion code, without YAML round trips or zero-value inference |
|
|
||||||
| Runtime clients, observers, artifact readers, request data | Retain normal Go injection and request APIs; do not load them through ConfigKit |
|
|
||||||
| Credentials | Retain optional environment-name and direct request-key semantics; do not replace these with ConfigKit's required typed secret resolution |
|
|
||||||
| Catalog manifest, ownership, reserved names, secret-safety checks | Retain the catalog adapter; reuse the common ConfigKit profile decoder |
|
|
||||||
| JSON Schema resources and generated-output validation | Retain JSON Schema implementation; these are not application configuration documents |
|
|
||||||
| Application imports, CLI overrides, deployment overlays, secrets-directory bootstrap | Leave with consuming applications unless a concrete PromptKit need is accepted |
|
|
||||||
| Generic source/decoding diagnostics | Reuse ConfigKit; add only PromptKit identity/context and domain relationship diagnostics |
|
|
||||||
|
|
||||||
## 3. Recommended Target Loading Model
|
|
||||||
|
|
||||||
### Definition Snapshot
|
|
||||||
|
|
||||||
Create one immutable definition snapshot during engine construction. Discover
|
|
||||||
the explicitly supplied sources, assemble their contributions through ConfigKit,
|
|
||||||
decode the effective configuration, normalize domain values, and resolve relationships
|
|
||||||
before publishing the engine. No provider call or credential availability check
|
|
||||||
belongs in this phase. Publish no partially initialized engine on failure.
|
|
||||||
|
|
||||||
Prefer keyed collections, such as `profiles.<name>`, assembled before one typed
|
|
||||||
loading lifecycle instead of independently decoding each profile and recombining
|
|
||||||
the results. Multiple files can contribute to those collections; this does not
|
|
||||||
require a monolithic file. Reconsider the previous one-typed-document-per-definition
|
|
||||||
recommendation. Settle prompt ID/version representation and catalog format migration
|
|
||||||
in the feature roadmap. Support OS directories, individual files, embedded/custom
|
|
||||||
`fs.FS` sources, and programmatic input through the new source model; source selection
|
|
||||||
and definition discovery remain PromptKit concerns, not a second composition engine.
|
|
||||||
|
|
||||||
ConfigKit expands structural inheritance before typed defaults/decoding. Perform
|
|
||||||
any remaining domain-specific resolution through its recorded lifecycle before
|
|
||||||
semantic validation and result publication. Reuse its presence and provenance;
|
|
||||||
do not add a generic merge/defaulting pass or reconstruct origins after loading.
|
|
||||||
Runtime client assembly and resource interpretation remain outside typed decoding.
|
|
||||||
|
|
||||||
Load and freeze referenced message text with definitions. Validate static prompt
|
|
||||||
structure and template syntax at loading, using the same template function setup
|
|
||||||
as execution; render only when request variables and artifacts exist. Preserve
|
|
||||||
source-relative content resolution and containment without expanding the threat
|
|
||||||
model. A snapshot is a set of captured reads, not an atomic filesystem transaction.
|
|
||||||
|
|
||||||
Resolve each effective content reference using its contributing source's context,
|
|
||||||
not the highest tier's directory. Use ConfigKit's released direct context
|
|
||||||
query rather than a consumer-maintained filesystem registry. Composite origins
|
|
||||||
must not be assigned an arbitrary directory; ambiguous or absent context needs
|
|
||||||
an actionable resource error or explicit consumer-supplied context. ConfigKit
|
|
||||||
does not snapshot external text: PromptKit opens it explicitly and freezes it
|
|
||||||
before engine publication. Caller-owned filesystem lifetimes remain caller-owned;
|
|
||||||
close resource handles acquired by PromptKit without retaining ConfigKit load handles.
|
|
||||||
|
|
||||||
Keep request artifacts and schema compilation at preparation. A request can
|
|
||||||
replace the output contract, so an unused default schema should not need eager
|
|
||||||
compilation. Prepared execution continues to freeze its effective resources.
|
|
||||||
The snapshot covers definitions/message text, not environment values, request
|
|
||||||
files, every schema file, or execution state.
|
|
||||||
|
|
||||||
### Composition And Validation
|
|
||||||
|
|
||||||
Use explicit source categories in the existing useful priority order:
|
|
||||||
programmatic profiles, operator source, application fallback, PromptKit-owned
|
|
||||||
profiles, then external catalogs. Express these as explicit ordered tiers, not
|
|
||||||
source discovery order. Within a tier, mappings compose recursively and overlapping
|
|
||||||
non-mapping contributions conflict, even when equal. Disjoint contributions to
|
|
||||||
the same profile may therefore combine; do not preserve a blanket duplicate-ID
|
|
||||||
rejection. Across tiers, mappings overlay recursively, retaining omitted lower
|
|
||||||
children. Effective prompt identities/versions must remain addressable, with an
|
|
||||||
omitted version ambiguous when multiple versions exist; finalize their keyed
|
|
||||||
representation without reintroducing whole-definition shadowing.
|
|
||||||
|
|
||||||
All sources undergo syntax, acquisition, and composition checks; higher tiers
|
|
||||||
cannot hide those failures. Decode and validate effective values after assembly,
|
|
||||||
not every intermediate tier against the final typed schema. A replaced lower
|
|
||||||
value may consequently have a type that would not fit the final schema. An unused
|
|
||||||
invalid effective definition still fails construction. Catalog ownership,
|
|
||||||
reserved-name, and secret-safety checks remain distinct source-policy requirements,
|
|
||||||
not a reason to impose intermediate typed validation on ordinary inputs.
|
|
||||||
|
|
||||||
Preserve profile templates that become usable through descendants or runtime
|
|
||||||
overrides: eager loading must not require every consumer profile to be an
|
|
||||||
independently runnable endpoint/model pair. Validate supplied fields and graph
|
|
||||||
integrity eagerly (ConfigKit handles missing bases and cycles), and check final execution
|
|
||||||
requirements after runtime selection/overrides. Prompt default-profile references
|
|
||||||
can be checked against the assembled collection; a prompt with no default remains
|
|
||||||
valid for callers that supply a profile explicitly.
|
|
||||||
|
|
||||||
ConfigKit's ordering is typed defaults, source-underlay tiers, strictly composed
|
|
||||||
main input, selected overlay, then CLI overrides. PromptKit's API must map its
|
|
||||||
source priorities explicitly onto supported roles; do not mislabel programmatic
|
|
||||||
input as CLI flags. Underlay-only loading avoids inventing a main document.
|
|
||||||
Imports remain opt-in, and the named-overlay convenience need not be exposed by
|
|
||||||
PromptKit merely because ConfigKit supports it.
|
|
||||||
|
|
||||||
A prompt's `version` is a domain identifier, not automatically a ConfigKit
|
|
||||||
application-schema version. Do not configure `VersionRule` for prompt versions.
|
|
||||||
If a separate configuration schema version is later needed, respect ConfigKit's
|
|
||||||
designated main-input ownership and pre-underlay check timing; do not assume
|
|
||||||
last-wins version selection across catalogs.
|
|
||||||
|
|
||||||
### Source Presence And Released Sibling Inheritance
|
|
||||||
|
|
||||||
Adopt ConfigKit source-layer semantics: absence retains lower input; explicit
|
|
||||||
zero, false, and empty strings replace it; mappings combine recursively; empty
|
|
||||||
mappings retain lower children; sequences replace as units; null explicitly
|
|
||||||
replaces, subject to final schema legality. No reset/delete operator is assumed.
|
|
||||||
Preserve presence through domain resolution, backend/framework
|
|
||||||
default application, request overrides, and outbound request construction.
|
|
||||||
Otherwise accepting `temperature: 0` during decoding would accomplish nothing.
|
|
||||||
|
|
||||||
Use ConfigKit metadata for both file and programmatic input. A final Go value
|
|
||||||
alone does not distinguish absent from explicit zero. Avoid eagerly defaulting
|
|
||||||
profile fields in ways that erase that distinction before relationship resolution.
|
|
||||||
|
|
||||||
Provider `extra_params` maps combine recursively across source tiers too: `{}`
|
|
||||||
does not clear lower keys. Retire the proposed atomic source-map replacement and
|
|
||||||
null-as-unspecified conventions; do not emulate them after loading. An operator
|
|
||||||
overriding only `model` also retains a lower `extends` declaration and other
|
|
||||||
omitted settings. Diagnose and document these consequences explicitly.
|
|
||||||
|
|
||||||
Replace `base_profile` with `extends` on sibling profile mappings. A string names
|
|
||||||
one exact immediate sibling; null cancels the relationship while preserving the
|
|
||||||
child's own settings. Omission retains a lower-tier directive. There is no ancestor
|
|
||||||
scope search or cross-collection reference. Superseded references are history,
|
|
||||||
not active relationships to validate; every effective relationship resolves,
|
|
||||||
including profiles not selected for execution.
|
|
||||||
|
|
||||||
Children recursively overlay their resolved parents with the same value rules:
|
|
||||||
zero/false/empty strings and sequences/null replace, mappings combine, and `{}`
|
|
||||||
does not clear inherited keys. Inherited values count as explicit input for
|
|
||||||
presence queries and absence-based defaults, even when false, zero, or null.
|
|
||||||
Do not repeat inheritance in PromptKit's application-resolution callback. Use
|
|
||||||
`Origin.Inheritance` and retained history for explicit operator inspection.
|
|
||||||
Inherited resource references retain their declaring source's context.
|
|
||||||
|
|
||||||
Enclosing inheritance resolves before nested relationships; nested inherited
|
|
||||||
directives resolve at each destination against its siblings. Defaults and
|
|
||||||
application-derived values do not trigger another inheritance pass. Model this
|
|
||||||
deliberately in PromptKit configuration rather than promising arbitrary graph
|
|
||||||
resolution over runtime values. Partial bases must fit the typed schema and
|
|
||||||
PromptKit's validation; ConfigKit grants them no special validation exemption.
|
|
||||||
|
|
||||||
`extends` is reserved in every input mapping, including provider-parameter maps,
|
|
||||||
across all source kinds. There is no opt-in switch or ordinary-data escape in the
|
|
||||||
released contract. Audit catalog and consumer payload needs before finalizing
|
|
||||||
their format; a literal provider field with that name cannot silently pass through
|
|
||||||
the configuration loader. Do not invent an encoding workaround in this assessment.
|
|
||||||
|
|
||||||
The existing request-level override API remains a separate runtime boundary.
|
|
||||||
The feature roadmap must review its interaction with configured values without
|
|
||||||
silently subjecting runtime request data to configuration inheritance directives.
|
|
||||||
|
|
||||||
### Reloading, Errors, And Application Integration
|
|
||||||
|
|
||||||
Later definition edits take effect through a newly constructed engine. Do not add
|
|
||||||
watchers, mutable registry reloads, or a second lazy compatibility mode. Applications
|
|
||||||
choose how to swap engines and let in-flight work finish. Old prepared handles
|
|
||||||
remain associated with their original engine. Simultaneously active engines have
|
|
||||||
separate backend capacity pools; migration guidance must not imply a global limit
|
|
||||||
across a reload overlap.
|
|
||||||
|
|
||||||
Return configuration failures from construction with file/path/location and
|
|
||||||
useful definition identity where available. Recommend preserving ConfigKit's
|
|
||||||
public `LoadError` through wrapping for `errors.As`, together with a PromptKit
|
|
||||||
configuration category; do not copy its entire diagnostic API into the facade.
|
|
||||||
Domain-only relationship failures remain PromptKit errors with relevant source
|
|
||||||
locations. Start with deterministic fail-fast loading across files and ConfigKit's
|
|
||||||
existing multi-diagnostic behavior within a load, rather than inventing a batch
|
|
||||||
diagnostic subsystem. Do not expose raw credentials or add implicit logging.
|
|
||||||
|
|
||||||
Runtime failures continue to distinguish invalid requests, missing selections,
|
|
||||||
rendering/artifact/schema errors, provider/transport errors, capacity rejection,
|
|
||||||
and content-validation results. Cancellation remains effective at
|
|
||||||
PromptKit-controlled boundaries; ConfigKit's current synchronous `Load` has no
|
|
||||||
context argument, so do not promise interruption inside it or create goroutines
|
|
||||||
to simulate that guarantee.
|
|
||||||
|
|
||||||
Applications using ConfigKit can present the same underlying loading diagnostics
|
|
||||||
for their application settings and PromptKit files. They can also translate
|
|
||||||
application-owned typed settings into PromptKit profiles/backends. Do not require
|
|
||||||
applications to encode Go clients into YAML or adopt ConfigKit merely to call
|
|
||||||
PromptKit's runtime API. The operator-inspection capability below supplies the
|
|
||||||
concrete consumer purpose for exposing effective configuration and its provenance.
|
|
||||||
|
|
||||||
### Explicit Configuration Inspection
|
|
||||||
|
|
||||||
Provide a consumer-facing API for inspecting effective configuration and its
|
|
||||||
provenance so applications can implement targeted CLI commands or debug output.
|
|
||||||
Keep three activities distinct:
|
|
||||||
|
|
||||||
- **Construction:** load and resolve configuration, retain ConfigKit's provenance
|
|
||||||
and inspection state, and publish execution-ready definitions. Resolve any
|
|
||||||
presence and origin information needed for definition construction here.
|
|
||||||
- **Ordinary execution:** use the ready definitions directly. Do not analyze
|
|
||||||
configuration provenance, traverse origin/history records, reconstruct
|
|
||||||
inheritance explanations, or render effective-configuration reports as part of
|
|
||||||
a prompt run. Carry execution-relevant presence explicitly in resolved values
|
|
||||||
rather than querying configuration metadata during each run.
|
|
||||||
- **Explicit inspection:** query effective settings and their contributing
|
|
||||||
sources or explanations only when a consumer invokes the inspection API.
|
|
||||||
Inspection uses the same loaded configuration snapshot as the engine; it does
|
|
||||||
not reread configuration files, open referenced resources, or contact providers.
|
|
||||||
|
|
||||||
Reuse ConfigKit's inspection and redaction facilities rather than maintaining a
|
|
||||||
second provenance system. The API should make targeted queries practical; exact
|
|
||||||
signatures, query scope, and safe result ownership belong in the feature roadmap.
|
|
||||||
Do not imply that configuration inspection predicts all runtime request overrides
|
|
||||||
or reveals the values of execution-time credentials. Existing prompt/profile
|
|
||||||
inspection, prepared-run details, and execution observations have distinct purposes;
|
|
||||||
configuration inspection does not replace them or add provenance to run events.
|
|
||||||
|
|
||||||
Applications decide whether and how to expose inspection through operator tools.
|
|
||||||
PromptKit adds no CLI, implicit debug logging, or automatic explanation output.
|
|
||||||
Redacted configuration can still contain sensitive application content, so callers
|
|
||||||
retain responsibility for access control and disclosure. Retention has a memory
|
|
||||||
cost, but explanation generation and provenance analysis stay off the run path.
|
|
||||||
|
|
||||||
## 4. Simplification Accounting
|
|
||||||
|
|
||||||
| Current implementation | Expected disposition |
|
|
||||||
| --- | --- |
|
|
||||||
| `internal/profile/filesystem_repository.go`: metadata scans, selected strict decode, stream-end checks, malformed-file selection arbitration | Replace with source acquisition, ConfigKit assembly/decoding, and indexed effective values; remove the metadata prepass and per-lookup filesystem path |
|
|
||||||
| `internal/promptdef/filesystem_repository.go`: metadata selection and repeated decoding/content opening | Replace with eager prompt loads and an ID/version index; keep domain normalization and contained content reading |
|
|
||||||
| Profile filesystem, `fs.FS`, eager, memory, and overlay repository variants | Collapse loading into source adapters plus immutable collections; preserve only interfaces that still serve a real boundary |
|
|
||||||
| `internal/profile/resolving_repository.go` | Remove custom graph traversal, cycle detection, and zero-sentinel merging in favor of ConfigKit sibling inheritance; retain only PromptKit domain validation |
|
|
||||||
| `internal/engine` raw repository-stack assembly | Replace with ConfigKit source tiers and one effective typed collection; engine capacity/client assembly remains |
|
|
||||||
| `internal/catalog` profile metadata extraction | Obtain explicit field information from ConfigKit; retain manifests, ownership, namespace, connection-metadata and secret checks |
|
|
||||||
| `internal/filecatalog` | Retain/reduce discovery and containment helpers actually needed by the eager loader; do not claim the entire package disappears |
|
|
||||||
| `internal/domain` source YAML tags and runtime/source coupling | Separate configuration DTOs from execution values where presence requires it; reuse domain validation rather than duplicating it |
|
|
||||||
| Direct `gopkg.in/yaml.v3` use for definition loading | Remove once all such consumers migrate; ConfigKit brings its own private YAML dependency |
|
|
||||||
|
|
||||||
New integration work is real: collection DTOs, ConfigKit calls and error wrapping,
|
|
||||||
source-context access, collection indexing, programmatic mapping ergonomics, and
|
|
||||||
conversion to runtime definitions. Catalog documents may need new keyed shapes;
|
|
||||||
do not hide that migration behind a permanent second loader.
|
|
||||||
Some defensive copies remain necessary at public/injected-client boundaries;
|
|
||||||
ConfigKit's detached load result is not permission to expose shared engine maps.
|
|
||||||
`internal/jsonvalue` still serves programmatic values, request parameters, and
|
|
||||||
prepared schema ownership, so adoption does not eliminate it.
|
|
||||||
|
|
||||||
Maintenance savings depend on actually deleting the old lazy paths and their
|
|
||||||
special-case tests. Move generic YAML/scalar behavior coverage to ConfigKit's
|
|
||||||
ownership; PromptKit retains representative adapter tests plus domain, precedence,
|
|
||||||
snapshot, inheritance, embedded-source, error, and execution integration tests.
|
|
||||||
Do not duplicate ConfigKit's entire parser suite or discard execution regression
|
|
||||||
coverage because construction fixtures need migration.
|
|
||||||
|
|
||||||
Runtime benefit: ordinary runs no longer rediscover definition directories,
|
|
||||||
redecode definitions, reread template text, or traverse inheritance chains.
|
|
||||||
Cost: construction loads even unused definitions and retains their text and
|
|
||||||
resolved values. Retain ConfigKit inspection state needed by the explicit operator
|
|
||||||
API without duplicating its history in runtime definitions or per-run values.
|
|
||||||
These are reasoned tradeoffs, not measured performance
|
|
||||||
claims. Adoption is not justified by the decoder replacement alone.
|
|
||||||
|
|
||||||
### Algorithmic Efficiency
|
|
||||||
|
|
||||||
Efficiency is an explicit design criterion for both the upstream work and this
|
|
||||||
shared-library refactor. Prefer optimal or near-optimal algorithms relative to
|
|
||||||
input and produced-data size, not a blanket O(n) claim with an undefined n.
|
|
||||||
|
|
||||||
- Acquire/parse each accepted source occurrence once, assemble input, and run
|
|
||||||
one typed loading lifecycle rather than decoding intermediate tiers. Reuse
|
|
||||||
captured content instead of reparsing it for metadata, selection, and validation.
|
|
||||||
- Build identity indexes in one pass, with expected constant-time map lookup.
|
|
||||||
Do not scan whole collections for every lookup or precedence decision.
|
|
||||||
- Delegate inheritance expansion to ConfigKit once during loading rather than
|
|
||||||
rebuilding its graph in PromptKit. Assess scaling against input, materialized
|
|
||||||
inherited values, nested destination resolution, and retained history; do not
|
|
||||||
claim unconditional O(V + E) total work from the count of profiles and edges.
|
|
||||||
ConfigKit bounds expansion and retained evidence through its existing budgets;
|
|
||||||
representative catalog sizes must fit them without new per-run processing.
|
|
||||||
- Avoid whole-collection copying per layer, repeated serialization, and repeated
|
|
||||||
prefix/history traversal. Analyze layer processing against total input nodes
|
|
||||||
and produced values/provenance, including intentionally retained history.
|
|
||||||
- Keep configuration discovery, decoding, and provenance analysis out of ordinary
|
|
||||||
execution. Perform explanation work only on explicit inspection calls. Copy only
|
|
||||||
where ownership requires it; do not trade caller isolation for speed.
|
|
||||||
|
|
||||||
These are design targets, not promised complexity for every operation. Sorting
|
|
||||||
for deterministic discovery may require O(F log F) for F files, and processing
|
|
||||||
message text costs work proportional to its bytes. Construction time, retained
|
|
||||||
memory, and frequent-call costs all matter. Prefer straightforward indexes and
|
|
||||||
clear immutable ownership over speculative global caches or intricate optimizers.
|
|
||||||
The implementation plan should include focused benchmarks for consequential
|
|
||||||
repeated-work risks, such as shared-ancestor profiles and many layers, rather than
|
|
||||||
machine-dependent timing gates or benchmarks for every helper.
|
|
||||||
|
|
||||||
## 5. Upstream Readiness And Remaining Opportunities
|
|
||||||
|
|
||||||
Filesystem sources, underlays, programmatic mappings, source contexts, and sibling
|
|
||||||
inheritance are released in v0.5.0. No additional upstream feature is currently a
|
|
||||||
demonstrated prerequisite. Context-aware loading remains a possible future request
|
|
||||||
if concrete consumers need it, not a reason to delay adoption. Existing loading
|
|
||||||
is synchronous; do not simulate cancellation with detached background work.
|
|
||||||
|
|
||||||
Do not propose a public raw AST, permissive identity-only decoder, or dynamic
|
|
||||||
subtree conversion solely to preserve PromptKit's current scanning design.
|
|
||||||
Do not upstream backend/model selection, credential timing for inference,
|
|
||||||
profile catalog reservations, or prompt-template functions.
|
|
||||||
|
|
||||||
ConfigKit and PromptKit both declare LGPL-3.0-or-later. Adoption must still update
|
|
||||||
dependency notices under the OSS policy and inspect the exact selected release
|
|
||||||
and its transitive notices. Their current Go minimums do not require a PromptKit
|
|
||||||
toolchain increase. Use a released module dependency, not a sibling-directory
|
|
||||||
replacement. Select v0.5.0 as the assessed adoption baseline; any newer selection
|
|
||||||
needs its own review. This assessment changes neither dependencies nor licenses.
|
|
||||||
|
|
||||||
## 6. Decisions And Migration
|
|
||||||
|
|
||||||
The agreed direction now incorporates ConfigKit's v0.5.0 contracts. This supersedes
|
|
||||||
the earlier recommendations for independently typed documents, whole-definition
|
|
||||||
shadowing, intermediate typed validation, and atomic maps/null-as-unspecified at
|
|
||||||
the source boundary.
|
|
||||||
|
|
||||||
1. **Loading lifetime and failure scope.** Keep eager construction and frozen
|
|
||||||
definitions/content. Syntax/acquisition/composition failures remain unmaskable;
|
|
||||||
typed and domain validation apply to effective values. A separate reusable
|
|
||||||
public snapshot lifecycle remains possible but is not currently needed.
|
|
||||||
2. **Source organization.** Prefer keyed collections composed across files and
|
|
||||||
tiers before typed loading. Settle exact profile and prompt/version shapes,
|
|
||||||
source-role mapping, and catalog migration in the feature roadmap. Independent
|
|
||||||
typed loads followed by custom combination are not the selected direction.
|
|
||||||
3. **Presence and relationships.** Accept recursive source-tier overlay, retained
|
|
||||||
children for empty mappings, explicit null replacement, and effective metadata.
|
|
||||||
Adopt built-in sibling inheritance, including reserved `extends`, recursive
|
|
||||||
mapping combination, null cancellation, and inherited explicit presence. Do
|
|
||||||
not preserve a parallel PromptKit structural resolver.
|
|
||||||
4. **Diagnostic boundary.** Keep wrapping ConfigKit's public loading errors rather
|
|
||||||
than mirroring them. Retain actionable domain relationship and resource errors
|
|
||||||
without duplicating ConfigKit's generic diagnostic/provenance system.
|
|
||||||
5. **Operator inspection.** Expose effective configuration and provenance through
|
|
||||||
an explicit consumer API backed by the engine's loaded snapshot. Retain the
|
|
||||||
required ConfigKit inspection state at construction; never analyze it during
|
|
||||||
ordinary prompt execution. Finalize targeted query shapes and ownership in the
|
|
||||||
feature roadmap, with presentation remaining application-owned.
|
|
||||||
|
|
||||||
Use released mapping-based programmatic input and direct filesystem resource-context
|
|
||||||
queries. If real consumer examples show
|
|
||||||
mapping construction to be repetitive, consider small typed PromptKit conveniences
|
|
||||||
with explicit presence rather than requiring an upstream typed-ingestion API.
|
|
||||||
|
|
||||||
Remaining PromptKit decisions are the exact collection/version format, catalog
|
|
||||||
migration, programmatic ergonomics, operator-inspection surface, and validation
|
|
||||||
of partial base profiles. Also settle how operators discard unwanted provider
|
|
||||||
settings or old relationships using the released null semantics without a hidden
|
|
||||||
replacement pass. Check the reserved-key impact on arbitrary provider maps. These
|
|
||||||
are integration decisions, not unresolved ConfigKit API proposals.
|
|
||||||
|
|
||||||
Expected consumer migration:
|
|
||||||
|
|
||||||
- Update definitions for ConfigKit's strict scalar rules, rejected YAML
|
|
||||||
anchors/aliases/tags, input limits, and null/type checks. Quote prompt version
|
|
||||||
strings rather than introducing a special numeric-version conversion merely
|
|
||||||
for historical YAML coercion.
|
|
||||||
- Move standalone definitions into the agreed keyed contribution format. Review
|
|
||||||
same-name contributions: disjoint fields within a tier can combine, and higher
|
|
||||||
tiers retain omitted lower fields, including `extends` references. Convert
|
|
||||||
`base_profile` to sibling `extends`; use `extends: null` to cancel a lower-tier
|
|
||||||
relationship, not to erase all settings already supplied to the child.
|
|
||||||
- Review zero/false/empty/null values explicitly. Provider maps merge across
|
|
||||||
tiers, `{}` does not clear them, and null replaces rather than means omission.
|
|
||||||
Use labeled programmatic mappings or approved presence-aware conveniences.
|
|
||||||
Apply those same combination rules to inherited settings. Rename ordinary
|
|
||||||
configuration keys named `extends`; examine provider maps for cases where
|
|
||||||
renaming would change their substantive meaning and resolve those explicitly.
|
|
||||||
- Handle effective-definition errors at construction, fix syntax/composition
|
|
||||||
failures even in overridden inputs, and deliberately rebuild/swap engines
|
|
||||||
after definition changes.
|
|
||||||
- Keep runtime credential lookup optional, avoid baking resolved secrets into
|
|
||||||
definition snapshots, and preserve request-scoped key overrides.
|
|
||||||
- Update maintained catalog data for collection shapes and any incompatible syntax;
|
|
||||||
coordinate catalog releases before the PromptKit dependency update. Preserve
|
|
||||||
maintained profile names and catalog ownership rules, not every old scalar
|
|
||||||
spelling. Do not validate live provider/model availability.
|
|
||||||
- Update error inspection, constructor/source options, and programmatic profile
|
|
||||||
literals as required. Keep representative preparation, execution, inspection,
|
|
||||||
custom backend, fallback, and embedded workflows in the migration guidance.
|
|
||||||
|
|
||||||
The subsequent feature roadmap should carry forward these choices, resolve the
|
|
||||||
remaining PromptKit integration decisions, and specify the target end state.
|
|
||||||
The staged implementation plan can now adopt released ConfigKit support without
|
|
||||||
waiting for upstream feature work, coordinating any catalog format releases before
|
|
||||||
their consumption. Cover deletions/replacements, contract tests, canonical documentation,
|
|
||||||
examples, attribution, and minor-release migration notes. Neither document should
|
|
||||||
become a reason to preserve obsolete compatibility machinery.
|
|
||||||
@@ -1,339 +0,0 @@
|
|||||||
# ConfigKit Adoption
|
|
||||||
|
|
||||||
Status: agreed feature roadmap. ConfigKit v0.5.0 supplies the required upstream
|
|
||||||
capabilities. The [implementation plan](implementation.md) owns execution steps.
|
|
||||||
The [configuration-boundary ADR](../adr/0001-configuration-boundary.md) records
|
|
||||||
the handoff decision. This document owns the intended end state, not
|
|
||||||
implementation authorization.
|
|
||||||
|
|
||||||
## Purpose
|
|
||||||
|
|
||||||
Use ConfigKit to standardize PromptKit's configuration semantics and simplify
|
|
||||||
its loading machinery, replacing custom logic wherever ConfigKit is the
|
|
||||||
appropriate owner. Preserve substantive consumer capabilities, not historical
|
|
||||||
file syntax, loading order, or Go signatures. Pre-1.0 breaking changes are
|
|
||||||
acceptable and must have migration guidance.
|
|
||||||
|
|
||||||
**This is an intentional breaking change. We are intentionally replacing PromptKit's configuration handling code in favor of adopting ConfigKit's simpler, standardized API. Our goal is to arrive at a new version of PromptKit that is leaner, simpler, and uses a ConfigKit-standard configuration syntax. We MUST NOT give any weight to preserving arbitrary implementation details from PromptKit's current codebase--those should be discarded in favor of the simplest viable implementation that supports PromptKit's substantive LLM-focused consumer capabilities.**
|
|
||||||
|
|
||||||
Applications using both libraries should share configuration interpretation,
|
|
||||||
layering, inheritance, presence, and inspection semantics. PromptKit should
|
|
||||||
concentrate on LLM-specific validation, resource interpretation, and execution.
|
|
||||||
The [adoption assessment](configkit-assessment.md) records the supporting analysis;
|
|
||||||
this roadmap owns the agreed feature end state. Where the earlier assessment
|
|
||||||
differs, this roadmap and the maintainer's direction above take precedence.
|
|
||||||
|
|
||||||
Do not preserve source-option wrappers, repository interfaces, incidental error
|
|
||||||
ordering, or old field shapes solely because they exist today. Reuse existing
|
|
||||||
code only when it remains the simplest appropriate owner of a substantive
|
|
||||||
capability. Migration documentation replaces compatibility machinery, not vice versa.
|
|
||||||
|
|
||||||
## Baseline And Dependency
|
|
||||||
|
|
||||||
The current PromptKit runtime baseline is v0.11.2. Its
|
|
||||||
[source implementation](../internal/sources.md), [format contract](../formats.md),
|
|
||||||
and [consumer guide](../consumers/pkg-promptkit.md) describe existing behavior,
|
|
||||||
not the target described here.
|
|
||||||
|
|
||||||
The reviewed ConfigKit release is v0.5.0 at
|
|
||||||
`dc09959cca4e105f49ec316c6d4ba585bd2340bd`. It provides native `fs.FS` sources,
|
|
||||||
`LoadLayered`, source-backed underlays, labeled programmatic mappings, passive
|
|
||||||
resource contexts, and built-in sibling inheritance. Its released configuration
|
|
||||||
contract and GoDoc own exact upstream semantics. No further upstream feature is
|
|
||||||
currently a demonstrated prerequisite. Review any newer version before selection.
|
|
||||||
|
|
||||||
Adopt a released Go module, not a local `replace` or workspace requirement.
|
|
||||||
Update attribution and transitive license notices under the OSS policy. This
|
|
||||||
feature does not change either project's license or require a Go minimum increase
|
|
||||||
relative to the reviewed versions.
|
|
||||||
|
|
||||||
## Substantive Capabilities To Preserve
|
|
||||||
|
|
||||||
- Named/versioned prompts, declared-input inspection, templates, inline and
|
|
||||||
referenced message content, variables, input helpers, roles, and cache control.
|
|
||||||
- Maintained catalogs, application defaults, operator configuration, programmatic
|
|
||||||
profiles, aliases/inheritance, built-in/custom/local backends, and endpoint-only
|
|
||||||
profiles.
|
|
||||||
- Preparation without generation, ordinary execution, and frozen single-claim
|
|
||||||
prepared execution; independent prompt and profile inspection.
|
|
||||||
- Runtime execution-setting, reasoning, session, credential, and output-contract
|
|
||||||
overrides; appended already-rendered messages.
|
|
||||||
- Inline/file input artifacts and injected artifact, model, and HTTP collaborators.
|
|
||||||
- Backend capacity management, cancellation, transient retries, structural output
|
|
||||||
validation, provider-native structured output, bounded repair, and observations.
|
|
||||||
- Results, effective identities/settings, usage, hashes, and meaningful errors.
|
|
||||||
|
|
||||||
Retain optional environment credentials and direct request credentials. ConfigKit
|
|
||||||
secret resolution must not introduce a new requirement that an upstream API key
|
|
||||||
exist. Domain validation must not probe provider/model availability.
|
|
||||||
|
|
||||||
## Target Configuration Model
|
|
||||||
|
|
||||||
### One Shared Loading Lifecycle
|
|
||||||
|
|
||||||
Construction supplies selected configuration inputs to ConfigKit, which composes,
|
|
||||||
overlays, expands inheritance, and decodes them. PromptKit interprets and validates
|
|
||||||
the final effective configuration at the ConfigKit handoff and produces
|
|
||||||
execution-ready definitions. ConfigKit's effective-value resolution and validation
|
|
||||||
hooks may implement that handoff. Publish no partially initialized engine on failure.
|
|
||||||
|
|
||||||
ConfigKit owns generic parsing, combination, presence, origins, and inheritance.
|
|
||||||
PromptKit owns input selection/discovery, definition indexing, LLM-specific rules,
|
|
||||||
and translation of final values into runtime settings. Input adapters select or
|
|
||||||
encode contributions; they do not inspect or validate raw source contents.
|
|
||||||
Effective presence/null information and resource context supplied by ConfigKit
|
|
||||||
support interpretation at handoff. Historical inspection remains an explicitly
|
|
||||||
invoked operator capability, separate from domain enforcement.
|
|
||||||
Runtime clients and callbacks remain normal Go dependencies, not YAML types.
|
|
||||||
|
|
||||||
Use `backends.<id>`, `profiles.<id>`, and `prompts.<id>.<version>` keyed mappings
|
|
||||||
assembled before typed decoding rather than independently decoding definitions and combining them
|
|
||||||
afterward. Keys own identity; do not duplicate `id`/`version` fields. These paths
|
|
||||||
describe mapping levels, not dot-splitting of literal IDs or version keys.
|
|
||||||
Multiple files can contribute to a collection; a monolithic file is not required.
|
|
||||||
When a prompt ID has multiple versions, selection without a version remains
|
|
||||||
ambiguous. This shape fits sibling inheritance without extra identity reconciliation.
|
|
||||||
|
|
||||||
Supply explicit source groups through a small construction surface compatible
|
|
||||||
with ConfigKit's `Input`/underlay concepts. Offer convenient file/directory/embedded
|
|
||||||
entry points where useful, not a parallel set of historical loading semantics.
|
|
||||||
|
|
||||||
Accept ConfigKit's labeled mapping sources for programmatic configuration. Add
|
|
||||||
only small typed conveniences justified by the minimal local-backend and
|
|
||||||
application-profile workflows. Helpers encode explicit presence and feed the same
|
|
||||||
loading lifecycle; they neither merge configurations nor infer omission from zeros.
|
|
||||||
Backend helpers also produce inputs to this lifecycle. Backend definitions,
|
|
||||||
including catalog connection and capacity settings, cross the same ConfigKit
|
|
||||||
handoff as prompts and profiles. The immutable runtime registry is built from
|
|
||||||
effective backend values. HTTP/model clients and other runtime collaborators
|
|
||||||
remain outside typed configuration.
|
|
||||||
|
|
||||||
Keep the public package a deliberate facade and implementation private. Do not
|
|
||||||
expose internal engine/domain types or introduce a parallel generic loader merely
|
|
||||||
to preserve old source options. ConfigKit's supported public types may be used
|
|
||||||
where intentional integration is simpler than maintaining equivalent wrappers.
|
|
||||||
|
|
||||||
### Sources, Tiers, And Presence
|
|
||||||
|
|
||||||
Support ordinary files/directories, embedded or caller-supplied `fs.FS`, and
|
|
||||||
programmatic definitions. Explicitly selected directories may be discovered by
|
|
||||||
PromptKit; ConfigKit handles their configuration contributions. Do not infer
|
|
||||||
priority from traversal order.
|
|
||||||
|
|
||||||
Retain the useful priority categories, lowest to highest: external
|
|
||||||
catalogs, PromptKit-owned profiles, application fallback, operator configuration,
|
|
||||||
and programmatic configuration. Express intentional priority as ordered tiers.
|
|
||||||
Within each tier, independent contributions compose strictly; overlapping
|
|
||||||
non-mapping values conflict, even when equal. Disjoint fields under the same
|
|
||||||
identity can combine. Across tiers, mappings recursively overlay.
|
|
||||||
These rules also govern backend configuration; consumers can add backend IDs or
|
|
||||||
overlay existing definitions through their selected tiers. Registry construction
|
|
||||||
validates the resulting values, not the identity of their contributors.
|
|
||||||
|
|
||||||
Use ConfigKit's source roles honestly: underlay groups are ordered defaults,
|
|
||||||
main input overlays underlays, and optional named overlay/CLI facilities retain
|
|
||||||
their upstream meaning. Underlay-only loading is supported; do not manufacture
|
|
||||||
a main document or disguise programmatic values as CLI flags. Exposing every
|
|
||||||
ConfigKit convenience through PromptKit is not a requirement.
|
|
||||||
|
|
||||||
Adopt the shared value rules: omission retains lower input; explicit zero, false,
|
|
||||||
empty string, sequence, and null replace; mappings combine recursively; empty
|
|
||||||
mappings do not clear lower children. Null remains explicit and subject to the
|
|
||||||
final schema. Provider-parameter maps obey these rules too. No hidden whole-profile
|
|
||||||
replacement, arbitrary merge callback, or new reset/delete operator belongs here.
|
|
||||||
|
|
||||||
Preserve presence into execution-ready settings so an explicit zero does not
|
|
||||||
disappear during backend/default resolution or outbound request construction.
|
|
||||||
Use ConfigKit metadata at construction, not as a per-run lookup mechanism.
|
|
||||||
|
|
||||||
### Sibling Inheritance
|
|
||||||
|
|
||||||
Use ConfigKit's reserved `extends` rather than PromptKit's `base_profile` and
|
|
||||||
custom resolver. A string identifies an immediate sibling mapping; null cancels
|
|
||||||
the relationship without deleting the child's own settings. Layering completes
|
|
||||||
before inheritance. Only effective relationships resolve, but all of them do,
|
|
||||||
including entries never selected for execution.
|
|
||||||
|
|
||||||
Children recursively overlay resolved parents. Inherited values count as explicit
|
|
||||||
input. Origins and resource context remain attached to their declaring source.
|
|
||||||
Nested relationships follow ConfigKit's destination-local rules; defaults and
|
|
||||||
application-derived values do not trigger another inheritance pass.
|
|
||||||
|
|
||||||
The keyword is reserved throughout configuration, including dynamic provider
|
|
||||||
maps; it is not opt-in. Keep runtime request provider-parameter maps ordinary
|
|
||||||
JSON-compatible data outside configuration parsing. Consumers needing a literal
|
|
||||||
provider field named `extends` supply it at that runtime boundary. Do not rename
|
|
||||||
the outbound key or introduce a second opaque configuration format to escape it.
|
|
||||||
Document and test this distinction, including any affected maintained payloads.
|
|
||||||
|
|
||||||
### Validation And Errors
|
|
||||||
|
|
||||||
ConfigKit owns acquisition, syntax, composition, inheritance, and typed-decoding
|
|
||||||
failures. PromptKit preserves those errors and validates final effective domain
|
|
||||||
values. An unused invalid effective definition fails construction; a lower-layer
|
|
||||||
value legitimately replaced by ConfigKit is not a separate validation target.
|
|
||||||
This boundary applies equally to application and maintained catalog configuration.
|
|
||||||
PromptKit does not perform catalog preflight, raw-source or intermediate-state
|
|
||||||
validation, or history-based enforcement, including equivalent test-only checks.
|
|
||||||
Catalog maintainers own publishing checks in their respective repositories.
|
|
||||||
|
|
||||||
Domain validators remain shared with runtime request validation where appropriate.
|
|
||||||
Allow schema-valid partial profiles without an `abstract` flag. Validate supplied
|
|
||||||
domain values at loading; final runnable-target requirements are checked after
|
|
||||||
profile selection and runtime overrides at execution preparation.
|
|
||||||
Do not require credentials, artifacts, or provider connectivity during construction.
|
|
||||||
|
|
||||||
Preserve absence versus explicit null in resolved settings. Null on an optional
|
|
||||||
provider setting deliberately suppresses it; it must not restore the same setting
|
|
||||||
from backend defaults. A null provider map suppresses the configured lower map,
|
|
||||||
while null entries inside a provider map remain literal JSON null values. This
|
|
||||||
supports reusable bases and deliberate removal of stale settings without changing
|
|
||||||
ConfigKit's merge rules. Framework-required settings such as timeouts have explicit
|
|
||||||
field-specific null semantics rather than blanket nil defaulting.
|
|
||||||
|
|
||||||
Preserve ConfigKit public loading errors through wrapping for `errors.As`, with a
|
|
||||||
PromptKit configuration category. Use source-aware domain/resource failures where
|
|
||||||
ConfigKit is not the owner. Do not mirror its diagnostic framework or promise
|
|
||||||
aggregation beyond what the underlying operation provides. Runtime operational
|
|
||||||
errors remain distinct from generated-content validation failures.
|
|
||||||
|
|
||||||
Prompt versions are domain identifiers, not ConfigKit `VersionRule` declarations.
|
|
||||||
Do not introduce an application schema-version policy merely to adopt ConfigKit.
|
|
||||||
|
|
||||||
## Snapshot, Resource, And Execution Boundaries
|
|
||||||
|
|
||||||
The engine captures effective definitions and referenced message text during
|
|
||||||
construction. Validate static template syntax then; render with request data
|
|
||||||
later. Resolve content references from the effective field's contributing source
|
|
||||||
context, not the last layer's directory. Ambiguous/contextless resource references
|
|
||||||
need an explicit source context or an actionable error, not an invented directory.
|
|
||||||
|
|
||||||
ConfigKit returns passive resource contexts, not snapshots of referenced text.
|
|
||||||
PromptKit explicitly opens and closes its resource handles. Caller filesystem
|
|
||||||
ownership remains unchanged. Preserve appropriate source containment without
|
|
||||||
claiming that every `fs.FS` is a security sandbox. Captured reads are not an atomic
|
|
||||||
transaction over a concurrently changing filesystem.
|
|
||||||
|
|
||||||
Definition changes require new engine construction. No watcher, background
|
|
||||||
refresh, or lazy compatibility mode is added. Applications own engine replacement;
|
|
||||||
prepared handles remain bound to their original engine and overlapping engines
|
|
||||||
retain separate capacity pools.
|
|
||||||
|
|
||||||
Request artifacts and schema preparation remain invocation work. A request can
|
|
||||||
replace the output contract, so construction need not compile unused schemas.
|
|
||||||
Prepared execution continues to freeze its effective resources. Environment
|
|
||||||
credential lookup remains at the established execution/preparation boundary,
|
|
||||||
not frozen into the configuration snapshot.
|
|
||||||
|
|
||||||
Ordinary execution uses indexed ready definitions and resolved presence values.
|
|
||||||
It performs no configuration discovery, parsing, inheritance traversal, or
|
|
||||||
provenance analysis. Context cancellation remains cooperative at controlled
|
|
||||||
boundaries; do not add detached work to pretend synchronous ConfigKit loading is
|
|
||||||
instantly interruptible.
|
|
||||||
|
|
||||||
## Explicit Operator Inspection
|
|
||||||
|
|
||||||
Add an API consumers can use to implement targeted CLI commands or debug output
|
|
||||||
for effective configuration, contributing sources, and explanations. Retain
|
|
||||||
ConfigKit inspection state with the engine's configuration snapshot and reuse
|
|
||||||
its redaction and provenance facilities.
|
|
||||||
|
|
||||||
Inspection is explicitly invoked and uses that snapshot without reloading files,
|
|
||||||
opening resources, or contacting providers. It does not predict every runtime
|
|
||||||
override, reveal execution-time credential values, or replace prepared-run details
|
|
||||||
and execution observations. No provenance traversal, explanation generation, or
|
|
||||||
configuration-report rendering occurs during ordinary prompt runs.
|
|
||||||
|
|
||||||
Expose a read-only inspection view backed by retained ConfigKit state, with
|
|
||||||
targeted path queries, source/history information, and explicit redacted effective
|
|
||||||
and explanation output. Reuse suitable public ConfigKit inspection values rather
|
|
||||||
than mirroring them. Do not expose mutable runtime definitions, resource capabilities
|
|
||||||
merely for diagnostics, or request credentials. The scope is generic targeted
|
|
||||||
inspection, not just narrow profile/prompt origin summaries.
|
|
||||||
|
|
||||||
PromptKit ships no CLI and emits no automatic debug reports. Applications own
|
|
||||||
presentation, access control, and disclosure; redaction does not make all prompt
|
|
||||||
or configuration content safe to publish.
|
|
||||||
|
|
||||||
## Required Refactoring And Coordination
|
|
||||||
|
|
||||||
- Replace profile/prompt metadata prepasses, selected-file decoding, and per-call
|
|
||||||
scans with ConfigKit collection loading and immutable indexed definitions.
|
|
||||||
- Remove the custom profile inheritance resolver and error-driven repository
|
|
||||||
overlay stack. Do not retain an old-format compatibility engine alongside the
|
|
||||||
new semantics.
|
|
||||||
- Route catalog and programmatic backend definitions through ConfigKit; retire
|
|
||||||
the separate backend-manifest decoder and pre-handoff registration validation.
|
|
||||||
Retain backend endpoint, capacity, and provider-parameter domain validation on
|
|
||||||
the final effective values.
|
|
||||||
- Retain only needed discovery and message-resource adapters; remove direct YAML
|
|
||||||
dependency usage once no remaining owner needs it. Preserve JSON Schema,
|
|
||||||
runtime JSON-value ownership, domain validation, and execution collaborators.
|
|
||||||
- Adapt maintained catalogs and PromptKit-owned definitions to the agreed keyed
|
|
||||||
format and released inheritance semantics, including keyed backend assets.
|
|
||||||
Preserve maintained name commitments and the intended shipped backend settings.
|
|
||||||
Catalog source-policy enforcement belongs to catalog maintainers. Coordinate
|
|
||||||
compatible catalog releases before consuming the changed asset format.
|
|
||||||
Publish new keyed assets directly in the existing data-only catalog projects,
|
|
||||||
recording the format transition rather than maintaining duplicate legacy assets
|
|
||||||
or a permanent conversion adapter. Catalog maintainers select release/version
|
|
||||||
handling under their own policies; old consumers must not be assumed capable
|
|
||||||
of decoding the new shape. Pin the compatible released versions in PromptKit.
|
|
||||||
This document requests no immediate changes in those repositories.
|
|
||||||
- Keep configuration conversion, normalization, and inspection connected to
|
|
||||||
ConfigKit's lifecycle instead of reconstructing effective values/history after
|
|
||||||
loading. Translate once into execution-owned values with necessary copying.
|
|
||||||
- Update canonical formats, consumer guidance, GoDoc, internal inventory, examples,
|
|
||||||
architecture boundaries, dependencies, and OSS notices when implementation lands.
|
|
||||||
|
|
||||||
## Efficiency And Validation Expectations
|
|
||||||
|
|
||||||
Target linear or near-linear work relative to actual input, output, and retained
|
|
||||||
history, accounting separately for sorting, resource reads, and domain work.
|
|
||||||
Acquire/parse each accepted source occurrence once, avoid decoding intermediate
|
|
||||||
tiers, and index definition selection. Do not duplicate inheritance expansion or
|
|
||||||
copy complete accumulated collections for every added layer.
|
|
||||||
|
|
||||||
Materialized inherited data and provenance can exceed source size. Exercise real
|
|
||||||
catalog shapes within ConfigKit's budgets; do not claim unconditional O(n) runtime
|
|
||||||
or memory. Prefer clear ownership and straightforward indexes over global caches
|
|
||||||
or elaborate optimization. Operator inspection may perform work proportional to
|
|
||||||
the requested report; that work remains off the execution path.
|
|
||||||
|
|
||||||
Tests should use real ConfigKit with offline sources and synthetic provider
|
|
||||||
collaborators. PromptKit owns its adapter, domain, lifecycle, and execution tests,
|
|
||||||
not a duplicate ConfigKit parser/graph suite. Retire tests whose only purpose was
|
|
||||||
the removed loading machinery or source-policy enforcement. Exercise catalog
|
|
||||||
integration through the final assembled configuration and resulting behavior.
|
|
||||||
Preserve meaningful execution regressions.
|
|
||||||
|
|
||||||
Representative acceptance evidence must cover:
|
|
||||||
|
|
||||||
- Minimal local-backend construction and execution without elaborate configuration.
|
|
||||||
- Embedded defaults plus operator and programmatic contributions, including
|
|
||||||
accurate inherited relative-resource origins.
|
|
||||||
- Profile inheritance, changing model/backend, canceling old relationships, and
|
|
||||||
discarding unwanted settings with documented semantics.
|
|
||||||
- Explicit zero/false/null behavior reaching execution, construction failures,
|
|
||||||
independent snapshots, and unchanged substantive preparation/execution features.
|
|
||||||
- Targeted inspection with useful redacted origins and no provenance work during
|
|
||||||
ordinary runs.
|
|
||||||
|
|
||||||
Use focused scaling/allocation benchmarks for large collections, shared bases,
|
|
||||||
and many tiers where they expose repeated-work risks. Do not add timing-based
|
|
||||||
release gates or provider-network tests.
|
|
||||||
|
|
||||||
## Migration And Completion
|
|
||||||
|
|
||||||
Ship as an intentionally incompatible minor release with concrete migration
|
|
||||||
guidance. Cover keyed files, `base_profile` to `extends`, reserved-key behavior,
|
|
||||||
recursive maps, explicit null versus omission, changed error timing, programmatic
|
|
||||||
construction, backend layering in place of registration-only restrictions, and
|
|
||||||
explicit engine replacement. Explain that overriding a model
|
|
||||||
does not discard lower endpoint, reasoning, provider, or inheritance settings.
|
|
||||||
|
|
||||||
The end state is one shared configuration model with the old loading/resolution
|
|
||||||
machinery removed, useful operator inspection, and all substantive execution
|
|
||||||
workflows still available. The six previously open design choices have been
|
|
||||||
approved and incorporated into the owning sections above. This is not a
|
|
||||||
compatibility-preserving reinterpretation of the existing configuration engine.
|
|
||||||
@@ -42,20 +42,21 @@ checkpoint equality or when a broader semantic identity design is selected.
|
|||||||
### Eager Source Validation
|
### Eager Source Validation
|
||||||
|
|
||||||
**Reason for deferral:** Exact prompt and profile inspection may already
|
**Reason for deferral:** Exact prompt and profile inspection may already
|
||||||
provide a sufficiently small validation surface. Experience from downstream
|
provide a sufficiently small validation surface alongside construction-time
|
||||||
adoption should establish whether an engine-wide operation would add enough
|
definition validation. Experience from downstream adoption should establish
|
||||||
value to justify its broader contract.
|
whether a separate operation would add enough value to justify its broader
|
||||||
|
contract.
|
||||||
|
|
||||||
Promptkit could provide an explicit offline operation that discovers and
|
Promptkit could provide an explicit offline operation that also checks schema
|
||||||
structurally validates configured prompt, profile, and schema sources without
|
resources without model generation. `NewEngine` already validates effective
|
||||||
model generation. The normal `NewEngine` path would remain lazy.
|
definitions and captures prompt content; schema resources remain
|
||||||
|
request-dependent.
|
||||||
|
|
||||||
An eager operation would need coherent handling for duplicate prompt IDs and
|
Such an operation would need coherent handling for schema syntax and transitive
|
||||||
versions, strict YAML decoding, referenced content files, profile/backend
|
references, context cancellation, and source-specific public errors. Credential
|
||||||
membership, schema syntax and transitive references, context cancellation,
|
declarations must remain separate from credential values; checking current
|
||||||
and source-specific public errors. Credential declarations must remain
|
environment availability, if supported at all, should be an explicit option
|
||||||
separate from credential values; checking current environment availability,
|
and must not expose secrets.
|
||||||
if supported at all, should be an explicit option and must not expose secrets.
|
|
||||||
|
|
||||||
Reconsider this idea after downstream use of `InspectPrompt`,
|
Reconsider this idea after downstream use of `InspectPrompt`,
|
||||||
`InspectProfile`, and fixture-based preparation demonstrates a concrete gap.
|
`InspectProfile`, and fixture-based preparation demonstrates a concrete gap.
|
||||||
|
|||||||
@@ -1,642 +0,0 @@
|
|||||||
# ConfigKit Adoption Implementation Plan
|
|
||||||
|
|
||||||
Status: ready for staged implementation under the agreed configuration boundary.
|
|
||||||
This is a planning document; writing it does not implement the feature or
|
|
||||||
authorize publishing dependencies.
|
|
||||||
|
|
||||||
## Authority And Working Method
|
|
||||||
|
|
||||||
The [ConfigKit feature roadmap](configkit.md) owns the intended end state and
|
|
||||||
approved policy. Its intentional-breaking-change direction is authoritative:
|
|
||||||
preserve substantive prompting capabilities, not the old loader, signatures,
|
|
||||||
source wrappers, incidental error ordering, or configuration syntax.
|
|
||||||
The [configuration-boundary ADR](../adr/0001-configuration-boundary.md) records
|
|
||||||
the ownership decision: PromptKit interprets and validates the final effective
|
|
||||||
configuration supplied by ConfigKit. Presence/null information and effective
|
|
||||||
resource context are part of that handoff. Source history serves explicit
|
|
||||||
operator inspection, not validation or catalog enforcement. This applies to
|
|
||||||
tests as well as production code.
|
|
||||||
|
|
||||||
Implement the stages below in order, one stage per coding-agent prompt. Each
|
|
||||||
stage includes its tests and affected current-state documentation. Read
|
|
||||||
[development](../development.md) and the applicable policies before starting;
|
|
||||||
in particular, follow [agents](../policy/agents.md),
|
|
||||||
[architecture](../policy/architecture.md), [testing](../policy/testing.md),
|
|
||||||
[documentation](../policy/documentation.md), and [OSS](../policy/oss.md).
|
|
||||||
Catalog work also follows each catalog repository's own instructions and release
|
|
||||||
procedure. Do not turn this plan's internal organization into new public promises.
|
|
||||||
|
|
||||||
Begin each stage by inspecting the worktree and earlier stages. Commit completed
|
|
||||||
units under the agent policy. Keep stages buildable; temporary private wiring
|
|
||||||
needed to connect successive stages must be removed by Stage 11. Do not ship a
|
|
||||||
legacy mode, dual loader, or old-format conversion layer. Do not add prototype
|
|
||||||
applications or a generic configuration framework of our own.
|
|
||||||
|
|
||||||
Planning baselines:
|
|
||||||
|
|
||||||
- PromptKit runtime v0.11.2, commit
|
|
||||||
`12d6d80024e80c552d122a076c22429bf6f2f6c2`; approved roadmap at `521291d`.
|
|
||||||
- ConfigKit v0.5.0, commit
|
|
||||||
`dc09959cca4e105f49ec316c6d4ba585bd2340bd`, module
|
|
||||||
`gitea.maximumdirect.net/eric/ConfigKit`.
|
|
||||||
- Both catalog dependencies currently select `/v2` at `v2.20260926.0`.
|
|
||||||
|
|
||||||
Use ConfigKit v0.5.0 for this work. Do not silently upgrade to a different
|
|
||||||
contract. The catalog asset migration needs coordinated releases, not a committed
|
|
||||||
workspace or local replacement. Changing the backend asset contract is broader
|
|
||||||
than the catalog profile-data compatibility exception: migrate the catalogs to
|
|
||||||
`/v3` with `v3.YYYYMMDD.N` tags, preserving their date/revision convention and
|
|
||||||
updating their own release documentation accordingly. Select exact dates and
|
|
||||||
unused revisions at publication; record versions and full commits consumed.
|
|
||||||
Publishing any repository requires separate authority.
|
|
||||||
|
|
||||||
## Implementation Decisions
|
|
||||||
|
|
||||||
### Construction Surface
|
|
||||||
|
|
||||||
Keep `NewEngine(Config, ...Option)` and the execution/injection surface where it
|
|
||||||
still serves its substantive purpose. Replace profile/prompt source options with
|
|
||||||
these fields and helpers; this is an intentional API break:
|
|
||||||
|
|
||||||
```go
|
|
||||||
type ConfigurationSources struct {
|
|
||||||
Underlays []configkit.Underlay
|
|
||||||
Main *configkit.Input
|
|
||||||
}
|
|
||||||
|
|
||||||
// Add Sources; remove PromptDir and ProfileDir.
|
|
||||||
// Retain SchemaDir, Timeout, HTTPClient, and TransientRetries.
|
|
||||||
type Config struct { /* existing non-definition settings; Sources ConfigurationSources */ }
|
|
||||||
|
|
||||||
func DirectoryInputs(directory string) ([]configkit.Input, error)
|
|
||||||
func FilesystemInputs(label string, filesystem fs.FS, root string) ([]configkit.Input, error)
|
|
||||||
```
|
|
||||||
|
|
||||||
The pointer distinguishes omitted main input without inspecting ConfigKit's
|
|
||||||
opaque Source. Nil main becomes `configkit.NoSource()`; a supplied invalid Input
|
|
||||||
is not silently ignored. Prepend two private underlays: external catalogs, then
|
|
||||||
PromptKit-owned definitions. Append caller underlays in their supplied order;
|
|
||||||
main is last and keeps its ConfigKit meaning. Imports follow each Input's flag.
|
|
||||||
Do not expose named overlays, CLI overrides, or VersionRule through another
|
|
||||||
PromptKit abstraction in this feature.
|
|
||||||
|
|
||||||
Show the usual application categories as caller underlays ordered fallback,
|
|
||||||
operator, programmatic, with main omitted. For a single highest-priority document,
|
|
||||||
main is also useful. These are explicit selections, not directory-name heuristics
|
|
||||||
or a second automatic precedence algorithm. Each directory helper returns inputs
|
|
||||||
for one strict-composition group, recursively finding `.yaml` and `.yml` files;
|
|
||||||
sorted discovery makes diagnostics reproducible, not one file higher priority.
|
|
||||||
An explicitly selected missing/unreadable root fails; an empty directory supplies
|
|
||||||
no contributions. Helpers set imports false; callers may enable them explicitly.
|
|
||||||
Use one ConfigKit filesystem namespace per helper invocation and root resource
|
|
||||||
contexts at the supplied directory. Single files use ConfigKit directly.
|
|
||||||
|
|
||||||
Remove `WithPromptFS`, `WithPromptFile`, `WithProfileFS`, `WithProfileFile`,
|
|
||||||
`WithFallbackProfileFS`, `WithProfiles`, and `WithBackend` at public cutover.
|
|
||||||
Retain schema source options and runtime collaborator injection. Backend
|
|
||||||
configuration moves into ConfigKit inputs as specified below. Do not require any
|
|
||||||
prompts to exist merely to construct an engine for profile inspection.
|
|
||||||
|
|
||||||
Programmatic configuration uses labeled `configkit.Mapping` inputs. Replace the
|
|
||||||
old zero-sentinel `Profile`/`OpenAICompatibleProfileConfig` pair with one small
|
|
||||||
`ProfileConfig` convenience and a source builder:
|
|
||||||
|
|
||||||
```go
|
|
||||||
type ProfileConfig struct {
|
|
||||||
Extends, Backend, Endpoint, Model *string
|
|
||||||
Temperature, TopP *float64
|
|
||||||
MaxTokens, TimeoutSeconds *int
|
|
||||||
ServiceTier, ReasoningEffort, APIKeyEnv *string
|
|
||||||
APIKeyRequired *bool
|
|
||||||
ExtraParams map[string]any
|
|
||||||
}
|
|
||||||
func ProfileSource(label string, profiles map[string]ProfileConfig) configkit.Source
|
|
||||||
```
|
|
||||||
|
|
||||||
Nil convenience fields mean omitted; nonnil pointers preserve zero/false/empty.
|
|
||||||
Nil ExtraParams omits the map; a nonnil empty map supplies an empty mapping, not a
|
|
||||||
clear operation. Use raw Mapping for explicit null, prompt definitions, or richer
|
|
||||||
configuration. Do not introduce a public generic optional-value type just for
|
|
||||||
this helper. Convert fields explicitly to ConfigKit-supported built-in values,
|
|
||||||
then call Mapping for snapshot ownership and load-time errors. Do not marshal
|
|
||||||
through YAML or infer absence from a value. Helpers encode values; domain
|
|
||||||
validation occurs on the final configuration at handoff.
|
|
||||||
|
|
||||||
Replace the registration-only Backend value with a presence-aware convenience:
|
|
||||||
|
|
||||||
```go
|
|
||||||
type BackendConfig struct {
|
|
||||||
Endpoint, APIKeyEnv *string
|
|
||||||
ConcurrencyLimit, QueueCapacity *int
|
|
||||||
ExtraParams map[string]any
|
|
||||||
}
|
|
||||||
func BackendSource(label string, backends map[string]BackendConfig) configkit.Source
|
|
||||||
func LocalBackend(endpoint string, concurrencyLimit int) BackendConfig
|
|
||||||
```
|
|
||||||
|
|
||||||
Use the same omission/pointer/map conventions as ProfileSource. LocalBackend
|
|
||||||
supplies endpoint and concurrency pointers and leaves the remaining fields
|
|
||||||
omitted; callers choose the ID as the map key. Neither helper validates settings
|
|
||||||
before ConfigKit processes them. For example, place BackendSource with
|
|
||||||
`map[string]BackendConfig{"local": LocalBackend(endpoint, 4)}` in an application
|
|
||||||
underlay. Raw Mapping supports explicit null and combined backend/profile/prompt
|
|
||||||
documents. Remove the old Backend registration value and its reservation rules;
|
|
||||||
retain useful built-in ID constants. All backend contributions follow the same
|
|
||||||
strict-composition and overlay rules as other configuration.
|
|
||||||
|
|
||||||
### Typed Configuration And Execution Values
|
|
||||||
|
|
||||||
Add `internal/configuration` as the cohesive ConfigKit adapter. Its private
|
|
||||||
document DTO has `backends: map[string]backendConfig`,
|
|
||||||
`profiles: map[string]profileConfig`, and
|
|
||||||
`prompts: map[string]map[string]promptConfig`. Collection entries are mappings,
|
|
||||||
not nullable definition pointers. No duplicate identity fields; ConfigKit consumes
|
|
||||||
`extends`, so no DTO field or second resolver handles it. Use `configkit.Keys`,
|
|
||||||
`Key`, and `Index` for semantic paths, never split IDs on dots.
|
|
||||||
|
|
||||||
Profile DTO fields match ProfileConfig's YAML spellings (`backend`,
|
|
||||||
`reasoning_effort`, etc.). Nullable pointers plus ConfigKit presence metadata
|
|
||||||
distinguish absent, value, and explicit null. Store that distinction in small
|
|
||||||
private execution-owned fields or presence bits, not retained metadata queries.
|
|
||||||
Runtime/domain packages must not import ConfigKit. Do not duplicate its tree,
|
|
||||||
merge engine, inheritance graph, diagnostics, or presence framework.
|
|
||||||
|
|
||||||
Backend DTO fields match BackendConfig using `endpoint`, `api_key_env`,
|
|
||||||
`concurrency_limit`, `queue_capacity`, and `extra_params`. Validate the final
|
|
||||||
endpoint as required, absolute HTTP(S) with the existing URL rules. An absent or
|
|
||||||
null API-key environment name means no lookup; a supplied name follows existing
|
|
||||||
normalization and naming rules. Absent/null concurrency means zero (unlimited);
|
|
||||||
supplied values must be nonnegative. Absent/null queue capacity selects the
|
|
||||||
existing default of 1024 for positive concurrency and no queue for unlimited
|
|
||||||
concurrency. Explicit queue capacity, including zero, requires positive
|
|
||||||
concurrency and must satisfy existing nonnegative/overflow checks. Null provider
|
|
||||||
maps mean no configured extra parameters; entries may contain JSON null.
|
|
||||||
Required endpoint failures, unknown fields, and domain errors apply to final
|
|
||||||
definitions, including unused ones. Registry construction consumes these final
|
|
||||||
normalized values; it does not replay registration or source-reservation checks.
|
|
||||||
|
|
||||||
Use ConfigKit's Resolve/Validate callbacks only on its effective post-layering,
|
|
||||||
post-inheritance typed values. These public lifecycle hooks implement the domain
|
|
||||||
handoff and record normalization; they are not hooks for inspecting earlier
|
|
||||||
contributions. Final presence lookup and resource-context lookup are sufficient
|
|
||||||
for interpretation. History/Explain/HistoricalPaths are operator inspection
|
|
||||||
facilities, not inputs to construction-time policy checks.
|
|
||||||
|
|
||||||
Use the existing prompt fields under each keyed version, without `id`/`version`.
|
|
||||||
Use pointers for `content`, `content_file`, and optional clearable strings;
|
|
||||||
preserve existing input, cache-control, output-contract, and role rules.
|
|
||||||
`messages` must be nonempty; each message has exactly one nonnull content source.
|
|
||||||
An explicitly supplied empty inline string is a valid template. Null content
|
|
||||||
removes that selection; null content_file removes its reference. Output remains a
|
|
||||||
required mapping with valid format and validation mode. Optional description,
|
|
||||||
session template, default profile, cache control, inputs, and schema path can be
|
|
||||||
cleared where their domain permits absence. Null required role/format/mode or a
|
|
||||||
null whole definition is invalid. Scalar `required` and `repair_attempts` retain
|
|
||||||
their false/zero defaults and are not nullable. Null inputs means no declarations.
|
|
||||||
Unknown fields are ConfigKit errors, including old `id`, `version`, and
|
|
||||||
`base_profile`. Prompt versions must be string mapping keys.
|
|
||||||
|
|
||||||
Literal definition keys remain exact; require nonblank identity but do not
|
|
||||||
normalize/rekey maps after ConfigKit has resolved relationships. Update selection
|
|
||||||
to use exact nonblank IDs/versions rather than silently trimming them into a
|
|
||||||
different identity. Scalar normalization such as endpoint trimming is recorded
|
|
||||||
through ConfigKit's resolution callbacks when
|
|
||||||
mutating its typed configuration. Message roles and cache control are normalized
|
|
||||||
on execution-owned copies instead: ConfigKit v0.5.0 treats sequence mutations as
|
|
||||||
whole-value replacements, losing unchanged content_file origins. Inspection
|
|
||||||
retains the effective input message metadata and its source context. Field
|
|
||||||
references to definition IDs remain exact as well.
|
|
||||||
|
|
||||||
Profile field rules after configuration overlay/inheritance:
|
|
||||||
|
|
||||||
| Fields | Absent | Explicit value | Explicit null |
|
|
||||||
| --- | --- | --- | --- |
|
|
||||||
| `backend` | No selected backend | Exact registered ID; blank means none | No backend |
|
|
||||||
| `endpoint` | Selected backend endpoint, if any | Normalize/validate nonblank URL; empty clears | Clear, do not recover backend endpoint |
|
|
||||||
| `model` | Unspecified | Nonblank supplied model; empty clears | Unspecified |
|
|
||||||
| `temperature`, `top_p`, `max_tokens` | Unspecified | Validate existing bounds; zero is explicit and sent | Suppress outbound control |
|
|
||||||
| `reasoning_effort`, `service_tier` | Unspecified | Existing normalization; blank suppresses | Suppress outbound control |
|
|
||||||
| `timeout_seconds` | Framework 600-second baseline | Nonnegative; zero disables generation deadline | Reset to framework baseline, not lower profile timeout |
|
|
||||||
| `api_key_env` | Selected backend's optional lookup name | Trimmed value; blank disables lookup | Disable lookup, do not recover backend name |
|
|
||||||
| `api_key_required` | False | Explicit boolean, including false overriding true | False |
|
|
||||||
| `extra_params` | Backend defaults | ConfigKit-composed profile map, recursively overlaid on backend map | Suppress entire configured provider map |
|
|
||||||
|
|
||||||
Profile API-key environment names follow the existing lookup contract, including
|
|
||||||
host-supported non-shell-identifier names. Do not extend the backend's naming
|
|
||||||
regular expression to profiles. Trimming, explicit clearing, optional lookup,
|
|
||||||
and direct request-key precedence remain unchanged.
|
|
||||||
|
|
||||||
Backend registry defaults are applied once during snapshot construction, after
|
|
||||||
effective profile inheritance. Keep them out of the inheritance input: a parent's
|
|
||||||
derived endpoint/key/defaults must not follow a child that changes backend.
|
|
||||||
Provider map null entries are literal JSON null; empty mappings retain children.
|
|
||||||
For backend-to-profile provider-map overlay, use a narrow JSON-value merge helper
|
|
||||||
with the same recursive-map/value-replacement semantics; it is not a configuration
|
|
||||||
loader. Both backend and profile configuration maps have already passed through
|
|
||||||
ConfigKit's reserved-key semantics; only request-level maps bypass that lifecycle.
|
|
||||||
|
|
||||||
Record any defaults/normalizations written into the typed configuration via
|
|
||||||
ConfigKit Resolve, with dependencies on relevant configuration paths. Do not
|
|
||||||
project runtime collaborators or credentials into the configuration tree just to
|
|
||||||
report them. Inspection includes effective backend configuration. It describes
|
|
||||||
resolved configuration, while InspectProfile and preparation describe
|
|
||||||
backend-completed execution settings. Make this boundary
|
|
||||||
explicit rather than inventing synthetic provenance for registry defaults.
|
|
||||||
|
|
||||||
Retain APIKeyRequired's direct-key workflow: true suppresses a backend-derived
|
|
||||||
environment source. An explicitly configured profile environment source, or a
|
|
||||||
request environment override, remains usable; false restores ordinary optional
|
|
||||||
credential semantics. No credential values belong in the DTO, inspection, or
|
|
||||||
construction-time environment resolution. Raw profile `api_key` remains invalid.
|
|
||||||
|
|
||||||
Validate all effective supplied values and referenced backend IDs at construction.
|
|
||||||
Do not require a complete model/endpoint there or reject unused partial profiles.
|
|
||||||
Do not require a prompt default_profile to be runnable before a request can
|
|
||||||
override it. Final target completeness is checked after runtime overrides.
|
|
||||||
InspectProfile can report a partial target without credential or provider access.
|
|
||||||
Keep request override contracts otherwise unchanged, including complete nonempty
|
|
||||||
runtime ExtraParams replacement and reasoning clearing. A runtime literal
|
|
||||||
`extra_params["extends"]` remains ordinary provider data. Do not expand this work
|
|
||||||
into a redesign of every request override.
|
|
||||||
|
|
||||||
### Snapshot And Inspection Design
|
|
||||||
|
|
||||||
The loader returns indexed, execution-owned definitions plus a private retained
|
|
||||||
ConfigKit result. An adapter may implement the runner's narrow lookup interfaces
|
|
||||||
over maps during migration; no repository chain, fallback-on-error, or per-call
|
|
||||||
configuration load survives. Preserve defensive ownership across injected clients
|
|
||||||
and public results. Cache prompt definition hashes at construction; hash effective
|
|
||||||
captured content, not a stale resource path. Hash bytes need not match the old
|
|
||||||
configuration format. Preserve rendered-prompt hashing semantics.
|
|
||||||
|
|
||||||
Resolve each effective content_file using its exact field origin and ConfigKit
|
|
||||||
SourceContext. Join its relative name to BaseDir inside the returned rooted FS;
|
|
||||||
reject absolute/outside-root paths and close every opened handle. Preserve path
|
|
||||||
text, including meaningful whitespace, after separately checking nonblankness.
|
|
||||||
Do not choose the child's source for an inherited reference. If there is no unique
|
|
||||||
contributing source with context, fail with an actionable path/source error; the
|
|
||||||
caller can supply an explicit Input filesystem context or override the field.
|
|
||||||
Retain no resource capability for the purpose of public diagnostics.
|
|
||||||
|
|
||||||
Capture message text and parse message/session template syntax at construction
|
|
||||||
with the same function names/options used for runtime rendering. Do not invoke
|
|
||||||
input helpers or require Vars/artifacts while parsing. Reuse compiled templates
|
|
||||||
where practical, binding request-specific input functions on safe per-call clones;
|
|
||||||
never mutate a shared template function closure with request state. Runtime
|
|
||||||
missing-variable/input failures remain runtime errors.
|
|
||||||
|
|
||||||
Expose a concrete read-only facade value:
|
|
||||||
|
|
||||||
```go
|
|
||||||
func (e *Engine) InspectConfiguration() (ConfigurationInspection, error)
|
|
||||||
func (v ConfigurationInspection) Metadata() configkit.Metadata
|
|
||||||
func (v ConfigurationInspection) Sources() []configkit.SourceInfo
|
|
||||||
func (v ConfigurationInspection) Diagnostics() []configkit.Diagnostic
|
|
||||||
func (v ConfigurationInspection) Explain(path configkit.Path) (configkit.Explanation, bool)
|
|
||||||
func (v ConfigurationInspection) WriteEffective(w io.Writer) error
|
|
||||||
func (v ConfigurationInspection) WriteExplanation(w io.Writer, path configkit.Path) error
|
|
||||||
```
|
|
||||||
|
|
||||||
Store an unexported narrow implementation reference; do not expose Result.Config,
|
|
||||||
ConfigKit SourceContext, mutable runtime values, or internal types. Delegate
|
|
||||||
queries and redaction to the retained result. Nil Engine returns ErrInvalidConfig;
|
|
||||||
the zero inspection view has empty query results and writer operations return
|
|
||||||
ErrInvalidConfig. Writer errors remain inspectable. Mark credential-shaped dynamic
|
|
||||||
keys sensitive using Definition.Secret; the existing secret-key vocabulary can
|
|
||||||
serve as a redaction predicate, not a source-rejection rule. Do not redact
|
|
||||||
`api_key_env` names as credential values. This is not a
|
|
||||||
promise that arbitrary prompt/provider content is safe to disclose.
|
|
||||||
|
|
||||||
Keep the runner independent of this view. Inspection calls perform no discovery,
|
|
||||||
resource reads, environment credential lookup, or provider operations. Schemas,
|
|
||||||
artifacts, preparation ownership, generation admission, retry/repair, observations,
|
|
||||||
and cancellation remain at their established runtime boundaries.
|
|
||||||
|
|
||||||
## Stage 1 — Add The Dependency And Typed Loading Core
|
|
||||||
|
|
||||||
**Completed:** 2026-09-30.
|
|
||||||
|
|
||||||
Scope: module metadata, `internal/configuration` DTOs, real ConfigKit loading,
|
|
||||||
and its focused tests. Leave engine assembly on the old path for this stage.
|
|
||||||
|
|
||||||
1. Add ConfigKit v0.5.0; update go.sum and factual OSS notices for it and its
|
|
||||||
selected transitive dependencies. Preserve project licenses and the Go floor.
|
|
||||||
2. Add keyed backend/profile/prompt DTOs and a private load entry point accepting
|
|
||||||
ConfigKit Inputs and underlays. Call LoadLayered once, with no VersionRule or
|
|
||||||
secret resolution bindings for API credentials. Use callback-scoped
|
|
||||||
Resolve/Validate correctly.
|
|
||||||
3. Implement effective presence extraction and the field rules above. Validate
|
|
||||||
final domain values through existing helpers, resolving backend references
|
|
||||||
against the final backend map; profiles have no final target requirement yet.
|
|
||||||
Record normalization/default mutations, not arbitrary DTO bookkeeping fields.
|
|
||||||
4. Preserve `*configkit.LoadError` through wrapping. Domain callback messages
|
|
||||||
identify paths/problems without interpolating raw values or credentials.
|
|
||||||
5. Add representative real-loader tests: keyed literal IDs, strict group conflict,
|
|
||||||
recursive cross-tier overlay, a shared base with explicit zero/null/false, an
|
|
||||||
unused invalid effective definition, and preservation of a ConfigKit load
|
|
||||||
error. Cover an invalid lower-layer domain value replaced by a valid final
|
|
||||||
value: construction succeeds. Do not reproduce ConfigKit's parser/inheritance
|
|
||||||
matrix or add preflight/source-policy tests.
|
|
||||||
|
|
||||||
Exit: focused tests and `go test ./...` pass; old engine behavior is not presented
|
|
||||||
as migrated. Internal inventory documents only the loader now actually present.
|
|
||||||
|
|
||||||
## Stage 2 — Add Source Discovery And Prompt Resource Capture
|
|
||||||
|
|
||||||
**Completed:** 2026-09-30.
|
|
||||||
|
|
||||||
Scope: source selection helpers and construction-time prompt materialization.
|
|
||||||
|
|
||||||
1. Implement private directory/FS discovery behind the chosen helper signatures;
|
|
||||||
wrap discovered paths as ConfigKit sources, retaining explicit root context.
|
|
||||||
Reuse useful filecatalog path handling without carrying profile metadata scans.
|
|
||||||
2. Materialize effective prompt message text using field origins and passive
|
|
||||||
contexts. Move reusable prompt/input/cache/output validation into the loader
|
|
||||||
or shared domain owner; do not copy entire legacy parsers.
|
|
||||||
3. Build immutable prompt indexes by exact ID/version and an ambiguity indicator
|
|
||||||
for omitted versions. Capture declared-input inspection data and prompt hashes.
|
|
||||||
4. Validate static template syntax using the renderer's grammar/function setup.
|
|
||||||
Refactor that setup into one private owner if needed; do not render at load.
|
|
||||||
5. Test native and embedded sources, imported/inherited relative content from a
|
|
||||||
different directory, explicit mapping context, contextless failure, handle
|
|
||||||
closure, malformed static syntax, and resource mutation after capture. Use
|
|
||||||
minimal fixtures, sharing scenarios where the same adapter is exercised.
|
|
||||||
|
|
||||||
Exit: loader creates usable frozen prompt definitions without request data.
|
|
||||||
`go test ./internal/configuration ./internal/prompt ./internal/promptdef` passes;
|
|
||||||
existing execution tests still work through the old assembly until cutover.
|
|
||||||
|
|
||||||
## Stage 3 — Make Execution Settings Presence-Aware
|
|
||||||
|
|
||||||
**Completed:** 2026-09-30.
|
|
||||||
|
|
||||||
Scope: domain settings, target resolution, wire-presence propagation, profile
|
|
||||||
inspection. No new source discovery in execution packages.
|
|
||||||
|
|
||||||
1. Build the immutable backend registry from the final loaded backend map. Add
|
|
||||||
execution-owned presence/suppression state and construct indexed profiles
|
|
||||||
from loader output plus that registry. Apply registry-derived
|
|
||||||
values after inheritance, using the field table above.
|
|
||||||
2. Replace scalar-zero/blank merge assumptions in target resolution. Configuration
|
|
||||||
explicit zero reaches injected clients and the built-in request body; null does
|
|
||||||
not resurrect a provider setting. Extend existing TargetPresence plumbing
|
|
||||||
through preparation/results as necessary rather than creating a parallel path.
|
|
||||||
3. Preserve request-level override precedence, API-key availability behavior,
|
|
||||||
endpoint-only profiles, backend identity/capacity, and partial-profile runtime
|
|
||||||
completion. Move final completeness checks after overrides. InspectProfile
|
|
||||||
returns partial settings without treating missing model/endpoint as load errors.
|
|
||||||
4. Test explicit numeric zero versus omission, map overlay and null, backend
|
|
||||||
change without derived defaults leaking from parent, timeout null/zero,
|
|
||||||
credential-source suppression, and completion through request overrides.
|
|
||||||
Include one built-in-client wire test, not just an injected-client assertion.
|
|
||||||
|
|
||||||
Exit: focused domain/usecase/client tests pass. Any temporary adaptation for the
|
|
||||||
old assembly is private, clearly limited to reaching Stage 7, and does not become
|
|
||||||
a compatibility promise. Update affected target/presence GoDoc with this change.
|
|
||||||
|
|
||||||
## Stage 4 — Migrate The OpenRouter Catalog Assets
|
|
||||||
|
|
||||||
**Completed:** 2026-09-30. Published OpenRouter catalog `v3.20260930.0`
|
|
||||||
from `f17d208cb7ad1fb043cda2266cd3011d11b40aa0`.
|
|
||||||
|
|
||||||
Scope: `../promptkit-backend-openrouter` only. Catalog maintainers own source and
|
|
||||||
publishing checks here; PromptKit consumes its configuration through ConfigKit.
|
|
||||||
|
|
||||||
1. Read that repository's instructions; preserve unrelated work. Convert each
|
|
||||||
profile asset to `profiles: {literal-id: {settings...}}`, preserving inventory
|
|
||||||
and values, changing `base_profile` to `extends`, and removing embedded ID.
|
|
||||||
Keep per-profile files and the exported FS/Root package; no dependency on
|
|
||||||
PromptKit and no second legacy asset tree.
|
|
||||||
2. Audit zero/empty values that previously meant unspecified. Remove those when
|
|
||||||
omission was intended; retain intentional explicit values. Review any dynamic
|
|
||||||
provider `extends` keys rather than silently changing outbound payloads.
|
|
||||||
3. Replace `catalog/backend.json` with `catalog/backend.yml` containing a
|
|
||||||
`backends: {openrouter: {settings...}}` mapping. Preserve endpoint, API-key
|
|
||||||
environment name, concurrency, queue, and extra-parameter values; remove the
|
|
||||||
duplicate id and old manifest schema_version. The file is an ordinary ConfigKit
|
|
||||||
input, not a separately decoded manifest. Update embedded assets, module path
|
|
||||||
and imports to `/v3`, tests, and catalog format/release documentation, with an
|
|
||||||
incompatible-format migration note. Keep FS/Root as the minimal export surface.
|
|
||||||
Catalog-local publishing checks stay in that repository. No provider probes.
|
|
||||||
4. Validate under its procedure and commit. If publication is authorized, publish
|
|
||||||
the next unused `v3.YYYYMMDD.N`; otherwise report the exact commit awaiting
|
|
||||||
publication. The date/revision is selected then, not guessed in this plan.
|
|
||||||
|
|
||||||
Exit: one reviewed migrated catalog commit; existing immutable tags still serve
|
|
||||||
old consumers. The new major module path identifies the changed asset contract;
|
|
||||||
old consumers retain the published `/v2` assets.
|
|
||||||
|
|
||||||
## Stage 5 — Migrate The Rakestrawhome Catalog Assets
|
|
||||||
|
|
||||||
**Completed:** 2026-09-30. Published Rakestrawhome catalog `v3.20260930.0`
|
|
||||||
from `993443d88c451901f4c917fa00c204fd3b54d28d`.
|
|
||||||
|
|
||||||
Scope: `../promptkit-backend-rakestrawhome` only. Apply the same keyed-format,
|
|
||||||
presence, notice, validation, and publication treatment as Stage 4 to its smaller
|
|
||||||
inventory, including the `/v3` transition and keyed `backends.rakestrawhome`
|
|
||||||
definition in `catalog/backend.yml`. Preserve backend endpoint, credentials
|
|
||||||
metadata, and capacity policy.
|
|
||||||
Do not couple its date/revision to OpenRouter's release.
|
|
||||||
|
|
||||||
Exit: both compatible catalog versions are published and ordinarily resolvable
|
|
||||||
before Stage 6 pins them. If permission or publication is unavailable, stop at
|
|
||||||
this external dependency checkpoint; do not commit a local replace, guessed
|
|
||||||
version, or legacy-to-new runtime converter to bypass it.
|
|
||||||
|
|
||||||
## Stage 6 — Integrate Catalog Sources And Maintained Profiles
|
|
||||||
|
|
||||||
**Completed:** 2026-09-30.
|
|
||||||
|
|
||||||
Scope: `internal/catalog`, maintained definitions, catalog integration tests,
|
|
||||||
dependency pins/notices. Keep this separate from public API migration.
|
|
||||||
|
|
||||||
1. Pin the published compatible catalog versions and exact attribution commits.
|
|
||||||
Update imports to `/v3`. Replace manifest decoding with ConfigKit file Inputs.
|
|
||||||
2. Supply backend and profile assets as ConfigKit Inputs in the first
|
|
||||||
underlay. Supply PromptKit-owned keyed Mapping definitions as the next group:
|
|
||||||
standard extends the selected catalog's current maintained base; auto retains
|
|
||||||
its current direct model/backend definition. Preserve both literal names.
|
|
||||||
3. Retire source-layout allowlists, catalog preflight, source ownership/reserved
|
|
||||||
prefix checks, connection-field prohibitions, and secret-key source rejection.
|
|
||||||
The adapter selects known asset entry points and discovers configuration files;
|
|
||||||
ConfigKit acquires their contents. Domain validation sees only the final
|
|
||||||
assembled configuration. Consumers may overlay catalog backends, profiles,
|
|
||||||
and their bases through the same configuration model.
|
|
||||||
4. Test real imported assets offline with the new loader, maintained-name
|
|
||||||
resolution, consumer override of a maintained base/backend, and rejection of
|
|
||||||
an invalid final backend. Test a replaced lower-layer domain error succeeds.
|
|
||||||
Do not independently validate catalog-only intermediate configurations, parse
|
|
||||||
raw assets in PromptKit tests, or audit origins to establish ownership.
|
|
||||||
|
|
||||||
Exit: catalog and owned definitions participate in the new keyed configuration
|
|
||||||
model without a legacy profile parser on that path. Catalog integration tests
|
|
||||||
pass against released modules with `GOWORK=off` and no replacements.
|
|
||||||
|
|
||||||
## Stage 7 — Cut Over Engine Assembly And Public Construction
|
|
||||||
|
|
||||||
**Completed:** 2026-09-30.
|
|
||||||
|
|
||||||
Scope: `engine.go`, configuration facade/helpers, `internal/engine`, facade
|
|
||||||
adapters, and migration of their direct test setup.
|
|
||||||
|
|
||||||
1. Introduce the Construction Surface exactly as specified above. Remove the
|
|
||||||
superseded source/profile options and convenience types rather than keeping
|
|
||||||
deprecated pass-through wrappers. Keep intentional runtime injection APIs.
|
|
||||||
2. Load the combined backend/profile/prompt configuration, construct its runtime
|
|
||||||
registry and indexed snapshot, then assemble renderer, validator, capacity
|
|
||||||
manager, and runner. Return no Engine on construction failure.
|
|
||||||
Translate construction errors to ErrInvalidConfig while preserving LoadError
|
|
||||||
via errors.As. Resource errors include semantic path and safe source context.
|
|
||||||
3. Connect indexed prompt/profile lookups to preparation and inspection. Missing
|
|
||||||
selections keep meaningful not-found errors; prompt version ambiguity stays a
|
|
||||||
selection error. Old parse/load error categories need not be manufactured for
|
|
||||||
errors now detected at construction.
|
|
||||||
4. Move root test fixtures/setup and maintained example construction to ConfigKit
|
|
||||||
sources, keyed definitions, ProfileSource, and BackendSource so all consumers
|
|
||||||
still compile. Update local-backend examples to choose their ID as a map key.
|
|
||||||
Update expected timing only for the deliberate eager-validation change. Keep
|
|
||||||
behavioral assertions for prompting, overrides, and errors.
|
|
||||||
5. Add public tests for a minimal custom local backend and programmatic profile,
|
|
||||||
plus embedded defaults/operator/programmatic tiers. No elaborate consumer
|
|
||||||
setup should be needed merely to send one prompt.
|
|
||||||
|
|
||||||
Exit: the public Engine uses only the new loading path. Root/package tests pass;
|
|
||||||
all package/example sources compile. Update GoDoc and consumer construction
|
|
||||||
guidance in this stage so removed options are not presented as usable.
|
|
||||||
|
|
||||||
## Stage 8 — Complete Frozen Rendering And Lifecycle Integration
|
|
||||||
|
|
||||||
**Completed:** 2026-09-30.
|
|
||||||
|
|
||||||
Scope: renderer/runtime consumption of the snapshot, ownership, and lifecycle
|
|
||||||
regressions. Stage 2 already owns acquisition and static syntax validation.
|
|
||||||
|
|
||||||
1. Use captured text and parsed templates without per-run configuration reads,
|
|
||||||
parsing, or inheritance. Bind input helpers per invocation safely; keep missing
|
|
||||||
request data errors and direct-session template bypass behavior.
|
|
||||||
2. Verify preparation and execution still handle artifacts and schema compilation
|
|
||||||
at invocation time, honor output overrides, freeze prepared resources, and
|
|
||||||
keep prepared handles bound to their engine. Credentials remain at runtime.
|
|
||||||
3. Test an old engine after configuration/resource edits and a newly constructed
|
|
||||||
engine observing the edits. Test concurrent renders with different Vars/input
|
|
||||||
helpers and no data leakage. Use an FS that rejects reads after construction
|
|
||||||
to demonstrate no configuration/content-file reopening during ordinary runs;
|
|
||||||
do not incorrectly prohibit request artifact/schema reads.
|
|
||||||
4. Preserve appended-message ordering, structural repair, retries, admission,
|
|
||||||
cancellation, observers, and result metadata through existing regression
|
|
||||||
tests; add only distinct missing cross-boundary cases.
|
|
||||||
|
|
||||||
Exit: focused race tests pass for assembled rendering/preparation and lifecycle.
|
|
||||||
No retained callbacks capture ConfigKit resolution contexts or request state.
|
|
||||||
|
|
||||||
## Stage 9 — Expose Explicit Configuration Inspection
|
|
||||||
|
|
||||||
**Completed:** 2026-09-30.
|
|
||||||
|
|
||||||
Scope: the read-only facade and retained ConfigKit inspection adapter.
|
|
||||||
|
|
||||||
1. Implement ConfigurationInspection and its specified methods without exposing
|
|
||||||
Result.Config or filesystem capabilities. Retain a private narrow view of the
|
|
||||||
ConfigKit result; no regeneration of origins from runtime values.
|
|
||||||
2. Ensure effective/explanation reports use ConfigKit redaction, including the
|
|
||||||
selected sensitivity predicate. Preserve safe value-free domain diagnostics.
|
|
||||||
3. Test a targeted literal-key path with layered/inherited history, detached
|
|
||||||
returned slices/records, writer failures, zero/nil behavior, and redacted secret
|
|
||||||
shaped provider values. Demonstrate inspection still works after source removal
|
|
||||||
and makes no filesystem/provider calls.
|
|
||||||
4. Document construction inspection versus InspectProfile and prepared details.
|
|
||||||
Consumers choose report disclosure; do not add CLI commands or automatic logs.
|
|
||||||
|
|
||||||
Exit: public inspection tests pass, including concurrent inspection/execution.
|
|
||||||
The runner has no reference to the inspector or ConfigKit metadata types.
|
|
||||||
|
|
||||||
## Stage 10 — Migrate Examples And Canonical Format Guidance
|
|
||||||
|
|
||||||
**Completed:** 2026-09-30.
|
|
||||||
|
|
||||||
Scope: maintained examples and user-facing documentation.
|
|
||||||
|
|
||||||
1. Complete the preparation, execution, and observation example migration begun
|
|
||||||
at public cutover. Preserve their distinct existing semantic assertions; extend
|
|
||||||
the preparation example with a small explicit inspection query if useful.
|
|
||||||
2. Update `docs/formats.md` to own the new keyed syntax, source-group semantics,
|
|
||||||
inheritance/presence/null table, eager validation, snapshot lifetime, and
|
|
||||||
resource-context rules. Link ConfigKit for generic grammar; do not copy its
|
|
||||||
complete parser/inheritance contract.
|
|
||||||
3. Update README and `docs/consumers/pkg-promptkit.md` with minimal local-backend,
|
|
||||||
file/embedded source, programmatic Mapping/ProfileSource, explicit inspection,
|
|
||||||
and engine-replacement workflows. Explain main versus ordered underlays and
|
|
||||||
backend layering in place of registration-only restrictions.
|
|
||||||
4. Add concise old-to-new migration guidance in the consumer guide: removed API
|
|
||||||
options, exact IDs, `base_profile` to `extends`, zero/false/null, recursive maps,
|
|
||||||
no clear via `{}`, reserved `extends` runtime escape, failure timing, and the
|
|
||||||
fact that changing a model/backend does not clear unrelated explicit fields.
|
|
||||||
Remove old automatic reload claims. Release notes later link to this owner.
|
|
||||||
|
|
||||||
Exit: all three offline examples run with their semantic checks; documentation
|
|
||||||
links and snippets match the implemented API. No unreleased version is tagged or
|
|
||||||
published, and no speculative release-note version is invented.
|
|
||||||
|
|
||||||
## Stage 11 — Remove Superseded Machinery And Consolidate Tests
|
|
||||||
|
|
||||||
**Completed:** 2026-09-30.
|
|
||||||
|
|
||||||
Scope: dead loading/resolution code and its obsolete tests, module cleanup, and
|
|
||||||
internal architectural documentation.
|
|
||||||
|
|
||||||
1. Delete lazy prompt/profile metadata prepasses, filesystem repositories,
|
|
||||||
fallback-on-error overlays, custom base-chain resolver/depth cap, zero-sentinel
|
|
||||||
profile merging, and temporary stage adapters once no caller needs them.
|
|
||||||
Remove the separate backend-manifest decoder, source-policy audits, and
|
|
||||||
pre-handoff backend registration validation. Keep final-value domain checks.
|
|
||||||
Retain narrow lookup interfaces only where they still simplify real consumers.
|
|
||||||
2. Retain useful filecatalog/schema utilities and JSON-value ownership/validation.
|
|
||||||
Remove direct yaml.v3 use and dependency if no remaining owner requires it;
|
|
||||||
do not claim all YAML dependencies disappear when ConfigKit still uses YAML.
|
|
||||||
3. Move meaningful regressions from deleted package tests to their new owning
|
|
||||||
boundary. Retire tests about old internals, repeated parser coverage,
|
|
||||||
source-policy enforcement, or superseded semantics. Do not replace lost runtime
|
|
||||||
protection with loader-only tests. Keep private assembly checks separate from
|
|
||||||
external facade tests.
|
|
||||||
4. Update `docs/internal/overview.md`, `docs/internal/sources.md`, relevant renderer
|
|
||||||
and engine documents, and the implemented-structure paragraphs in architecture
|
|
||||||
policy. Describe the new package/dependency boundary without turning names or
|
|
||||||
chosen adapters into permanent invariants. Update OSS notices with module tidy.
|
|
||||||
|
|
||||||
Exit: no legacy source path remains, no internal package imports the public root,
|
|
||||||
runtime/domain packages do not import ConfigKit, and `go test ./...` passes with
|
|
||||||
the reduced implementation/test inventory.
|
|
||||||
|
|
||||||
## Stage 12 — Validate Scale And The Complete Consumer Workflow
|
|
||||||
|
|
||||||
**Completed:** 2026-09-30.
|
|
||||||
|
|
||||||
Scope: final integration evidence and high-confidence simplification, not another
|
|
||||||
feature expansion or release gate.
|
|
||||||
|
|
||||||
1. Add focused benchmarks for construction with many independent definitions,
|
|
||||||
shared inherited bases, and several tiers; add an already-constructed selection
|
|
||||||
benchmark. Report allocations and input/output shape; inherited materialized
|
|
||||||
output/history may dominate. Do not impose timing thresholds or new caps.
|
|
||||||
2. Inspect for repeated full-map copies, per-run provenance calls, rescans,
|
|
||||||
repeated inheritance, and unnecessary double parsing. Fix demonstrated local
|
|
||||||
inefficiencies with straightforward ownership/indexing; no global cache.
|
|
||||||
3. Run the repository's existing complete local check script once on the final
|
|
||||||
candidate, including race tests and offline examples. Investigate failures;
|
|
||||||
do not weaken meaningful tests to get a green result. Stage intended new
|
|
||||||
files before the script's tracked-file checks.
|
|
||||||
4. Review the feature-roadmap capability list against actual public workflows:
|
|
||||||
independent inspection, partial profile completion, optional credentials,
|
|
||||||
source contexts, artifacts/schemas, prepared ownership, backend capacity,
|
|
||||||
overrides, appended messages, repair/retries, observations, and hashes.
|
|
||||||
5. Report completed stages, selected catalog versions, validation evidence, and
|
|
||||||
remaining issues. Minor-release documentation/version selection and roadmap
|
|
||||||
retirement belong to the separately authorized release preparation workflow.
|
|
||||||
|
|
||||||
Exit: the feature roadmap's end state is demonstrably implemented, with one
|
|
||||||
configuration lifecycle, explicit operator inspection, no legacy loader, and
|
|
||||||
no newly invented provider checks or human release gates.
|
|
||||||
|
|
||||||
The catalog-validation placement question is resolved by the configuration-boundary
|
|
||||||
ADR. No design question remains open here. Catalog publication authority and
|
|
||||||
ordinary module availability remain operational checkpoints for the catalog stages.
|
|
||||||
@@ -1,126 +0,0 @@
|
|||||||
# ConfigKit Adoption Review Findings
|
|
||||||
|
|
||||||
Review date: 2026-10-03. Reviewed implementation commits `2b847ac` through
|
|
||||||
`66c55b7` against the [feature roadmap](configkit.md),
|
|
||||||
[implementation plan](implementation.md), and accepted
|
|
||||||
[configuration-boundary ADR](../adr/0001-configuration-boundary.md).
|
|
||||||
|
|
||||||
## Summary
|
|
||||||
|
|
||||||
The principal architecture is in place: one ConfigKit load assembles backend,
|
|
||||||
profile, and prompt configuration; PromptKit validates effective values, captures
|
|
||||||
resources/templates, and builds execution-owned indexes. Catalog preflight and
|
|
||||||
legacy profile resolution are removed. Operator inspection delegates to the
|
|
||||||
retained ConfigKit result separately from ordinary execution.
|
|
||||||
|
|
||||||
Both findings have been addressed. The original review evidence is retained
|
|
||||||
below; the resolution section records the changes and regression coverage.
|
|
||||||
|
|
||||||
## 1. P2 — Message Normalization Loses Content-File Resource Origins
|
|
||||||
|
|
||||||
Location: `internal/configuration/prompts.go`, particularly `resolvePrompts`
|
|
||||||
lines 69–99 and `readContent`'s effective-origin lookup.
|
|
||||||
|
|
||||||
When a role or cache-control value needs normalization, `resolvePrompts` records
|
|
||||||
a normalization operation for the entire `messages` sequence in addition to the
|
|
||||||
field-level operations. The resulting metadata no longer provides the source
|
|
||||||
location needed by `readContent` for an unchanged `content_file`. Construction
|
|
||||||
then rejects an otherwise valid prompt.
|
|
||||||
|
|
||||||
Confirmed through the public API using FilesystemInputs over an fs.FS containing
|
|
||||||
this document and a readable `body.txt`:
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
prompts:
|
|
||||||
p:
|
|
||||||
v1:
|
|
||||||
messages:
|
|
||||||
- role: " USER "
|
|
||||||
content_file: body.txt
|
|
||||||
output:
|
|
||||||
format: text
|
|
||||||
validation_mode: none
|
|
||||||
```
|
|
||||||
|
|
||||||
NewEngine fails with
|
|
||||||
`/prompts/p/v1/messages/0/content_file: content_file needs one contributing source with a filesystem context`.
|
|
||||||
Changing only the role to `user` succeeds. Role trimming/lowercasing and
|
|
||||||
source-relative message capture are both intended capabilities; requiring
|
|
||||||
already-normalized metadata is an accidental restriction.
|
|
||||||
|
|
||||||
Required outcome:
|
|
||||||
|
|
||||||
- Preserve ConfigKit's effective resource context while normalizing neighboring
|
|
||||||
message metadata. Review the broad sequence normalization record and how the
|
|
||||||
final input origin is retained across domain normalization.
|
|
||||||
- Use effective handoff metadata/context, not historical traversal, raw-source
|
|
||||||
inspection, source rereading, or an invented fallback directory.
|
|
||||||
- Add focused regressions combining file-backed content with role and
|
|
||||||
cache-control normalization, including inherited/imported content. Existing
|
|
||||||
tests cover normalization and resource capture separately but miss this
|
|
||||||
interaction. Verify inspection remains accurate for untouched content fields.
|
|
||||||
|
|
||||||
## 2. P2 — Profiles Gain An Unrequested Environment-Name Restriction
|
|
||||||
|
|
||||||
Location: `internal/configuration/load.go`, profile validation at lines 278–279,
|
|
||||||
using the package's `envName` regular expression.
|
|
||||||
|
|
||||||
Profile `api_key_env` values now must match `[A-Za-z_][A-Za-z0-9_]*`. The previous
|
|
||||||
profile path accepted environment lookup names without this shell-identifier
|
|
||||||
restriction; the regular expression previously governed backend registration.
|
|
||||||
The refactor extends it to profiles even though arbitrary environment names
|
|
||||||
supported by the host remain usable through Go's environment lookup and the
|
|
||||||
request override path.
|
|
||||||
|
|
||||||
Confirmed through NewEngine with an otherwise valid endpoint/model profile:
|
|
||||||
`api_key_env: NORMAL_KEY` succeeds, while `api_key_env: service.api-key` fails
|
|
||||||
construction. The latter is a usable environment name on the reviewed Linux
|
|
||||||
platform. This restriction is unrelated to configuration layering, ConfigKit
|
|
||||||
typing, or optional credential availability.
|
|
||||||
|
|
||||||
The implementation plan's reference to existing naming rules is not evidence
|
|
||||||
of an express user decision to introduce this new profile restriction. The
|
|
||||||
requested compatibility break permits adopting ConfigKit semantics; it does
|
|
||||||
not itself authorize additional unrelated name constraints.
|
|
||||||
|
|
||||||
Required outcome:
|
|
||||||
|
|
||||||
- Remove the newly introduced profile shell-identifier check, preserving the
|
|
||||||
established trimming, explicit clearing, optional environment lookup, and
|
|
||||||
direct-key precedence behavior.
|
|
||||||
- Add a public-path regression for a supported non-shell-identifier environment
|
|
||||||
name and for its unset/empty optional behavior, with controlled test environment
|
|
||||||
state and a synthetic credential. Keep any prerequisite validation limited to
|
|
||||||
actual supported environment lookup requirements.
|
|
||||||
- Correct the plan's profile-field guidance so later work does not reintroduce
|
|
||||||
this restriction. Treat any broader change to existing backend naming policy
|
|
||||||
separately rather than silently extending or weakening it in this fix.
|
|
||||||
|
|
||||||
## Validation Performed
|
|
||||||
|
|
||||||
- `go test ./...` passed on Go 1.26.8.
|
|
||||||
- `GOTOOLCHAIN=go1.25.5 go test ./...` passed on the declared minimum toolchain.
|
|
||||||
- `go test -race ./...` passed.
|
|
||||||
- Separate offline public-API reproductions confirmed both findings. Missing
|
|
||||||
message/session variables also remained runtime errors in targeted checks.
|
|
||||||
|
|
||||||
The passing suite does not cover the two failing cases above. No provider
|
|
||||||
requests were made. Beyond these findings, the reviewed implementation did not
|
|
||||||
show another actionable ownership-boundary violation or strengthened guarantee.
|
|
||||||
|
|
||||||
## Resolution
|
|
||||||
|
|
||||||
1. Message role/cache-control normalization now occurs on execution-owned copies
|
|
||||||
during materialization. ConfigKit's captured message sequence and its origins
|
|
||||||
remain intact. The imported/inherited content regression now includes role
|
|
||||||
and cache-control normalization and asserts the unchanged field's source in
|
|
||||||
inspection metadata. This also removes duplicate cache-control normalization.
|
|
||||||
2. Removed the profile environment-name regular expression check, leaving backend
|
|
||||||
validation unchanged. A public engine/HTTP-transport regression covers a
|
|
||||||
non-shell environment name, unset/empty/whitespace values, explicit clearing,
|
|
||||||
trimming, and direct-key precedence. The implementation plan now states this
|
|
||||||
distinction explicitly.
|
|
||||||
|
|
||||||
Both regressions failed against the reviewed implementation before the fixes.
|
|
||||||
After remediation, the full ordinary and race-enabled suites, affected packages
|
|
||||||
on Go 1.25.5, `go vet ./...`, and documentation/whitespace checks passed.
|
|
||||||
@@ -82,12 +82,12 @@ var (
|
|||||||
ErrValidation = errors.New("failed to validate output")
|
ErrValidation = errors.New("failed to validate output")
|
||||||
)
|
)
|
||||||
|
|
||||||
// Engine inspects prompts and profiles and prepares and runs Promptkit prompt
|
// Engine inspects prompts and profiles, enumerates prompt requirements, and
|
||||||
// requests.
|
// prepares and runs Promptkit prompt requests.
|
||||||
//
|
//
|
||||||
// An Engine is safe for concurrent calls to [Engine.InspectConfiguration],
|
// An Engine is safe for concurrent calls to [Engine.InspectConfiguration],
|
||||||
// [Engine.InspectPrompt],
|
// [Engine.InspectPrompt],
|
||||||
// [Engine.InspectProfile], [Engine.Prepare], [Engine.PrepareExecution],
|
// [Engine.InspectRequirements], [Engine.InspectProfile], [Engine.Prepare], [Engine.PrepareExecution],
|
||||||
// [Engine.Run], and [Engine.RunPrepared]. Each Engine owns independent
|
// [Engine.Run], and [Engine.RunPrepared]. Each Engine owns independent
|
||||||
// backend-capacity pools that coordinate Run and RunPrepared admission and
|
// backend-capacity pools that coordinate Run and RunPrepared admission and
|
||||||
// model generation. Injected collaborators may still be invoked concurrently
|
// model generation. Injected collaborators may still be invoked concurrently
|
||||||
|
|||||||
@@ -36,6 +36,17 @@ func main() {
|
|||||||
if _, found := view.Explain(configkit.Keys("profiles", "offline-example", "model")); !found {
|
if _, found := view.Explain(configkit.Keys("profiles", "offline-example", "model")); !found {
|
||||||
exit(fmt.Errorf("configured profile model was not found"))
|
exit(fmt.Errorf("configured profile model was not found"))
|
||||||
}
|
}
|
||||||
|
requirements, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{
|
||||||
|
PromptID: "example.prepare",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
exit(err)
|
||||||
|
}
|
||||||
|
if !requirements.Complete || len(requirements.Inputs) != 1 ||
|
||||||
|
requirements.Inputs[0].Name != "note" ||
|
||||||
|
requirements.Inputs[0].Kind != promptkit.RequirementRequired {
|
||||||
|
exit(fmt.Errorf("required note input was not discovered before preparation"))
|
||||||
|
}
|
||||||
|
|
||||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||||
PromptID: "example.prepare",
|
PromptID: "example.prepare",
|
||||||
|
|||||||
9
go.mod
9
go.mod
@@ -1,15 +1,20 @@
|
|||||||
module gitea.maximumdirect.net/eric/promptkit
|
module gitea.maximumdirect.net/eric/promptkit
|
||||||
|
|
||||||
go 1.25.5
|
go 1.26.0
|
||||||
|
|
||||||
require (
|
require (
|
||||||
gitea.maximumdirect.net/eric/ConfigKit v0.5.0
|
gitea.maximumdirect.net/eric/ConfigKit v0.6.0
|
||||||
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v3 v3.20260930.0
|
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v3 v3.20260930.0
|
||||||
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v3 v3.20260930.0
|
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v3 v3.20260930.0
|
||||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2
|
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2
|
||||||
)
|
)
|
||||||
|
|
||||||
require (
|
require (
|
||||||
|
gitea.maximumdirect.net/eric/ReleaseKit v0.3.0 // indirect
|
||||||
|
github.com/yuin/goldmark/v2 v2.1.6 // indirect
|
||||||
go.yaml.in/yaml/v4 v4.0.0-rc.6 // indirect
|
go.yaml.in/yaml/v4 v4.0.0-rc.6 // indirect
|
||||||
|
golang.org/x/mod v0.40.0 // indirect
|
||||||
golang.org/x/text v0.14.0 // indirect
|
golang.org/x/text v0.14.0 // indirect
|
||||||
)
|
)
|
||||||
|
|
||||||
|
tool gitea.maximumdirect.net/eric/ReleaseKit/cmd/releasekit
|
||||||
|
|||||||
10
go.sum
10
go.sum
@@ -1,5 +1,7 @@
|
|||||||
gitea.maximumdirect.net/eric/ConfigKit v0.5.0 h1:2KwG10+kApWM3avOoYA+p768xcbHUp7tQ2rqizBNpyg=
|
gitea.maximumdirect.net/eric/ConfigKit v0.6.0 h1:KdsCVO5kKctdAUQJ6X7dJ/GemlVjCYCcn/xZrMAE/aw=
|
||||||
gitea.maximumdirect.net/eric/ConfigKit v0.5.0/go.mod h1:tYWpsrbLyu05r8+1MapWp5fN6ZrX9I8hU2xEPedYmYk=
|
gitea.maximumdirect.net/eric/ConfigKit v0.6.0/go.mod h1:xN4dRdDGIW4yh/e9GghDF/hSIzc8/77blRhwyf90KSA=
|
||||||
|
gitea.maximumdirect.net/eric/ReleaseKit v0.3.0 h1:aTMn+50MEpHpHI2RoCCmJYsy7Ohrpa+RSva63Kq0/tU=
|
||||||
|
gitea.maximumdirect.net/eric/ReleaseKit v0.3.0/go.mod h1:LVodqNQ3OIKv9NBq+fFeyMByd1lHY25YOmmpah/NaNI=
|
||||||
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v3 v3.20260930.0 h1:lzfCDbzJQRGzIlRPRprnAV9hVVL5zUQy80VMQ0kbYa0=
|
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v3 v3.20260930.0 h1:lzfCDbzJQRGzIlRPRprnAV9hVVL5zUQy80VMQ0kbYa0=
|
||||||
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v3 v3.20260930.0/go.mod h1:HjS0f+3q0IdHCTEz+VVAEgIwIWJFAZUjuKdAtvtqKdg=
|
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v3 v3.20260930.0/go.mod h1:HjS0f+3q0IdHCTEz+VVAEgIwIWJFAZUjuKdAtvtqKdg=
|
||||||
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v3 v3.20260930.0 h1:h12ZTRldAmpLAomj0oQRGteFxHLN1LPE6XDl9zJNbm8=
|
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v3 v3.20260930.0 h1:h12ZTRldAmpLAomj0oQRGteFxHLN1LPE6XDl9zJNbm8=
|
||||||
@@ -8,8 +10,12 @@ github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxK
|
|||||||
github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
|
github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
|
||||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ=
|
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ=
|
||||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU=
|
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU=
|
||||||
|
github.com/yuin/goldmark/v2 v2.1.6 h1:igy0q9JcqovoFXFx/XjtU8sHVhcUHavPj2sA3EoKJUE=
|
||||||
|
github.com/yuin/goldmark/v2 v2.1.6/go.mod h1:G6M4/qOFtfn01/o14BU1UR2Lo5N3S9Qo7xCuz/sHjGQ=
|
||||||
go.yaml.in/yaml/v4 v4.0.0-rc.6 h1:1h7H1ohdUh93/FyE4YaDa1Zh64K6VVbjF4K6WUxMtH4=
|
go.yaml.in/yaml/v4 v4.0.0-rc.6 h1:1h7H1ohdUh93/FyE4YaDa1Zh64K6VVbjF4K6WUxMtH4=
|
||||||
go.yaml.in/yaml/v4 v4.0.0-rc.6/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0=
|
go.yaml.in/yaml/v4 v4.0.0-rc.6/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0=
|
||||||
|
golang.org/x/mod v0.40.0 h1:hUv+3cXcdRHz08UmSiOob7sadHig73uo5bkXxQ/tvUs=
|
||||||
|
golang.org/x/mod v0.40.0/go.mod h1:0/weTWkPWGBikyTWAX3dkjVztMmBA5hM0DH6BElSupE=
|
||||||
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
|
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
|
||||||
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||||
|
|||||||
@@ -21,6 +21,7 @@ var (
|
|||||||
)
|
)
|
||||||
|
|
||||||
const fileReadChunkSize = 64 * 1024
|
const fileReadChunkSize = 64 * 1024
|
||||||
|
const fileReadInitialCapacityLimit = 1 << 20
|
||||||
|
|
||||||
// Reader resolves artifact references into actual artifacts.
|
// Reader resolves artifact references into actual artifacts.
|
||||||
type Reader interface {
|
type Reader interface {
|
||||||
@@ -133,7 +134,11 @@ func readFileArtifact(ctx context.Context, path string, open func(string) (artif
|
|||||||
return nil, fmt.Errorf("%w: %s", ErrUnsupportedFile, path)
|
return nil, fmt.Errorf("%w: %s", ErrUnsupportedFile, path)
|
||||||
}
|
}
|
||||||
|
|
||||||
data := make([]byte, 0)
|
capacity := 0
|
||||||
|
if size := openedInfo.Size(); size > 0 {
|
||||||
|
capacity = int(min(size, int64(fileReadInitialCapacityLimit)))
|
||||||
|
}
|
||||||
|
data := make([]byte, 0, capacity)
|
||||||
chunk := make([]byte, fileReadChunkSize)
|
chunk := make([]byte, fileReadChunkSize)
|
||||||
for {
|
for {
|
||||||
if err := ctx.Err(); err != nil {
|
if err := ctx.Err(); err != nil {
|
||||||
@@ -153,6 +158,13 @@ func readFileArtifact(ctx context.Context, path string, open func(string) (artif
|
|||||||
return nil, fmt.Errorf("failed to read file %s: %w", path, readErr)
|
return nil, fmt.Errorf("failed to read file %s: %w", path, readErr)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if cap(data)-len(data) > len(data) {
|
||||||
|
// A stale size hint should not make a small completed artifact retain a
|
||||||
|
// disproportionately large backing array.
|
||||||
|
compact := make([]byte, len(data))
|
||||||
|
copy(compact, data)
|
||||||
|
data = compact
|
||||||
|
}
|
||||||
|
|
||||||
contentType := mime.TypeByExtension(filepath.Ext(path))
|
contentType := mime.TypeByExtension(filepath.Ext(path))
|
||||||
if contentType == "" {
|
if contentType == "" {
|
||||||
|
|||||||
180
internal/artifact/reader_benchmark_test.go
Normal file
180
internal/artifact/reader_benchmark_test.go
Normal file
@@ -0,0 +1,180 @@
|
|||||||
|
package artifact
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"crypto/sha256"
|
||||||
|
"fmt"
|
||||||
|
"mime"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"runtime"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||||
|
)
|
||||||
|
|
||||||
|
var artifactFileReadBenchmarkSink *domain.Artifact
|
||||||
|
|
||||||
|
func BenchmarkArtifactFileRead(b *testing.B) {
|
||||||
|
for _, fixture := range makeArtifactReadBenchmarkFiles(b) {
|
||||||
|
b.Run(fixture.name, func(b *testing.B) {
|
||||||
|
reader := &fileReader{open: openArtifactFile}
|
||||||
|
ref := domain.ArtifactRef{Type: domain.ArtifactRefFile, URI: fixture.path}
|
||||||
|
checkArtifactFileRead(b, reader, ref, fixture.body)
|
||||||
|
|
||||||
|
b.ReportAllocs()
|
||||||
|
b.SetBytes(int64(len(fixture.body)))
|
||||||
|
b.ResetTimer()
|
||||||
|
for range b.N {
|
||||||
|
artifact, err := reader.Read(context.Background(), ref)
|
||||||
|
if err != nil {
|
||||||
|
b.Fatalf("read artifact: %v", err)
|
||||||
|
}
|
||||||
|
artifactFileReadBenchmarkSink = artifact
|
||||||
|
}
|
||||||
|
b.StopTimer()
|
||||||
|
runtime.KeepAlive(artifactFileReadBenchmarkSink)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// BenchmarkRetainedArtifactFileRead measures live artifact bodies when an
|
||||||
|
// opened file reports a size larger than the content that reaches EOF.
|
||||||
|
func BenchmarkRetainedArtifactFileRead(b *testing.B) {
|
||||||
|
for _, fixture := range []struct {
|
||||||
|
name string
|
||||||
|
body []byte
|
||||||
|
}{
|
||||||
|
{name: "small", body: bytes.Repeat([]byte("x"), 256<<10)},
|
||||||
|
{name: "empty", body: []byte{}},
|
||||||
|
} {
|
||||||
|
b.Run(fixture.name, func(b *testing.B) {
|
||||||
|
benchmarkRetainedArtifactFileRead(b, fixture.body)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func benchmarkRetainedArtifactFileRead(b *testing.B, body []byte) {
|
||||||
|
path := filepath.Join(b.TempDir(), "overreported.txt")
|
||||||
|
if err := os.WriteFile(path, body, 0o600); err != nil {
|
||||||
|
b.Fatal(err)
|
||||||
|
}
|
||||||
|
mime.TypeByExtension(filepath.Ext(path))
|
||||||
|
|
||||||
|
reader := &fileReader{open: func(path string) (artifactFile, error) {
|
||||||
|
file, err := os.Open(path)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return &sizeReportingArtifactFile{artifactFile: file, reportedSize: 16 << 20}, nil
|
||||||
|
}}
|
||||||
|
ref := domain.ArtifactRef{Type: domain.ArtifactRefFile, URI: path}
|
||||||
|
checkArtifactFileRead(b, reader, ref, body)
|
||||||
|
|
||||||
|
const retainedBatchSize = 4
|
||||||
|
retained := make([]*domain.Artifact, retainedBatchSize)
|
||||||
|
artifactFileReadBenchmarkSink = nil
|
||||||
|
runtime.GC()
|
||||||
|
var setupMemory runtime.MemStats
|
||||||
|
runtime.ReadMemStats(&setupMemory)
|
||||||
|
runtime.KeepAlive(reader)
|
||||||
|
runtime.KeepAlive(ref)
|
||||||
|
|
||||||
|
b.ReportAllocs()
|
||||||
|
b.SetBytes(int64(len(body)))
|
||||||
|
b.ResetTimer()
|
||||||
|
for iteration := 0; iteration < b.N; iteration++ {
|
||||||
|
artifact, err := reader.Read(context.Background(), ref)
|
||||||
|
if err != nil {
|
||||||
|
b.Fatalf("read retained artifact: %v", err)
|
||||||
|
}
|
||||||
|
retained[iteration%len(retained)] = artifact
|
||||||
|
}
|
||||||
|
b.StopTimer()
|
||||||
|
for i, artifact := range retained {
|
||||||
|
if artifact != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
artifact, err := reader.Read(context.Background(), ref)
|
||||||
|
if err != nil {
|
||||||
|
b.Fatalf("fill retained artifact batch: %v", err)
|
||||||
|
}
|
||||||
|
retained[i] = artifact
|
||||||
|
}
|
||||||
|
runtime.GC()
|
||||||
|
var retainedMemory runtime.MemStats
|
||||||
|
runtime.ReadMemStats(&retainedMemory)
|
||||||
|
b.ReportMetric(float64(int64(retainedMemory.HeapAlloc)-int64(setupMemory.HeapAlloc)), "retained-B")
|
||||||
|
runtime.KeepAlive(retained)
|
||||||
|
runtime.KeepAlive(reader)
|
||||||
|
runtime.KeepAlive(ref)
|
||||||
|
}
|
||||||
|
|
||||||
|
type artifactReadBenchmarkFile struct {
|
||||||
|
name string
|
||||||
|
path string
|
||||||
|
body []byte
|
||||||
|
}
|
||||||
|
|
||||||
|
func makeArtifactReadBenchmarkFiles(b *testing.B) []artifactReadBenchmarkFile {
|
||||||
|
b.Helper()
|
||||||
|
fixtures := []struct {
|
||||||
|
name string
|
||||||
|
size int
|
||||||
|
}{
|
||||||
|
{name: "small_4KiB", size: 4 << 10},
|
||||||
|
{name: "1MiB", size: 1 << 20},
|
||||||
|
{name: "large_4MiB", size: 4 << 20},
|
||||||
|
}
|
||||||
|
results := make([]artifactReadBenchmarkFile, 0, len(fixtures))
|
||||||
|
for _, fixture := range fixtures {
|
||||||
|
body := bytes.Repeat([]byte("x"), fixture.size)
|
||||||
|
path := filepath.Join(b.TempDir(), fixture.name+".txt")
|
||||||
|
if err := os.WriteFile(path, body, 0o600); err != nil {
|
||||||
|
b.Fatal(err)
|
||||||
|
}
|
||||||
|
mime.TypeByExtension(filepath.Ext(path))
|
||||||
|
results = append(results, artifactReadBenchmarkFile{
|
||||||
|
name: fixture.name,
|
||||||
|
path: path,
|
||||||
|
body: body,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return results
|
||||||
|
}
|
||||||
|
|
||||||
|
func checkArtifactFileRead(b *testing.B, reader Reader, ref domain.ArtifactRef, want []byte) {
|
||||||
|
b.Helper()
|
||||||
|
artifact, err := reader.Read(context.Background(), ref)
|
||||||
|
if err != nil {
|
||||||
|
b.Fatalf("check artifact read: %v", err)
|
||||||
|
}
|
||||||
|
if !bytes.Equal(artifact.Body, want) || artifact.Size != int64(len(want)) {
|
||||||
|
b.Fatalf("artifact size=%d body length=%d, want %d", artifact.Size, len(artifact.Body), len(want))
|
||||||
|
}
|
||||||
|
wantHash := fmt.Sprintf("%x", sha256.Sum256(want))
|
||||||
|
if artifact.Hash != wantHash {
|
||||||
|
b.Fatalf("artifact hash=%q, want %q", artifact.Hash, wantHash)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type sizeReportingArtifactFile struct {
|
||||||
|
artifactFile
|
||||||
|
reportedSize int64
|
||||||
|
}
|
||||||
|
|
||||||
|
func (file *sizeReportingArtifactFile) Stat() (os.FileInfo, error) {
|
||||||
|
info, err := file.artifactFile.Stat()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return reportedArtifactFileInfo{FileInfo: info, size: file.reportedSize}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type reportedArtifactFileInfo struct {
|
||||||
|
os.FileInfo
|
||||||
|
size int64
|
||||||
|
}
|
||||||
|
|
||||||
|
func (info reportedArtifactFileInfo) Size() int64 { return info.size }
|
||||||
@@ -3,7 +3,9 @@ package artifact
|
|||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
|
"crypto/sha256"
|
||||||
"errors"
|
"errors"
|
||||||
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
@@ -232,6 +234,64 @@ func TestFileReaderFailuresAndMetadata(t *testing.T) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestFileReaderUsesOpenedSizeOnlyAsACapacityHint(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
body []byte
|
||||||
|
reportedSize int64
|
||||||
|
}{
|
||||||
|
{name: "overreported size", body: []byte("actual file content"), reportedSize: 128},
|
||||||
|
{name: "overreported size with empty content", body: []byte{}, reportedSize: 16 << 20},
|
||||||
|
{name: "underreported size", body: bytes.Repeat([]byte("x"), fileReadChunkSize+17), reportedSize: 1},
|
||||||
|
{name: "very large hint with small content", body: []byte("small actual content"), reportedSize: int64(^uint64(0) >> 1)},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
filePath := filepath.Join(t.TempDir(), "artifact.txt")
|
||||||
|
if err := os.WriteFile(filePath, tc.body, 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
reader := &fileReader{open: func(path string) (artifactFile, error) {
|
||||||
|
file, err := os.Open(path)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
info, err := file.Stat()
|
||||||
|
if err != nil {
|
||||||
|
file.Close()
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return &reportedInfoFile{
|
||||||
|
artifactFile: file,
|
||||||
|
info: reportedFileSizeInfo{FileInfo: info, size: tc.reportedSize},
|
||||||
|
}, nil
|
||||||
|
}}
|
||||||
|
|
||||||
|
artifact, err := reader.Read(context.Background(), domain.ArtifactRef{
|
||||||
|
Type: domain.ArtifactRefFile,
|
||||||
|
URI: filePath,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("read artifact: %v", err)
|
||||||
|
}
|
||||||
|
if !bytes.Equal(artifact.Body, tc.body) {
|
||||||
|
t.Fatalf("artifact body length=%d, want %d", len(artifact.Body), len(tc.body))
|
||||||
|
}
|
||||||
|
if artifact.Size != int64(len(tc.body)) {
|
||||||
|
t.Fatalf("artifact size=%d, want %d", artifact.Size, len(tc.body))
|
||||||
|
}
|
||||||
|
if artifact.Name != filepath.Base(filePath) || artifact.URI != filePath || artifact.ContentType == "" {
|
||||||
|
t.Fatalf("unexpected artifact metadata: name=%q URI=%q content type=%q", artifact.Name, artifact.URI, artifact.ContentType)
|
||||||
|
}
|
||||||
|
wantHash := fmt.Sprintf("%x", sha256.Sum256(tc.body))
|
||||||
|
if artifact.Hash != wantHash {
|
||||||
|
t.Fatalf("artifact hash=%q, want %q", artifact.Hash, wantHash)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestFileReaderCancelsAfterReadProgress(t *testing.T) {
|
func TestFileReaderCancelsAfterReadProgress(t *testing.T) {
|
||||||
filePath := filepath.Join(t.TempDir(), "artifact.bin")
|
filePath := filepath.Join(t.TempDir(), "artifact.bin")
|
||||||
content := bytes.Repeat([]byte("x"), fileReadChunkSize*2)
|
content := bytes.Repeat([]byte("x"), fileReadChunkSize*2)
|
||||||
@@ -268,6 +328,13 @@ func (f *reportedInfoFile) Stat() (os.FileInfo, error) {
|
|||||||
return f.info, nil
|
return f.info, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type reportedFileSizeInfo struct {
|
||||||
|
os.FileInfo
|
||||||
|
size int64
|
||||||
|
}
|
||||||
|
|
||||||
|
func (info reportedFileSizeInfo) Size() int64 { return info.size }
|
||||||
|
|
||||||
type cancelAfterProgressFile struct {
|
type cancelAfterProgressFile struct {
|
||||||
artifactFile
|
artifactFile
|
||||||
cancel context.CancelFunc
|
cancel context.CancelFunc
|
||||||
|
|||||||
@@ -13,9 +13,12 @@ import (
|
|||||||
const benchmarkDefinitionCount = 128
|
const benchmarkDefinitionCount = 128
|
||||||
|
|
||||||
var (
|
var (
|
||||||
constructionResult *loaded
|
constructionResult *loaded
|
||||||
selectedPrompt *domain.PromptDefinition
|
selectedPrompt *domain.PromptDefinition
|
||||||
selectedProfile *domain.ExecutionProfile
|
selectedPromptHash string
|
||||||
|
selectedPromptInspection *domain.PromptInspection
|
||||||
|
selectedProfile *domain.ExecutionProfile
|
||||||
|
selectedProfileIdentity string
|
||||||
)
|
)
|
||||||
|
|
||||||
type configurationShape struct {
|
type configurationShape struct {
|
||||||
@@ -177,3 +180,116 @@ func BenchmarkConfigurationSelection(b *testing.B) {
|
|||||||
b.ReportMetric(float64(shape.effectiveEntries), "effective-entries/op")
|
b.ReportMetric(float64(shape.effectiveEntries), "effective-entries/op")
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func inspectionExtraParams(entries int) map[string]any {
|
||||||
|
providers := make(map[string]any, entries)
|
||||||
|
for index := range entries {
|
||||||
|
providers[fmt.Sprintf("provider-%03d", index)] = map[string]any{
|
||||||
|
"enabled": true,
|
||||||
|
"labels": []any{fmt.Sprintf("tier-%d", index%4), fmt.Sprintf("region-%d", index%3)},
|
||||||
|
"limits": map[string]any{"burst": index + 1, "period": "minute"},
|
||||||
|
"routes": []any{map[string]any{"weight": index + 1, "fallback": true}},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return map[string]any{"provider_options": providers}
|
||||||
|
}
|
||||||
|
|
||||||
|
func inspectionDefinitions(extraParams map[string]any) configurationShape {
|
||||||
|
profiles := map[string]any{
|
||||||
|
"small": map[string]any{"model": "inspection-model", "extra_params": extraParams},
|
||||||
|
}
|
||||||
|
prompt := benchmarkPrompt("small")
|
||||||
|
prompt["inputs"] = []any{map[string]any{"name": "transcript", "required": true}}
|
||||||
|
prompts := map[string]any{
|
||||||
|
"inspection": map[string]any{"1": prompt},
|
||||||
|
}
|
||||||
|
return configurationShape{
|
||||||
|
main: configkit.Input{Source: configkit.Mapping("inspection definitions", map[string]any{
|
||||||
|
"profiles": profiles,
|
||||||
|
"prompts": prompts,
|
||||||
|
})},
|
||||||
|
inputEntries: 2, effectiveEntries: 2, tiers: 1,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func BenchmarkInspectionSelection(b *testing.B) {
|
||||||
|
for _, scenario := range []struct {
|
||||||
|
name string
|
||||||
|
entries int
|
||||||
|
}{
|
||||||
|
{name: "small_extra", entries: 2},
|
||||||
|
{name: "nested_extra", entries: 64},
|
||||||
|
} {
|
||||||
|
shape := inspectionDefinitions(inspectionExtraParams(scenario.entries))
|
||||||
|
result, err := load(shape.main, nil)
|
||||||
|
if err != nil {
|
||||||
|
b.Fatal(err)
|
||||||
|
}
|
||||||
|
ctx := context.Background()
|
||||||
|
definition, hash, err := result.GetPromptSnapshot(ctx, "inspection", "1")
|
||||||
|
if err != nil || definition.ID != "inspection" || definition.Version != "1" || hash == "" ||
|
||||||
|
len(definition.Inputs) != 1 || definition.Inputs[0].Name != "transcript" || !definition.Inputs[0].Required {
|
||||||
|
b.Fatalf("prompt lookup fixture is invalid: definition=%+v hash=%q err=%v", definition, hash, err)
|
||||||
|
}
|
||||||
|
profile, err := result.GetProfile(ctx, "small")
|
||||||
|
if err != nil || profile.ID != "small" || profile.ResolvedTarget.Model != "inspection-model" ||
|
||||||
|
len(profile.ResolvedTarget.ExtraParams["provider_options"].(map[string]any)) != scenario.entries {
|
||||||
|
b.Fatalf("profile lookup fixture is invalid: profile=%+v err=%v", profile, err)
|
||||||
|
}
|
||||||
|
inspection, err := result.GetPromptInspection(ctx, "inspection", "1")
|
||||||
|
if err != nil || inspection.PromptID != "inspection" || inspection.PromptVersion != "1" ||
|
||||||
|
len(inspection.Inputs) != 1 || inspection.Inputs[0].Name != "transcript" {
|
||||||
|
b.Fatalf("prompt inspection fixture is invalid: inspection=%+v err=%v", inspection, err)
|
||||||
|
}
|
||||||
|
identity, err := result.GetProfileIdentity(ctx, "small")
|
||||||
|
if err != nil || identity != "small" {
|
||||||
|
b.Fatalf("profile identity fixture is invalid: identity=%q err=%v", identity, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
b.Run("prompt/"+scenario.name, func(b *testing.B) {
|
||||||
|
b.ReportAllocs()
|
||||||
|
for b.Loop() {
|
||||||
|
definition, hash, err := result.GetPromptSnapshot(ctx, "inspection", "1")
|
||||||
|
if err != nil {
|
||||||
|
b.Fatal(err)
|
||||||
|
}
|
||||||
|
selectedPrompt = definition
|
||||||
|
selectedPromptHash = hash
|
||||||
|
}
|
||||||
|
b.ReportMetric(float64(scenario.entries), "extra-entries/op")
|
||||||
|
})
|
||||||
|
b.Run("prompt_inspection/"+scenario.name, func(b *testing.B) {
|
||||||
|
b.ReportAllocs()
|
||||||
|
for b.Loop() {
|
||||||
|
inspection, err := result.GetPromptInspection(ctx, "inspection", "1")
|
||||||
|
if err != nil {
|
||||||
|
b.Fatal(err)
|
||||||
|
}
|
||||||
|
selectedPromptInspection = inspection
|
||||||
|
}
|
||||||
|
b.ReportMetric(float64(scenario.entries), "extra-entries/op")
|
||||||
|
})
|
||||||
|
b.Run("profile/"+scenario.name, func(b *testing.B) {
|
||||||
|
b.ReportAllocs()
|
||||||
|
for b.Loop() {
|
||||||
|
profile, err := result.GetProfile(ctx, "small")
|
||||||
|
if err != nil {
|
||||||
|
b.Fatal(err)
|
||||||
|
}
|
||||||
|
selectedProfile = profile
|
||||||
|
}
|
||||||
|
b.ReportMetric(float64(scenario.entries), "extra-entries/op")
|
||||||
|
})
|
||||||
|
b.Run("profile_identity/"+scenario.name, func(b *testing.B) {
|
||||||
|
b.ReportAllocs()
|
||||||
|
for b.Loop() {
|
||||||
|
identity, err := result.GetProfileIdentity(ctx, "small")
|
||||||
|
if err != nil {
|
||||||
|
b.Fatal(err)
|
||||||
|
}
|
||||||
|
selectedProfileIdentity = identity
|
||||||
|
}
|
||||||
|
b.ReportMetric(float64(scenario.entries), "extra-entries/op")
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
29
internal/configuration/listing.go
Normal file
29
internal/configuration/listing.go
Normal file
@@ -0,0 +1,29 @@
|
|||||||
|
package configuration
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||||
|
)
|
||||||
|
|
||||||
|
func (state *loaded) ListProfiles(ctx context.Context) ([]string, error) {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
profiles := append([]string(nil), state.profileIDs...)
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return profiles, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (state *loaded) ListPrompts(ctx context.Context) ([]domain.PromptIdentity, error) {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
prompts := append([]domain.PromptIdentity(nil), state.promptIDs...)
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return prompts, nil
|
||||||
|
}
|
||||||
142
internal/configuration/listing_test.go
Normal file
142
internal/configuration/listing_test.go
Normal file
@@ -0,0 +1,142 @@
|
|||||||
|
package configuration
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"reflect"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/ConfigKit"
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestLoadedListsSortedEffectiveIdentities(t *testing.T) {
|
||||||
|
state, err := Load(mapping("identities", map[string]any{
|
||||||
|
"backends": map[string]any{
|
||||||
|
"local": map[string]any{"endpoint": "http://localhost:8080/v1"},
|
||||||
|
},
|
||||||
|
"profiles": map[string]any{
|
||||||
|
"z/profile": map[string]any{"backend": "local", "model": "z-model"},
|
||||||
|
"partial/profile": map[string]any{"backend": "local"},
|
||||||
|
"a/profile": map[string]any{"backend": "local", "model": "a-model"},
|
||||||
|
" z/profile ": map[string]any{"backend": "local", "model": "spaced-model"},
|
||||||
|
},
|
||||||
|
"prompts": map[string]any{
|
||||||
|
"z/prompt": map[string]any{"v1": listingPromptDefinition()},
|
||||||
|
"a/prompt": map[string]any{"v1": listingPromptDefinition()},
|
||||||
|
" z/prompt ": map[string]any{
|
||||||
|
"2": listingPromptDefinition(),
|
||||||
|
"10": listingPromptDefinition(),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}), nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
profiles, err := state.ListProfiles(context.Background())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
wantProfiles := []string{" z/profile ", "a/profile", "partial/profile", "z/profile"}
|
||||||
|
if !reflect.DeepEqual(profiles, wantProfiles) {
|
||||||
|
t.Fatalf("profile IDs = %#v, want %#v", profiles, wantProfiles)
|
||||||
|
}
|
||||||
|
|
||||||
|
prompts, err := state.ListPrompts(context.Background())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
wantPrompts := []domain.PromptIdentity{
|
||||||
|
{ID: " z/prompt ", Version: "10"},
|
||||||
|
{ID: " z/prompt ", Version: "2"},
|
||||||
|
{ID: "a/prompt", Version: "v1"},
|
||||||
|
{ID: "z/prompt", Version: "v1"},
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(prompts, wantPrompts) {
|
||||||
|
t.Fatalf("prompt identities = %#v, want %#v", prompts, wantPrompts)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLoadedListsAreIndependent(t *testing.T) {
|
||||||
|
state, err := Load(mapping("identities", map[string]any{
|
||||||
|
"profiles": map[string]any{"profile": map[string]any{}},
|
||||||
|
"prompts": map[string]any{"example": map[string]any{"v1": listingPromptDefinition()}},
|
||||||
|
}), nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
profiles, err := state.ListProfiles(context.Background())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
profiles = append(profiles, "caller profile")
|
||||||
|
profiles[0] = "changed profile"
|
||||||
|
|
||||||
|
prompts, err := state.ListPrompts(context.Background())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
prompts[0].ID = "changed prompt"
|
||||||
|
prompts[0].Version = "changed version"
|
||||||
|
|
||||||
|
profiles, err = state.ListProfiles(context.Background())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(profiles, []string{"profile"}) {
|
||||||
|
t.Fatalf("caller mutation changed profile inventory: %#v", profiles)
|
||||||
|
}
|
||||||
|
prompts, err = state.ListPrompts(context.Background())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
wantPrompts := []domain.PromptIdentity{{ID: "example", Version: "v1"}}
|
||||||
|
if !reflect.DeepEqual(prompts, wantPrompts) {
|
||||||
|
t.Fatalf("caller mutation changed prompt inventory: %#v", prompts)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLoadedListsEmptyInventory(t *testing.T) {
|
||||||
|
state, err := Load(configkit.Input{Source: configkit.NoSource()}, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
profiles, err := state.ListProfiles(context.Background())
|
||||||
|
if err != nil || len(profiles) != 0 {
|
||||||
|
t.Fatalf("empty profile listing = %#v, %v", profiles, err)
|
||||||
|
}
|
||||||
|
prompts, err := state.ListPrompts(context.Background())
|
||||||
|
if err != nil || len(prompts) != 0 {
|
||||||
|
t.Fatalf("empty prompt listing = %#v, %v", prompts, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLoadedListsHonorCanceledContext(t *testing.T) {
|
||||||
|
state, err := Load(mapping("identities", map[string]any{
|
||||||
|
"profiles": map[string]any{"partial": map[string]any{}},
|
||||||
|
"prompts": map[string]any{"example": map[string]any{"v1": listingPromptDefinition()}},
|
||||||
|
}), nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
cancel()
|
||||||
|
|
||||||
|
profiles, err := state.ListProfiles(ctx)
|
||||||
|
if !errors.Is(err, context.Canceled) || len(profiles) != 0 {
|
||||||
|
t.Fatalf("canceled profile listing = %#v, %v", profiles, err)
|
||||||
|
}
|
||||||
|
prompts, err := state.ListPrompts(ctx)
|
||||||
|
if !errors.Is(err, context.Canceled) || len(prompts) != 0 {
|
||||||
|
t.Fatalf("canceled prompt listing = %#v, %v", prompts, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func listingPromptDefinition() map[string]any {
|
||||||
|
return map[string]any{
|
||||||
|
"messages": []any{map[string]any{"role": "user", "content": "Hello."}},
|
||||||
|
"output": map[string]any{"format": "text", "validation_mode": "basic"},
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -54,10 +54,10 @@ type promptConfig struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type promptInputConfig struct {
|
type promptInputConfig struct {
|
||||||
Name string `yaml:"name"`
|
Name string `yaml:"name"`
|
||||||
Required bool `yaml:"required"`
|
Required bool `yaml:"required"`
|
||||||
ContentType string `yaml:"content_type"`
|
ContentType *string `yaml:"content_type"`
|
||||||
Description string `yaml:"description"`
|
Description *string `yaml:"description"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type promptMessageConfig struct {
|
type promptMessageConfig struct {
|
||||||
@@ -93,10 +93,13 @@ type loaded struct {
|
|||||||
result *configkit.Result[document]
|
result *configkit.Result[document]
|
||||||
backends map[string]map[string]fieldState
|
backends map[string]map[string]fieldState
|
||||||
profiles map[string]map[string]fieldState
|
profiles map[string]map[string]fieldState
|
||||||
|
promptMetadata map[string]map[string][]promptInputPresence
|
||||||
prompts map[string]map[string]*promptEntry
|
prompts map[string]map[string]*promptEntry
|
||||||
ambiguous map[string]bool
|
ambiguous map[string]bool
|
||||||
registry *backend.Registry
|
registry *backend.Registry
|
||||||
indexedProfiles map[string]domain.ExecutionProfile
|
indexedProfiles map[string]domain.ExecutionProfile
|
||||||
|
profileIDs []string
|
||||||
|
promptIDs []domain.PromptIdentity
|
||||||
}
|
}
|
||||||
|
|
||||||
var envName = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`)
|
var envName = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`)
|
||||||
@@ -112,8 +115,9 @@ func Load(main configkit.Input, underlays []configkit.Underlay) (*loaded, error)
|
|||||||
// load performs one ConfigKit lifecycle over the selected source tiers.
|
// load performs one ConfigKit lifecycle over the selected source tiers.
|
||||||
func load(main configkit.Input, underlays []configkit.Underlay) (*loaded, error) {
|
func load(main configkit.Input, underlays []configkit.Underlay) (*loaded, error) {
|
||||||
state := &loaded{
|
state := &loaded{
|
||||||
backends: make(map[string]map[string]fieldState),
|
backends: make(map[string]map[string]fieldState),
|
||||||
profiles: make(map[string]map[string]fieldState),
|
profiles: make(map[string]map[string]fieldState),
|
||||||
|
promptMetadata: make(map[string]map[string][]promptInputPresence),
|
||||||
}
|
}
|
||||||
result, err := configkit.LoadLayered(main, configkit.LayeredOptions[document]{
|
result, err := configkit.LoadLayered(main, configkit.LayeredOptions[document]{
|
||||||
Underlays: underlays,
|
Underlays: underlays,
|
||||||
@@ -122,6 +126,7 @@ func load(main configkit.Input, underlays []configkit.Underlay) (*loaded, error)
|
|||||||
Resolve: func(config *document, resolution *configkit.Resolution) error {
|
Resolve: func(config *document, resolution *configkit.Resolution) error {
|
||||||
captureFields(config.Backends, "backends", backendFields, resolution, state.backends)
|
captureFields(config.Backends, "backends", backendFields, resolution, state.backends)
|
||||||
captureFields(config.Profiles, "profiles", profileFields, resolution, state.profiles)
|
captureFields(config.Profiles, "profiles", profileFields, resolution, state.profiles)
|
||||||
|
capturePromptMetadata(config.Prompts, resolution, state.promptMetadata)
|
||||||
resolveBackends(config.Backends, resolution)
|
resolveBackends(config.Backends, resolution)
|
||||||
resolveProfiles(config.Profiles, resolution)
|
resolveProfiles(config.Profiles, resolution)
|
||||||
resolvePrompts(config.Prompts, resolution)
|
resolvePrompts(config.Prompts, resolution)
|
||||||
@@ -140,9 +145,53 @@ func load(main configkit.Input, underlays []configkit.Underlay) (*loaded, error)
|
|||||||
if err := state.buildExecutionProfiles(); err != nil {
|
if err := state.buildExecutionProfiles(); err != nil {
|
||||||
return nil, fmt.Errorf("load configuration: %w", err)
|
return nil, fmt.Errorf("load configuration: %w", err)
|
||||||
}
|
}
|
||||||
|
state.profileIDs = sortedKeys(state.indexedProfiles)
|
||||||
|
for _, id := range sortedKeys(state.prompts) {
|
||||||
|
for _, version := range sortedKeys(state.prompts[id]) {
|
||||||
|
state.promptIDs = append(state.promptIDs, domain.PromptIdentity{ID: id, Version: version})
|
||||||
|
}
|
||||||
|
}
|
||||||
return state, nil
|
return state, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type promptInputPresence struct {
|
||||||
|
contentType fieldState
|
||||||
|
description fieldState
|
||||||
|
}
|
||||||
|
|
||||||
|
func capturePromptMetadata(prompts map[string]map[string]promptConfig, resolution *configkit.Resolution, destination map[string]map[string][]promptInputPresence) {
|
||||||
|
for _, id := range sortedKeys(prompts) {
|
||||||
|
if destination[id] == nil {
|
||||||
|
destination[id] = make(map[string][]promptInputPresence, len(prompts[id]))
|
||||||
|
}
|
||||||
|
for _, version := range sortedKeys(prompts[id]) {
|
||||||
|
inputs := prompts[id][version].Inputs
|
||||||
|
states := make([]promptInputPresence, len(inputs))
|
||||||
|
base := configkit.Keys("prompts", id, version, "inputs")
|
||||||
|
for index := range inputs {
|
||||||
|
path := base.Index(index)
|
||||||
|
states[index] = promptInputPresence{
|
||||||
|
contentType: captureFieldState(path.Key("content_type"), resolution),
|
||||||
|
description: captureFieldState(path.Key("description"), resolution),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
destination[id][version] = states
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func captureFieldState(path configkit.Path, resolution *configkit.Resolution) fieldState {
|
||||||
|
info, exists := resolution.InputInfo(path)
|
||||||
|
switch {
|
||||||
|
case !exists || !info.Explicit:
|
||||||
|
return fieldAbsent
|
||||||
|
case info.Null:
|
||||||
|
return fieldNull
|
||||||
|
default:
|
||||||
|
return fieldValue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func captureFields[T any](entries map[string]T, collection string, fields []string, resolution *configkit.Resolution, destination map[string]map[string]fieldState) {
|
func captureFields[T any](entries map[string]T, collection string, fields []string, resolution *configkit.Resolution, destination map[string]map[string]fieldState) {
|
||||||
for id := range entries {
|
for id := range entries {
|
||||||
states := make(map[string]fieldState, len(fields))
|
states := make(map[string]fieldState, len(fields))
|
||||||
|
|||||||
@@ -21,11 +21,12 @@ var (
|
|||||||
)
|
)
|
||||||
|
|
||||||
type promptEntry struct {
|
type promptEntry struct {
|
||||||
definition domain.PromptDefinition
|
definition domain.PromptDefinition
|
||||||
inspection domain.PromptInspection
|
inspection domain.PromptInspection
|
||||||
hash string
|
requirementInputs []prompt.InputMetadata
|
||||||
session *template.Template
|
hash string
|
||||||
messages []*template.Template
|
session *template.Template
|
||||||
|
messages []*template.Template
|
||||||
}
|
}
|
||||||
|
|
||||||
func resolvePrompts(prompts map[string]map[string]promptConfig, resolution *configkit.Resolution) {
|
func resolvePrompts(prompts map[string]map[string]promptConfig, resolution *configkit.Resolution) {
|
||||||
@@ -48,18 +49,24 @@ func resolvePrompts(prompts map[string]map[string]promptConfig, resolution *conf
|
|||||||
for index := range definition.Inputs {
|
for index := range definition.Inputs {
|
||||||
input := &definition.Inputs[index]
|
input := &definition.Inputs[index]
|
||||||
inputPath := base.Key("inputs").Index(index)
|
inputPath := base.Key("inputs").Index(index)
|
||||||
|
normalizedName := strings.TrimSpace(input.Name)
|
||||||
|
if normalizedName != input.Name {
|
||||||
|
configkit.Normalize(resolution, inputPath.Key("name"), &input.Name, normalizedName, "normalize input metadata")
|
||||||
|
inputsChanged = true
|
||||||
|
}
|
||||||
for _, field := range []struct {
|
for _, field := range []struct {
|
||||||
name string
|
name string
|
||||||
value *string
|
value **string
|
||||||
}{
|
}{
|
||||||
{"name", &input.Name},
|
|
||||||
{"content_type", &input.ContentType},
|
{"content_type", &input.ContentType},
|
||||||
{"description", &input.Description},
|
{"description", &input.Description},
|
||||||
} {
|
} {
|
||||||
normalized := strings.TrimSpace(*field.value)
|
if *field.value != nil {
|
||||||
if normalized != *field.value {
|
normalized := strings.TrimSpace(**field.value)
|
||||||
configkit.Normalize(resolution, inputPath.Key(field.name), field.value, normalized, "normalize input metadata")
|
if normalized != **field.value {
|
||||||
inputsChanged = true
|
configkit.Normalize(resolution, inputPath.Key(field.name), field.value, &normalized, "normalize input metadata")
|
||||||
|
inputsChanged = true
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -132,6 +139,10 @@ func (state *loaded) materializePrompt(id, version string, raw promptConfig, loo
|
|||||||
}
|
}
|
||||||
seenInputs := make(map[string]bool, len(raw.Inputs))
|
seenInputs := make(map[string]bool, len(raw.Inputs))
|
||||||
definition.Inputs = make([]domain.PromptInput, 0, len(raw.Inputs))
|
definition.Inputs = make([]domain.PromptInput, 0, len(raw.Inputs))
|
||||||
|
entry := &promptEntry{
|
||||||
|
messages: make([]*template.Template, 0, len(raw.Messages)),
|
||||||
|
requirementInputs: make([]prompt.InputMetadata, 0, len(raw.Inputs)),
|
||||||
|
}
|
||||||
for index, input := range raw.Inputs {
|
for index, input := range raw.Inputs {
|
||||||
path := base.Key("inputs").Index(index)
|
path := base.Key("inputs").Index(index)
|
||||||
if input.Name == "" {
|
if input.Name == "" {
|
||||||
@@ -141,11 +152,27 @@ func (state *loaded) materializePrompt(id, version string, raw promptConfig, loo
|
|||||||
return nil, fmt.Errorf("%s: input name is duplicated", path.Key("name"))
|
return nil, fmt.Errorf("%s: input name is duplicated", path.Key("name"))
|
||||||
}
|
}
|
||||||
seenInputs[input.Name] = true
|
seenInputs[input.Name] = true
|
||||||
|
contentType := ""
|
||||||
|
if input.ContentType != nil {
|
||||||
|
contentType = *input.ContentType
|
||||||
|
}
|
||||||
|
description := ""
|
||||||
|
if input.Description != nil {
|
||||||
|
description = *input.Description
|
||||||
|
}
|
||||||
definition.Inputs = append(definition.Inputs, domain.PromptInput{
|
definition.Inputs = append(definition.Inputs, domain.PromptInput{
|
||||||
Name: input.Name, Required: input.Required, ContentType: input.ContentType, Description: input.Description,
|
Name: input.Name, Required: input.Required, ContentType: contentType, Description: description,
|
||||||
})
|
})
|
||||||
|
presence := state.promptMetadata[id][version][index]
|
||||||
|
metadata := prompt.InputMetadata{Name: input.Name, Required: input.Required}
|
||||||
|
if presence.contentType == fieldValue {
|
||||||
|
metadata.ContentType = stringPointer(contentType)
|
||||||
|
}
|
||||||
|
if presence.description == fieldValue {
|
||||||
|
metadata.Description = stringPointer(description)
|
||||||
|
}
|
||||||
|
entry.requirementInputs = append(entry.requirementInputs, metadata)
|
||||||
}
|
}
|
||||||
entry := &promptEntry{messages: make([]*template.Template, 0, len(raw.Messages))}
|
|
||||||
definition.Templates = make([]domain.PromptMessageTemplate, 0, len(raw.Messages))
|
definition.Templates = make([]domain.PromptMessageTemplate, 0, len(raw.Messages))
|
||||||
for index, message := range raw.Messages {
|
for index, message := range raw.Messages {
|
||||||
path := base.Key("messages").Index(index)
|
path := base.Key("messages").Index(index)
|
||||||
@@ -233,6 +260,11 @@ func (state *loaded) materializePrompt(id, version string, raw promptConfig, loo
|
|||||||
return entry, nil
|
return entry, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func stringPointer(value string) *string {
|
||||||
|
copy := value
|
||||||
|
return ©
|
||||||
|
}
|
||||||
|
|
||||||
func (state *loaded) readContent(fieldPath configkit.Path, name string, lookupOrigin func(configkit.Path) (configkit.FieldInfo, bool)) (string, error) {
|
func (state *loaded) readContent(fieldPath configkit.Path, name string, lookupOrigin func(configkit.Path) (configkit.FieldInfo, bool)) (string, error) {
|
||||||
if strings.TrimSpace(name) == "" {
|
if strings.TrimSpace(name) == "" {
|
||||||
return "", fmt.Errorf("%s: content_file path must not be blank", fieldPath)
|
return "", fmt.Errorf("%s: content_file path must not be blank", fieldPath)
|
||||||
@@ -265,14 +297,14 @@ func (state *loaded) readContent(fieldPath configkit.Path, name string, lookupOr
|
|||||||
return string(content), nil
|
return string(content), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (state *loaded) prompt(id, version string) (*domain.PromptDefinition, domain.PromptInspection, error) {
|
func (state *loaded) lookupPromptEntry(id, version string) (*promptEntry, error) {
|
||||||
if state == nil || strings.TrimSpace(id) == "" || (version != "" && strings.TrimSpace(version) == "") {
|
if state == nil || strings.TrimSpace(id) == "" || (version != "" && strings.TrimSpace(version) == "") {
|
||||||
return nil, domain.PromptInspection{}, errPromptNotFound
|
return nil, errPromptNotFound
|
||||||
}
|
}
|
||||||
versions := state.prompts[id]
|
versions := state.prompts[id]
|
||||||
if version == "" {
|
if version == "" {
|
||||||
if state.ambiguous[id] {
|
if state.ambiguous[id] {
|
||||||
return nil, domain.PromptInspection{}, errPromptAmbiguous
|
return nil, errPromptAmbiguous
|
||||||
}
|
}
|
||||||
for selected := range versions {
|
for selected := range versions {
|
||||||
version = selected
|
version = selected
|
||||||
@@ -280,9 +312,12 @@ func (state *loaded) prompt(id, version string) (*domain.PromptDefinition, domai
|
|||||||
}
|
}
|
||||||
entry := versions[version]
|
entry := versions[version]
|
||||||
if entry == nil {
|
if entry == nil {
|
||||||
return nil, domain.PromptInspection{}, errPromptNotFound
|
return nil, errPromptNotFound
|
||||||
}
|
}
|
||||||
definition := entry.definition
|
return entry, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func ownPromptDefinition(definition domain.PromptDefinition) *domain.PromptDefinition {
|
||||||
definition.Inputs = append([]domain.PromptInput(nil), definition.Inputs...)
|
definition.Inputs = append([]domain.PromptInput(nil), definition.Inputs...)
|
||||||
definition.Templates = append([]domain.PromptMessageTemplate(nil), definition.Templates...)
|
definition.Templates = append([]domain.PromptMessageTemplate(nil), definition.Templates...)
|
||||||
for index := range definition.Templates {
|
for index := range definition.Templates {
|
||||||
@@ -291,9 +326,27 @@ func (state *loaded) prompt(id, version string) (*domain.PromptDefinition, domai
|
|||||||
definition.Templates[index].CacheControl = ©
|
definition.Templates[index].CacheControl = ©
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
inspection := entry.inspection
|
return &definition
|
||||||
|
}
|
||||||
|
|
||||||
|
func ownPromptInspection(inspection domain.PromptInspection) *domain.PromptInspection {
|
||||||
inspection.Inputs = append([]domain.PromptInput(nil), inspection.Inputs...)
|
inspection.Inputs = append([]domain.PromptInput(nil), inspection.Inputs...)
|
||||||
return &definition, inspection, nil
|
return &inspection
|
||||||
|
}
|
||||||
|
|
||||||
|
func promptLookupError(err error) error {
|
||||||
|
if errors.Is(err, errPromptNotFound) {
|
||||||
|
return promptdef.ErrPromptDefinitionNotFound
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (state *loaded) prompt(id, version string) (*domain.PromptDefinition, domain.PromptInspection, error) {
|
||||||
|
entry, err := state.lookupPromptEntry(id, version)
|
||||||
|
if err != nil {
|
||||||
|
return nil, domain.PromptInspection{}, err
|
||||||
|
}
|
||||||
|
return ownPromptDefinition(entry.definition), *ownPromptInspection(entry.inspection), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetPromptSnapshot returns an independently owned captured definition and its
|
// GetPromptSnapshot returns an independently owned captured definition and its
|
||||||
@@ -302,14 +355,24 @@ func (state *loaded) GetPromptSnapshot(ctx context.Context, id, version string)
|
|||||||
if err := ctx.Err(); err != nil {
|
if err := ctx.Err(); err != nil {
|
||||||
return nil, "", err
|
return nil, "", err
|
||||||
}
|
}
|
||||||
definition, inspection, err := state.prompt(id, version)
|
entry, err := state.lookupPromptEntry(id, version)
|
||||||
if errors.Is(err, errPromptNotFound) {
|
|
||||||
return nil, "", promptdef.ErrPromptDefinitionNotFound
|
|
||||||
}
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, "", err
|
return nil, "", promptLookupError(err)
|
||||||
}
|
}
|
||||||
return definition, inspection.PromptHash, nil
|
return ownPromptDefinition(entry.definition), entry.hash, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetPromptInspection returns an independently owned inspection of one
|
||||||
|
// captured prompt without copying its execution templates.
|
||||||
|
func (state *loaded) GetPromptInspection(ctx context.Context, id, version string) (*domain.PromptInspection, error) {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
entry, err := state.lookupPromptEntry(id, version)
|
||||||
|
if err != nil {
|
||||||
|
return nil, promptLookupError(err)
|
||||||
|
}
|
||||||
|
return ownPromptInspection(entry.inspection), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetPromptDefinition implements the runner's prompt lookup boundary.
|
// GetPromptDefinition implements the runner's prompt lookup boundary.
|
||||||
@@ -329,3 +392,33 @@ func (state *loaded) GetCompiledTemplates(id, version string) (prompt.CompiledTe
|
|||||||
}
|
}
|
||||||
return prompt.CompiledTemplates{Session: entry.session, Messages: entry.messages}, true
|
return prompt.CompiledTemplates{Session: entry.session, Messages: entry.messages}, true
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// GetRequirementSnapshot returns captured templates and input metadata for
|
||||||
|
// one exact prompt identity. Returned metadata and slices are independently
|
||||||
|
// owned; the compiled templates are shared immutable construction-time parses.
|
||||||
|
func (state *loaded) GetRequirementSnapshot(id, version string) (prompt.RequirementSnapshot, bool) {
|
||||||
|
if state == nil {
|
||||||
|
return prompt.RequirementSnapshot{}, false
|
||||||
|
}
|
||||||
|
entry := state.prompts[id][version]
|
||||||
|
if entry == nil {
|
||||||
|
return prompt.RequirementSnapshot{}, false
|
||||||
|
}
|
||||||
|
snapshot := prompt.RequirementSnapshot{
|
||||||
|
Templates: prompt.CompiledTemplates{
|
||||||
|
Session: entry.session,
|
||||||
|
Messages: append([]*template.Template(nil), entry.messages...),
|
||||||
|
},
|
||||||
|
Inputs: make([]prompt.InputMetadata, len(entry.requirementInputs)),
|
||||||
|
}
|
||||||
|
for index, input := range entry.requirementInputs {
|
||||||
|
snapshot.Inputs[index] = input
|
||||||
|
if input.ContentType != nil {
|
||||||
|
snapshot.Inputs[index].ContentType = stringPointer(*input.ContentType)
|
||||||
|
}
|
||||||
|
if input.Description != nil {
|
||||||
|
snapshot.Inputs[index].Description = stringPointer(*input.Description)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return snapshot, true
|
||||||
|
}
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
package configuration
|
package configuration
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
"io/fs"
|
"io/fs"
|
||||||
"os"
|
"os"
|
||||||
@@ -10,6 +11,7 @@ import (
|
|||||||
"testing/fstest"
|
"testing/fstest"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
"gitea.maximumdirect.net/eric/ConfigKit"
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/promptdef"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestLoadCapturesInheritedImportedContent(t *testing.T) {
|
func TestLoadCapturesInheritedImportedContent(t *testing.T) {
|
||||||
@@ -134,6 +136,94 @@ func TestLoadRejectsMalformedStaticSyntaxAndAmbiguousSelection(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestPromptInspectionLookupMatchesSnapshotSelectionAndOwnsInputs(t *testing.T) {
|
||||||
|
makePrompt := func(content string) map[string]any {
|
||||||
|
return map[string]any{
|
||||||
|
"default_profile": "partial",
|
||||||
|
"inputs": []any{map[string]any{
|
||||||
|
"name": "document", "required": true,
|
||||||
|
"content_type": " text/plain ", "description": " source document ",
|
||||||
|
}},
|
||||||
|
"messages": []any{map[string]any{"role": "user", "content": content}},
|
||||||
|
"output": map[string]any{"format": "text", "validation_mode": "none"},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
state, err := load(mapping("prompt inspection lookup", map[string]any{
|
||||||
|
"profiles": map[string]any{"partial": map[string]any{}},
|
||||||
|
"prompts": map[string]any{
|
||||||
|
"multi": map[string]any{"v1": makePrompt("first"), "v2": makePrompt("second")},
|
||||||
|
"single": map[string]any{"v1": makePrompt("only")},
|
||||||
|
},
|
||||||
|
}), nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, selection := range []struct {
|
||||||
|
id, version string
|
||||||
|
}{
|
||||||
|
{id: "multi", version: "v2"},
|
||||||
|
{id: "single"},
|
||||||
|
} {
|
||||||
|
inspection, err := state.GetPromptInspection(context.Background(), selection.id, selection.version)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("inspect %q/%q: %v", selection.id, selection.version, err)
|
||||||
|
}
|
||||||
|
definition, hash, err := state.GetPromptSnapshot(context.Background(), selection.id, selection.version)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("snapshot %q/%q: %v", selection.id, selection.version, err)
|
||||||
|
}
|
||||||
|
if inspection.PromptID != definition.ID || inspection.PromptVersion != definition.Version ||
|
||||||
|
inspection.PromptHash != hash || inspection.DefaultProfileID != "partial" ||
|
||||||
|
len(inspection.Inputs) != 1 || inspection.Inputs[0].Name != "document" ||
|
||||||
|
inspection.Inputs[0].ContentType != "text/plain" || inspection.Inputs[0].Description != "source document" {
|
||||||
|
t.Fatalf("inspection did not match the selected snapshot: inspection=%+v definition=%+v hash=%q", inspection, definition, hash)
|
||||||
|
}
|
||||||
|
if len(definition.Inputs) != len(inspection.Inputs) || definition.Inputs[0].Name != inspection.Inputs[0].Name {
|
||||||
|
t.Fatalf("definition and inspection inputs diverged: definition=%+v inspection=%+v", definition.Inputs, inspection.Inputs)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
owned, err := state.GetPromptInspection(context.Background(), "single", "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
owned.Inputs[0].Name = "caller mutation"
|
||||||
|
owned.Inputs[0].Description = "caller mutation"
|
||||||
|
again, err := state.GetPromptInspection(context.Background(), "single", "v1")
|
||||||
|
if err != nil || again.Inputs[0].Name != "document" || again.Inputs[0].Description != "source document" {
|
||||||
|
t.Fatalf("inspection mutation escaped into captured state: inspection=%+v err=%v", again, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, selection := range []struct {
|
||||||
|
id, version string
|
||||||
|
}{
|
||||||
|
{id: "missing", version: "v1"},
|
||||||
|
{id: "multi", version: "V2"},
|
||||||
|
{id: " ", version: "v1"},
|
||||||
|
} {
|
||||||
|
inspection, err := state.GetPromptInspection(context.Background(), selection.id, selection.version)
|
||||||
|
if inspection != nil || !errors.Is(err, promptdef.ErrPromptDefinitionNotFound) {
|
||||||
|
t.Errorf("inspection selection %q/%q=(%+v, %v), want not found", selection.id, selection.version, inspection, err)
|
||||||
|
}
|
||||||
|
definition, _, err := state.GetPromptSnapshot(context.Background(), selection.id, selection.version)
|
||||||
|
if definition != nil || !errors.Is(err, promptdef.ErrPromptDefinitionNotFound) {
|
||||||
|
t.Errorf("snapshot selection %q/%q=(%+v, %v), want not found", selection.id, selection.version, definition, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if _, err := state.GetPromptInspection(context.Background(), "multi", ""); !errors.Is(err, errPromptAmbiguous) {
|
||||||
|
t.Errorf("ambiguous inspection selection error=%v", err)
|
||||||
|
}
|
||||||
|
if _, _, err := state.GetPromptSnapshot(context.Background(), "multi", ""); !errors.Is(err, errPromptAmbiguous) {
|
||||||
|
t.Errorf("ambiguous snapshot selection error=%v", err)
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
cancel()
|
||||||
|
if _, err := state.GetPromptInspection(ctx, "single", "v1"); !errors.Is(err, context.Canceled) {
|
||||||
|
t.Errorf("canceled inspection selection error=%v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestLoadAcceptsEmptyInlineContentAndRejectsInvalidPromptFields(t *testing.T) {
|
func TestLoadAcceptsEmptyInlineContentAndRejectsInvalidPromptFields(t *testing.T) {
|
||||||
state, err := load(mapping("empty-content", map[string]any{"prompts": map[string]any{" exact.id ": map[string]any{"v1": map[string]any{
|
state, err := load(mapping("empty-content", map[string]any{"prompts": map[string]any{" exact.id ": map[string]any{"v1": map[string]any{
|
||||||
"inputs": []any{map[string]any{"name": " note ", "required": true}},
|
"inputs": []any{map[string]any{"name": " note ", "required": true}},
|
||||||
@@ -170,6 +260,113 @@ func TestLoadAcceptsEmptyInlineContentAndRejectsInvalidPromptFields(t *testing.T
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRequirementSnapshotCapturesMetadataPresenceAndNormalizedValues(t *testing.T) {
|
||||||
|
state, err := load(mapping("requirement-metadata", map[string]any{
|
||||||
|
"prompts": map[string]any{"example": map[string]any{"v1": map[string]any{
|
||||||
|
"session_id": "{{.session}}",
|
||||||
|
"inputs": []any{
|
||||||
|
map[string]any{"name": "absent"},
|
||||||
|
map[string]any{"name": "null", "content_type": nil, "description": nil},
|
||||||
|
map[string]any{"name": "empty", "content_type": "", "description": ""},
|
||||||
|
map[string]any{"name": "normalized", "content_type": " text/plain ", "description": " final text "},
|
||||||
|
},
|
||||||
|
"messages": []any{map[string]any{"role": "user", "content": "{{input \"absent\"}}"}},
|
||||||
|
"output": map[string]any{"format": "text", "validation_mode": "basic"},
|
||||||
|
}}},
|
||||||
|
}), nil)
|
||||||
|
if err != nil {
|
||||||
|
var loadError *configkit.LoadError
|
||||||
|
if errors.As(err, &loadError) {
|
||||||
|
t.Fatalf("%v: %+v", err, loadError.Diagnostics())
|
||||||
|
}
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
snapshot, ok := state.GetRequirementSnapshot("example", "v1")
|
||||||
|
if !ok || len(snapshot.Inputs) != 4 || len(snapshot.Templates.Messages) != 1 || snapshot.Templates.Session == nil {
|
||||||
|
t.Fatalf("requirement snapshot was not captured: %+v, %t", snapshot, ok)
|
||||||
|
}
|
||||||
|
for _, index := range []int{0, 1} {
|
||||||
|
if snapshot.Inputs[index].ContentType != nil || snapshot.Inputs[index].Description != nil {
|
||||||
|
t.Errorf("input %q should not have explicit string metadata: %+v", snapshot.Inputs[index].Name, snapshot.Inputs[index])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if snapshot.Inputs[2].ContentType == nil || *snapshot.Inputs[2].ContentType != "" ||
|
||||||
|
snapshot.Inputs[2].Description == nil || *snapshot.Inputs[2].Description != "" {
|
||||||
|
t.Errorf("explicit empty metadata presence was lost: %+v", snapshot.Inputs[2])
|
||||||
|
}
|
||||||
|
if snapshot.Inputs[3].ContentType == nil || *snapshot.Inputs[3].ContentType != "text/plain" ||
|
||||||
|
snapshot.Inputs[3].Description == nil || *snapshot.Inputs[3].Description != "final text" {
|
||||||
|
t.Errorf("snapshot did not use normalized final metadata: %+v", snapshot.Inputs[3])
|
||||||
|
}
|
||||||
|
|
||||||
|
definition, inspection, err := state.prompt("example", "v1")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(definition.Inputs) != 4 || len(inspection.Inputs) != 4 || inspection.Inputs[2].ContentType != "" || inspection.Inputs[2].Description != "" ||
|
||||||
|
inspection.Inputs[3].ContentType != "text/plain" || inspection.Inputs[3].Description != "final text" {
|
||||||
|
t.Fatalf("existing prompt inspection shape or normalized values changed: %+v", inspection)
|
||||||
|
}
|
||||||
|
|
||||||
|
*snapshot.Inputs[2].ContentType = "caller mutation"
|
||||||
|
snapshot.Inputs[3].Name = "caller mutation"
|
||||||
|
snapshot.Templates.Messages[0] = nil
|
||||||
|
again, ok := state.GetRequirementSnapshot("example", "v1")
|
||||||
|
if !ok || *again.Inputs[2].ContentType != "" || again.Inputs[3].Name != "normalized" || again.Templates.Messages[0] == nil {
|
||||||
|
t.Fatalf("snapshot mutation escaped into captured state: %+v, %t", again, ok)
|
||||||
|
}
|
||||||
|
if _, ok := state.GetRequirementSnapshot("example", "V1"); ok {
|
||||||
|
t.Fatal("snapshot lookup unexpectedly changed exact version matching")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestInputMetadataPresenceDoesNotChangePromptHashOrInspection(t *testing.T) {
|
||||||
|
variants := []struct {
|
||||||
|
name string
|
||||||
|
fields map[string]any
|
||||||
|
}{
|
||||||
|
{name: "absent"},
|
||||||
|
{name: "null", fields: map[string]any{"content_type": nil, "description": nil}},
|
||||||
|
{name: "empty", fields: map[string]any{"content_type": "", "description": ""}},
|
||||||
|
}
|
||||||
|
var expectedHash string
|
||||||
|
for _, variant := range variants {
|
||||||
|
t.Run(variant.name, func(t *testing.T) {
|
||||||
|
input := map[string]any{"name": "document"}
|
||||||
|
for name, value := range variant.fields {
|
||||||
|
input[name] = value
|
||||||
|
}
|
||||||
|
state, err := load(mapping("hash-"+variant.name, map[string]any{
|
||||||
|
"prompts": map[string]any{"same": map[string]any{"v1": map[string]any{
|
||||||
|
"inputs": []any{input},
|
||||||
|
"messages": []any{map[string]any{"role": "user", "content": "hello"}},
|
||||||
|
"output": map[string]any{"format": "text", "validation_mode": "basic"},
|
||||||
|
}}},
|
||||||
|
}), nil)
|
||||||
|
if err != nil {
|
||||||
|
var loadError *configkit.LoadError
|
||||||
|
if errors.As(err, &loadError) {
|
||||||
|
t.Fatalf("%v: %+v", err, loadError.Diagnostics())
|
||||||
|
}
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
_, inspection, err := state.prompt("same", "v1")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(inspection.Inputs) != 1 || inspection.Inputs[0].ContentType != "" || inspection.Inputs[0].Description != "" {
|
||||||
|
t.Fatalf("old inspection metadata changed: %+v", inspection.Inputs)
|
||||||
|
}
|
||||||
|
if expectedHash == "" {
|
||||||
|
expectedHash = inspection.PromptHash
|
||||||
|
} else if inspection.PromptHash != expectedHash {
|
||||||
|
t.Fatalf("metadata presence changed the definition hash: got %q, want %q", inspection.PromptHash, expectedHash)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func promptWithFile(name string) map[string]any {
|
func promptWithFile(name string) map[string]any {
|
||||||
return map[string]any{"prompts": map[string]any{"example": map[string]any{"v1": map[string]any{
|
return map[string]any{"prompts": map[string]any{"example": map[string]any{"v1": map[string]any{
|
||||||
"messages": []any{map[string]any{"role": "user", "content_file": name}},
|
"messages": []any{map[string]any{"role": "user", "content_file": name}},
|
||||||
|
|||||||
@@ -147,18 +147,26 @@ func overlayProviderMap(lower, upper map[string]any) (map[string]any, error) {
|
|||||||
return merged, nil
|
return merged, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetProfile returns an independently owned indexed profile for the internal
|
func (state *loaded) lookupProfile(ctx context.Context, id string) (domain.ExecutionProfile, error) {
|
||||||
// runner's lookup boundary.
|
|
||||||
func (state *loaded) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) {
|
|
||||||
if err := ctx.Err(); err != nil {
|
if err := ctx.Err(); err != nil {
|
||||||
return nil, err
|
return domain.ExecutionProfile{}, err
|
||||||
}
|
}
|
||||||
if state == nil {
|
if state == nil {
|
||||||
return nil, fmt.Errorf("%w: %q", profilepkg.ErrProfileNotFound, id)
|
return domain.ExecutionProfile{}, fmt.Errorf("%w: %q", profilepkg.ErrProfileNotFound, id)
|
||||||
}
|
}
|
||||||
profile, found := state.indexedProfiles[id]
|
profile, found := state.indexedProfiles[id]
|
||||||
if !found {
|
if !found {
|
||||||
return nil, fmt.Errorf("%w: %q", profilepkg.ErrProfileNotFound, id)
|
return domain.ExecutionProfile{}, fmt.Errorf("%w: %q", profilepkg.ErrProfileNotFound, id)
|
||||||
|
}
|
||||||
|
return profile, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetProfile returns an independently owned indexed profile for the internal
|
||||||
|
// runner's lookup boundary.
|
||||||
|
func (state *loaded) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) {
|
||||||
|
profile, err := state.lookupProfile(ctx, id)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
}
|
}
|
||||||
target := *profile.ResolvedTarget
|
target := *profile.ResolvedTarget
|
||||||
params, err := jsonvalue.CopyMap(target.ExtraParams)
|
params, err := jsonvalue.CopyMap(target.ExtraParams)
|
||||||
@@ -170,6 +178,15 @@ func (state *loaded) GetProfile(ctx context.Context, id string) (*domain.Executi
|
|||||||
return &profile, nil
|
return &profile, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// GetProfileIdentity returns the exact lookup ID for a captured profile
|
||||||
|
// without copying its execution target.
|
||||||
|
func (state *loaded) GetProfileIdentity(ctx context.Context, id string) (string, error) {
|
||||||
|
if _, err := state.lookupProfile(ctx, id); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
return id, nil
|
||||||
|
}
|
||||||
|
|
||||||
// GetBackend returns an independently owned effective backend definition.
|
// GetBackend returns an independently owned effective backend definition.
|
||||||
func (state *loaded) GetBackend(id string) (domain.Backend, error) {
|
func (state *loaded) GetBackend(id string) (domain.Backend, error) {
|
||||||
if state == nil {
|
if state == nil {
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ package configuration
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
"reflect"
|
"reflect"
|
||||||
@@ -11,6 +12,7 @@ import (
|
|||||||
"gitea.maximumdirect.net/eric/promptkit/internal/defaults"
|
"gitea.maximumdirect.net/eric/promptkit/internal/defaults"
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/llm"
|
"gitea.maximumdirect.net/eric/promptkit/internal/llm"
|
||||||
|
profilepkg "gitea.maximumdirect.net/eric/promptkit/internal/profile"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestLoadBuildsEffectiveTargetsAfterInheritance(t *testing.T) {
|
func TestLoadBuildsEffectiveTargetsAfterInheritance(t *testing.T) {
|
||||||
@@ -89,6 +91,18 @@ func TestLoadBuildsEffectiveTargetsAfterInheritance(t *testing.T) {
|
|||||||
if partial.ResolvedTarget.Endpoint != "" || partial.ResolvedTarget.Model != "" {
|
if partial.ResolvedTarget.Endpoint != "" || partial.ResolvedTarget.Model != "" {
|
||||||
t.Fatalf("partial profile gained a runnable target: %+v", partial.ResolvedTarget)
|
t.Fatalf("partial profile gained a runnable target: %+v", partial.ResolvedTarget)
|
||||||
}
|
}
|
||||||
|
identity, err := state.GetProfileIdentity(context.Background(), "partial")
|
||||||
|
if err != nil || identity != "partial" {
|
||||||
|
t.Fatalf("partial profile identity=(%q, %v)", identity, err)
|
||||||
|
}
|
||||||
|
if _, err := state.GetProfileIdentity(context.Background(), "missing"); !errors.Is(err, profilepkg.ErrProfileNotFound) {
|
||||||
|
t.Fatalf("missing profile identity error=%v", err)
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
cancel()
|
||||||
|
if _, err := state.GetProfileIdentity(ctx, "partial"); !errors.Is(err, context.Canceled) {
|
||||||
|
t.Fatalf("canceled profile identity lookup error=%v", err)
|
||||||
|
}
|
||||||
endpointOnly, _ := state.GetProfile(context.Background(), "endpoint-only")
|
endpointOnly, _ := state.GetProfile(context.Background(), "endpoint-only")
|
||||||
if endpointOnly.ResolvedTarget.Endpoint != "https://standalone.example/v1" ||
|
if endpointOnly.ResolvedTarget.Endpoint != "https://standalone.example/v1" ||
|
||||||
endpointOnly.ResolvedTarget.BackendID != "" || endpointOnly.ResolvedTarget.Model != "" {
|
endpointOnly.ResolvedTarget.BackendID != "" || endpointOnly.ResolvedTarget.Model != "" {
|
||||||
|
|||||||
@@ -135,6 +135,12 @@ type Artifact struct {
|
|||||||
Hash string
|
Hash string
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// PromptIdentity identifies one configured prompt version.
|
||||||
|
type PromptIdentity struct {
|
||||||
|
ID string
|
||||||
|
Version string
|
||||||
|
}
|
||||||
|
|
||||||
// PromptDefinition represents a configured prompt execution definition.
|
// PromptDefinition represents a configured prompt execution definition.
|
||||||
type PromptDefinition struct {
|
type PromptDefinition struct {
|
||||||
ID string `yaml:"id"`
|
ID string `yaml:"id"`
|
||||||
|
|||||||
58
internal/domain/requirements.go
Normal file
58
internal/domain/requirements.go
Normal file
@@ -0,0 +1,58 @@
|
|||||||
|
package domain
|
||||||
|
|
||||||
|
// RequirementKind classifies how a prompt may depend on an input or variable.
|
||||||
|
type RequirementKind string
|
||||||
|
|
||||||
|
const (
|
||||||
|
RequirementRequired RequirementKind = "required"
|
||||||
|
RequirementConditional RequirementKind = "conditional"
|
||||||
|
RequirementOptional RequirementKind = "optional"
|
||||||
|
)
|
||||||
|
|
||||||
|
// InputRequirement describes one declared or template-referenced input.
|
||||||
|
type InputRequirement struct {
|
||||||
|
Name string
|
||||||
|
Kind RequirementKind
|
||||||
|
Declared bool
|
||||||
|
ContentType *string
|
||||||
|
Description *string
|
||||||
|
Explanation string
|
||||||
|
}
|
||||||
|
|
||||||
|
// VariableRequirement describes one caller variable referenced by a prompt.
|
||||||
|
type VariableRequirement struct {
|
||||||
|
Name string
|
||||||
|
Kind RequirementKind
|
||||||
|
Supplied bool
|
||||||
|
Explanation string
|
||||||
|
}
|
||||||
|
|
||||||
|
// RequirementIssueCode identifies a limit that may affect requirement
|
||||||
|
// discovery.
|
||||||
|
type RequirementIssueCode string
|
||||||
|
|
||||||
|
const (
|
||||||
|
RequirementIssueMissingVariable RequirementIssueCode = "missing_variable"
|
||||||
|
RequirementIssueInputContent RequirementIssueCode = "input_content"
|
||||||
|
RequirementIssueDynamicInputName RequirementIssueCode = "dynamic_input_name"
|
||||||
|
RequirementIssueUnsupportedConstruct RequirementIssueCode = "unsupported_construct"
|
||||||
|
)
|
||||||
|
|
||||||
|
// RequirementIssue describes one limitation on requirement discovery.
|
||||||
|
type RequirementIssue struct {
|
||||||
|
Code RequirementIssueCode
|
||||||
|
InputName string
|
||||||
|
VariableName string
|
||||||
|
Description string
|
||||||
|
}
|
||||||
|
|
||||||
|
// RequirementInspection contains the inventories discovered for one prompt.
|
||||||
|
type RequirementInspection struct {
|
||||||
|
PromptID string
|
||||||
|
PromptVersion string
|
||||||
|
ProfileID string
|
||||||
|
Inputs []InputRequirement
|
||||||
|
Variables []VariableRequirement
|
||||||
|
Complete bool
|
||||||
|
Issues []RequirementIssue
|
||||||
|
}
|
||||||
@@ -83,6 +83,12 @@ func (o *Options) SetSchemaFile(path string) error {
|
|||||||
type Engine struct {
|
type Engine struct {
|
||||||
runner *usecase.Runner
|
runner *usecase.Runner
|
||||||
inspection configuration.Inspection
|
inspection configuration.Inspection
|
||||||
|
listing identityInventory
|
||||||
|
}
|
||||||
|
|
||||||
|
type identityInventory interface {
|
||||||
|
ListProfiles(context.Context) ([]string, error)
|
||||||
|
ListPrompts(context.Context) ([]domain.PromptIdentity, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
// New assembles an engine from facade-selected configuration and collaborators.
|
// New assembles an engine from facade-selected configuration and collaborators.
|
||||||
@@ -135,6 +141,7 @@ func New(cfg Config, options Options) (*Engine, error) {
|
|||||||
return &Engine{
|
return &Engine{
|
||||||
runner: usecase.NewRunnerWithRepairer(snapshot, snapshot, reader, prompt.NewCompiledRenderer(snapshot), client, validator, usecase.NewDefaultOutputRepairer(client), manager, defaultRetries),
|
runner: usecase.NewRunnerWithRepairer(snapshot, snapshot, reader, prompt.NewCompiledRenderer(snapshot), client, validator, usecase.NewDefaultOutputRepairer(client), manager, defaultRetries),
|
||||||
inspection: snapshot.Inspection(),
|
inspection: snapshot.Inspection(),
|
||||||
|
listing: snapshot,
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -178,9 +185,23 @@ func fileSource(name string) (fs.FS, string, string, error) {
|
|||||||
func (e *Engine) InspectPrompt(ctx context.Context, id, version string) (*domain.PromptInspection, error) {
|
func (e *Engine) InspectPrompt(ctx context.Context, id, version string) (*domain.PromptInspection, error) {
|
||||||
return e.runner.InspectPrompt(ctx, id, version)
|
return e.runner.InspectPrompt(ctx, id, version)
|
||||||
}
|
}
|
||||||
|
func (e *Engine) InspectRequirements(
|
||||||
|
ctx context.Context,
|
||||||
|
request usecase.RequirementInspectionRequest,
|
||||||
|
) (*domain.RequirementInspection, error) {
|
||||||
|
return e.runner.InspectRequirements(ctx, request)
|
||||||
|
}
|
||||||
func (e *Engine) InspectProfile(ctx context.Context, id string) (*domain.ProfileInspection, error) {
|
func (e *Engine) InspectProfile(ctx context.Context, id string) (*domain.ProfileInspection, error) {
|
||||||
return e.runner.InspectProfile(ctx, id)
|
return e.runner.InspectProfile(ctx, id)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (e *Engine) ListProfiles(ctx context.Context) ([]string, error) {
|
||||||
|
return e.listing.ListProfiles(ctx)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *Engine) ListPrompts(ctx context.Context) ([]domain.PromptIdentity, error) {
|
||||||
|
return e.listing.ListPrompts(ctx)
|
||||||
|
}
|
||||||
func (e *Engine) Prepare(ctx context.Context, request domain.RunRequest) (*domain.PreparedRun, error) {
|
func (e *Engine) Prepare(ctx context.Context, request domain.RunRequest) (*domain.PreparedRun, error) {
|
||||||
return e.runner.Prepare(ctx, request)
|
return e.runner.Prepare(ctx, request)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -152,6 +152,11 @@ func (c *OpenAICompatibleClient) PrepareGeneration(req domain.GenerateRequest) (
|
|||||||
|
|
||||||
payload, err := json.Marshal(wirePayload)
|
payload, err := json.Marshal(wirePayload)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
for key, value := range req.Target.ExtraParams {
|
||||||
|
if _, extraErr := json.Marshal(value); extraErr != nil {
|
||||||
|
return nil, fmt.Errorf("%w: extra_params.%s must be JSON-serializable: %v", ErrInvalidRequest, key, extraErr)
|
||||||
|
}
|
||||||
|
}
|
||||||
return nil, fmt.Errorf("%w: failed to encode request: %v", ErrRequestFailed, err)
|
return nil, fmt.Errorf("%w: failed to encode request: %v", ErrRequestFailed, err)
|
||||||
}
|
}
|
||||||
apiKey := strings.TrimSpace(req.Target.APIKey)
|
apiKey := strings.TrimSpace(req.Target.APIKey)
|
||||||
@@ -446,9 +451,6 @@ func openAIChatRequestPayload(req openAIChatRequest) (map[string]any, error) {
|
|||||||
if IsReservedOpenAIChatRequestField(key) {
|
if IsReservedOpenAIChatRequestField(key) {
|
||||||
return nil, fmt.Errorf("extra_params key %q collides with reserved request field", key)
|
return nil, fmt.Errorf("extra_params key %q collides with reserved request field", key)
|
||||||
}
|
}
|
||||||
if _, err := json.Marshal(value); err != nil {
|
|
||||||
return nil, fmt.Errorf("extra_params.%s must be JSON-serializable: %w", key, err)
|
|
||||||
}
|
|
||||||
out[key] = value
|
out[key] = value
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1113,6 +1113,13 @@ func TestOpenAICompatibleClientRejectsInvalidExtraParamsBeforeProviderCall(t *te
|
|||||||
}{
|
}{
|
||||||
{name: "empty key", extraParams: map[string]any{"": "empty"}, want: "key must not be empty"},
|
{name: "empty key", extraParams: map[string]any{"": "empty"}, want: "key must not be empty"},
|
||||||
{name: "unserializable value", extraParams: map[string]any{"bad": math.Inf(1)}, want: "JSON-serializable"},
|
{name: "unserializable value", extraParams: map[string]any{"bad": math.Inf(1)}, want: "JSON-serializable"},
|
||||||
|
{
|
||||||
|
name: "nested unserializable value",
|
||||||
|
extraParams: map[string]any{
|
||||||
|
"nested": map[string]any{"values": []any{"valid", math.NaN()}},
|
||||||
|
},
|
||||||
|
want: "extra_params.nested",
|
||||||
|
},
|
||||||
}
|
}
|
||||||
for _, key := range []string{
|
for _, key := range []string{
|
||||||
"model",
|
"model",
|
||||||
@@ -1161,6 +1168,56 @@ func TestOpenAICompatibleClientRejectsInvalidExtraParamsBeforeProviderCall(t *te
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestOpenAICompatibleClientClassifiesSchemaEncodingFailures(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
extraParams map[string]any
|
||||||
|
want error
|
||||||
|
unwanted error
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "serializable extra keeps full-request failure",
|
||||||
|
extraParams: map[string]any{"client_option": "accepted"},
|
||||||
|
want: ErrRequestFailed,
|
||||||
|
unwanted: ErrInvalidRequest,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "invalid extra keeps invalid-request failure",
|
||||||
|
extraParams: map[string]any{"nested": map[string]any{"bad": math.Inf(1)}},
|
||||||
|
want: ErrInvalidRequest,
|
||||||
|
unwanted: ErrRequestFailed,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
provider := newRecordingProvider(t)
|
||||||
|
client := newProviderClient(t, provider, OpenAICompatibleConfig{})
|
||||||
|
request := ordinaryGenerateRequest()
|
||||||
|
request.Target.ExtraParams = tc.extraParams
|
||||||
|
request.StructuredOutput = &domain.StructuredOutputSpec{
|
||||||
|
Type: domain.StructuredOutputJSONSchema,
|
||||||
|
JSONSchema: &domain.StructuredOutputJSONSpec{
|
||||||
|
Name: "invalid-schema",
|
||||||
|
Strict: true,
|
||||||
|
Schema: json.RawMessage("{"),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err := client.Generate(context.Background(), request)
|
||||||
|
if err == nil || !errors.Is(err, tc.want) {
|
||||||
|
t.Fatalf("error = %v, want identity %v", err, tc.want)
|
||||||
|
}
|
||||||
|
if errors.Is(err, tc.unwanted) {
|
||||||
|
t.Fatalf("error = %v, unexpectedly has identity %v", err, tc.unwanted)
|
||||||
|
}
|
||||||
|
if calls := provider.requestCount(); calls != 0 {
|
||||||
|
t.Fatalf("provider calls = %d, want 0", calls)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func checkConfiguredModelFallback(t *testing.T) {
|
func checkConfiguredModelFallback(t *testing.T) {
|
||||||
provider := newRecordingProvider(t)
|
provider := newRecordingProvider(t)
|
||||||
client := newProviderClient(t, provider, OpenAICompatibleConfig{Model: "default-model"})
|
client := newProviderClient(t, provider, OpenAICompatibleConfig{Model: "default-model"})
|
||||||
|
|||||||
119
internal/llm/prepare_generation_benchmark_test.go
Normal file
119
internal/llm/prepare_generation_benchmark_test.go
Normal file
@@ -0,0 +1,119 @@
|
|||||||
|
package llm
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"runtime"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||||
|
)
|
||||||
|
|
||||||
|
var prepareGenerationBenchmarkSink *PreparedGeneration
|
||||||
|
|
||||||
|
func BenchmarkPrepareGeneration(b *testing.B) {
|
||||||
|
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||||
|
BaseURL: "https://api.example.test/v1",
|
||||||
|
Model: "benchmark-model",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
b.Fatalf("construct benchmark client: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
prompt := domain.RenderedPrompt{
|
||||||
|
Messages: []domain.RenderedMessage{{Role: "user", Content: "Summarize this request."}},
|
||||||
|
}
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
request domain.GenerateRequest
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "no_extras",
|
||||||
|
request: domain.GenerateRequest{
|
||||||
|
Prompt: prompt,
|
||||||
|
Target: domain.ExecutionTarget{Model: "benchmark-model"},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "small_extras",
|
||||||
|
request: domain.GenerateRequest{
|
||||||
|
Prompt: prompt,
|
||||||
|
Target: domain.ExecutionTarget{Model: "benchmark-model", ExtraParams: map[string]any{
|
||||||
|
"priority": "low",
|
||||||
|
"attempt": 3,
|
||||||
|
"enabled": true,
|
||||||
|
}},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "1MiB_string_extra",
|
||||||
|
request: domain.GenerateRequest{
|
||||||
|
Prompt: prompt,
|
||||||
|
Target: domain.ExecutionTarget{Model: "benchmark-model", ExtraParams: map[string]any{
|
||||||
|
"payload": strings.Repeat("x", 1<<20),
|
||||||
|
}},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "nested_json_extras",
|
||||||
|
request: domain.GenerateRequest{
|
||||||
|
Prompt: prompt,
|
||||||
|
Target: domain.ExecutionTarget{Model: "benchmark-model", ExtraParams: map[string]any{
|
||||||
|
"options": map[string]any{
|
||||||
|
"enabled": true,
|
||||||
|
"limits": []any{2, 4, 8},
|
||||||
|
"tags": []string{"summary", "offline"},
|
||||||
|
},
|
||||||
|
}},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "structured_output",
|
||||||
|
request: domain.GenerateRequest{
|
||||||
|
Prompt: prompt,
|
||||||
|
Target: domain.ExecutionTarget{Model: "benchmark-model"},
|
||||||
|
StructuredOutput: &domain.StructuredOutputSpec{
|
||||||
|
Type: domain.StructuredOutputJSONSchema,
|
||||||
|
JSONSchema: &domain.StructuredOutputJSONSpec{
|
||||||
|
Name: "summary",
|
||||||
|
Strict: true,
|
||||||
|
Schema: map[string]any{
|
||||||
|
"type": "object",
|
||||||
|
"properties": map[string]any{
|
||||||
|
"summary": map[string]any{"type": "string"},
|
||||||
|
"count": map[string]any{"type": "integer"},
|
||||||
|
},
|
||||||
|
"required": []string{"summary", "count"},
|
||||||
|
"additionalProperties": false,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range tests {
|
||||||
|
b.Run(tc.name, func(b *testing.B) {
|
||||||
|
prepared, err := client.PrepareGeneration(tc.request)
|
||||||
|
if err != nil {
|
||||||
|
b.Fatalf("check prepared request: %v", err)
|
||||||
|
}
|
||||||
|
if !json.Valid(prepared.payload) {
|
||||||
|
b.Fatal("prepared request is not valid JSON")
|
||||||
|
}
|
||||||
|
|
||||||
|
b.ReportAllocs()
|
||||||
|
b.SetBytes(int64(len(prepared.payload)))
|
||||||
|
b.ResetTimer()
|
||||||
|
for range b.N {
|
||||||
|
prepared, err := client.PrepareGeneration(tc.request)
|
||||||
|
if err != nil {
|
||||||
|
b.Fatalf("prepare generation: %v", err)
|
||||||
|
}
|
||||||
|
prepareGenerationBenchmarkSink = prepared
|
||||||
|
}
|
||||||
|
b.StopTimer()
|
||||||
|
runtime.KeepAlive(prepareGenerationBenchmarkSink)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
254
internal/prompt/compiled_renderer_benchmark_test.go
Normal file
254
internal/prompt/compiled_renderer_benchmark_test.go
Normal file
@@ -0,0 +1,254 @@
|
|||||||
|
package prompt
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"runtime"
|
||||||
|
"strconv"
|
||||||
|
"testing"
|
||||||
|
"text/template"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||||
|
)
|
||||||
|
|
||||||
|
const rendererBenchmarkArtifactSize = 256 << 10
|
||||||
|
|
||||||
|
var (
|
||||||
|
renderedPromptBenchmarkSink *domain.RenderedPrompt
|
||||||
|
renderedPromptsBenchmarkSink []*domain.RenderedPrompt
|
||||||
|
)
|
||||||
|
|
||||||
|
type rendererBenchmarkIdentity struct {
|
||||||
|
id string
|
||||||
|
version string
|
||||||
|
}
|
||||||
|
|
||||||
|
type rendererBenchmarkLookup struct {
|
||||||
|
identity rendererBenchmarkIdentity
|
||||||
|
templates CompiledTemplates
|
||||||
|
}
|
||||||
|
|
||||||
|
func (lookup rendererBenchmarkLookup) GetCompiledTemplates(id, version string) (CompiledTemplates, bool) {
|
||||||
|
if id != lookup.identity.id || version != lookup.identity.version {
|
||||||
|
return CompiledTemplates{}, false
|
||||||
|
}
|
||||||
|
return lookup.templates, true
|
||||||
|
}
|
||||||
|
|
||||||
|
type compiledRenderingBenchmarkCase struct {
|
||||||
|
name string
|
||||||
|
renderer Renderer
|
||||||
|
definition *domain.PromptDefinition
|
||||||
|
inputs map[string]*domain.Artifact
|
||||||
|
vars map[string]string
|
||||||
|
ctx context.Context
|
||||||
|
outputBytes int
|
||||||
|
check func(*testing.B, *domain.RenderedPrompt)
|
||||||
|
}
|
||||||
|
|
||||||
|
func BenchmarkCompiledRendering(b *testing.B) {
|
||||||
|
body := bytes.Repeat([]byte("x"), rendererBenchmarkArtifactSize)
|
||||||
|
bodyText := string(body)
|
||||||
|
|
||||||
|
small := newCompiledRenderingBenchmarkCase(b,
|
||||||
|
"small_message", "", []string{"Summary for {{.audience}}."},
|
||||||
|
&domain.PromptDefinition{
|
||||||
|
ID: "compiled-render-benchmark", Version: "v1",
|
||||||
|
Templates: []domain.PromptMessageTemplate{{Role: "user", Content: "Summary for {{.audience}}."}},
|
||||||
|
}, nil, map[string]string{"audience": "project leads"},
|
||||||
|
len("Summary for project leads."),
|
||||||
|
func(b *testing.B, rendered *domain.RenderedPrompt) {
|
||||||
|
if rendered.SessionID != "" || len(rendered.Messages) != 1 || rendered.Messages[0].Content != "Summary for project leads." {
|
||||||
|
b.Fatalf("unexpected small compiled render: %+v", rendered)
|
||||||
|
}
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
artifactInputs := map[string]*domain.Artifact{
|
||||||
|
"document": {Name: "document", Body: body},
|
||||||
|
}
|
||||||
|
largeContent := `{{input "document"}}`
|
||||||
|
largeDefinition := &domain.PromptDefinition{
|
||||||
|
ID: "compiled-render-benchmark", Version: "v1",
|
||||||
|
Inputs: []domain.PromptInput{{Name: "document"}},
|
||||||
|
Templates: []domain.PromptMessageTemplate{{Role: "user", Content: largeContent}},
|
||||||
|
}
|
||||||
|
large := newCompiledRenderingBenchmarkCase(b,
|
||||||
|
"artifact_backed_256KiB", "", []string{largeContent},
|
||||||
|
largeDefinition, artifactInputs, nil, len(bodyText),
|
||||||
|
func(b *testing.B, rendered *domain.RenderedPrompt) {
|
||||||
|
if len(rendered.Messages) != 1 {
|
||||||
|
b.Fatalf("artifact render message count=%d, want 1", len(rendered.Messages))
|
||||||
|
}
|
||||||
|
if rendered.Messages[0].Content != bodyText {
|
||||||
|
b.Fatalf("artifact render length=%d, want %d", len(rendered.Messages[0].Content), len(bodyText))
|
||||||
|
}
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
repeatedSession := `document-size={{len (input "document")}}`
|
||||||
|
repeatedMessages := []string{
|
||||||
|
`{{input "document"}}`,
|
||||||
|
`{{input "document"}}|{{input "document"}}`,
|
||||||
|
}
|
||||||
|
repeatedDefinition := &domain.PromptDefinition{
|
||||||
|
ID: "compiled-render-benchmark", Version: "v1", SessionID: repeatedSession,
|
||||||
|
Inputs: []domain.PromptInput{{Name: "document"}},
|
||||||
|
Templates: []domain.PromptMessageTemplate{
|
||||||
|
{Role: "system", Content: repeatedMessages[0]},
|
||||||
|
{Role: "user", Content: repeatedMessages[1]},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
repeatedOutputBytes := len(bodyText)*3 + 1
|
||||||
|
repeated := newCompiledRenderingBenchmarkCase(b,
|
||||||
|
"repeated_across_session_and_messages", repeatedSession, repeatedMessages,
|
||||||
|
repeatedDefinition, artifactInputs, nil, repeatedOutputBytes,
|
||||||
|
func(b *testing.B, rendered *domain.RenderedPrompt) {
|
||||||
|
if rendered.SessionID != "document-size="+strconv.Itoa(len(bodyText)) || len(rendered.Messages) != 2 ||
|
||||||
|
rendered.Messages[0].Content != bodyText || rendered.Messages[1].Content != bodyText+"|"+bodyText {
|
||||||
|
b.Fatalf("unexpected repeated compiled render: session=%q message lengths=%v", rendered.SessionID, renderedMessageLengths(rendered))
|
||||||
|
}
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
for _, benchmark := range []compiledRenderingBenchmarkCase{small, large, repeated} {
|
||||||
|
b.Run(benchmark.name, func(b *testing.B) {
|
||||||
|
rendered, err := benchmark.renderer.Render(benchmark.ctx, benchmark.definition, benchmark.inputs, benchmark.vars)
|
||||||
|
if err != nil {
|
||||||
|
b.Fatalf("check compiled renderer output: %v", err)
|
||||||
|
}
|
||||||
|
benchmark.check(b, rendered)
|
||||||
|
|
||||||
|
b.ReportAllocs()
|
||||||
|
b.SetBytes(int64(benchmark.outputBytes))
|
||||||
|
b.ResetTimer()
|
||||||
|
for range b.N {
|
||||||
|
rendered, err := benchmark.renderer.Render(benchmark.ctx, benchmark.definition, benchmark.inputs, benchmark.vars)
|
||||||
|
if err != nil {
|
||||||
|
b.Fatalf("render compiled template: %v", err)
|
||||||
|
}
|
||||||
|
renderedPromptBenchmarkSink = rendered
|
||||||
|
}
|
||||||
|
b.StopTimer()
|
||||||
|
runtime.KeepAlive(renderedPromptBenchmarkSink)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// BenchmarkRetainedRenderedOutput reports post-GC live heap for a fixed batch
|
||||||
|
// of rendered outputs above the same fixture's setup-only baseline.
|
||||||
|
func BenchmarkRetainedRenderedOutput(b *testing.B) {
|
||||||
|
body := bytes.Repeat([]byte("x"), rendererBenchmarkArtifactSize)
|
||||||
|
bodyText := string(body)
|
||||||
|
content := `{{input "document"}}`
|
||||||
|
definition := &domain.PromptDefinition{
|
||||||
|
ID: "retained-render-benchmark", Version: "v1",
|
||||||
|
Inputs: []domain.PromptInput{{Name: "document"}},
|
||||||
|
Templates: []domain.PromptMessageTemplate{{Role: "user", Content: content}},
|
||||||
|
}
|
||||||
|
compiled := CompiledTemplates{Messages: []*template.Template{mustCompileRendererBenchmarkTemplate(b, content)}}
|
||||||
|
renderer := NewCompiledRenderer(rendererBenchmarkLookup{
|
||||||
|
identity: rendererBenchmarkIdentity{id: definition.ID, version: definition.Version},
|
||||||
|
templates: compiled,
|
||||||
|
})
|
||||||
|
inputs := map[string]*domain.Artifact{"document": {Name: "document", Body: body}}
|
||||||
|
ctx := context.Background()
|
||||||
|
const retainedBatchSize = 4
|
||||||
|
retained := make([]*domain.RenderedPrompt, retainedBatchSize)
|
||||||
|
|
||||||
|
check, err := renderer.Render(ctx, definition, inputs, nil)
|
||||||
|
if err != nil {
|
||||||
|
b.Fatalf("check retained-output fixture: %v", err)
|
||||||
|
}
|
||||||
|
if len(check.Messages) != 1 || check.Messages[0].Content != bodyText {
|
||||||
|
b.Fatalf("retained-output fixture returned unexpected content")
|
||||||
|
}
|
||||||
|
check = nil
|
||||||
|
// Repeated -count runs must start from the same setup-only heap baseline.
|
||||||
|
renderedPromptBenchmarkSink = nil
|
||||||
|
renderedPromptsBenchmarkSink = nil
|
||||||
|
|
||||||
|
runtime.GC()
|
||||||
|
var setupMemory runtime.MemStats
|
||||||
|
runtime.ReadMemStats(&setupMemory)
|
||||||
|
runtime.KeepAlive(renderer)
|
||||||
|
runtime.KeepAlive(definition)
|
||||||
|
runtime.KeepAlive(inputs)
|
||||||
|
|
||||||
|
b.ReportAllocs()
|
||||||
|
b.SetBytes(int64(len(bodyText)))
|
||||||
|
b.ResetTimer()
|
||||||
|
for iteration := 0; iteration < b.N; iteration++ {
|
||||||
|
rendered, err := renderer.Render(ctx, definition, inputs, nil)
|
||||||
|
if err != nil {
|
||||||
|
b.Fatalf("render retained-output fixture: %v", err)
|
||||||
|
}
|
||||||
|
retained[iteration%len(retained)] = rendered
|
||||||
|
}
|
||||||
|
b.StopTimer()
|
||||||
|
for i, rendered := range retained {
|
||||||
|
if rendered != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
rendered, err := renderer.Render(ctx, definition, inputs, nil)
|
||||||
|
if err != nil {
|
||||||
|
b.Fatalf("fill retained-output fixture: %v", err)
|
||||||
|
}
|
||||||
|
retained[i] = rendered
|
||||||
|
}
|
||||||
|
renderedPromptsBenchmarkSink = retained
|
||||||
|
runtime.GC()
|
||||||
|
var retainedMemory runtime.MemStats
|
||||||
|
runtime.ReadMemStats(&retainedMemory)
|
||||||
|
b.ReportMetric(float64(int64(retainedMemory.HeapAlloc)-int64(setupMemory.HeapAlloc)), "retained-B")
|
||||||
|
runtime.KeepAlive(retained)
|
||||||
|
runtime.KeepAlive(renderer)
|
||||||
|
runtime.KeepAlive(definition)
|
||||||
|
runtime.KeepAlive(inputs)
|
||||||
|
}
|
||||||
|
|
||||||
|
func newCompiledRenderingBenchmarkCase(
|
||||||
|
b *testing.B,
|
||||||
|
name string,
|
||||||
|
session string,
|
||||||
|
messages []string,
|
||||||
|
definition *domain.PromptDefinition,
|
||||||
|
inputs map[string]*domain.Artifact,
|
||||||
|
vars map[string]string,
|
||||||
|
outputBytes int,
|
||||||
|
check func(*testing.B, *domain.RenderedPrompt),
|
||||||
|
) compiledRenderingBenchmarkCase {
|
||||||
|
compiled := CompiledTemplates{}
|
||||||
|
if session != "" {
|
||||||
|
compiled.Session = mustCompileRendererBenchmarkTemplate(b, session)
|
||||||
|
}
|
||||||
|
for _, content := range messages {
|
||||||
|
compiled.Messages = append(compiled.Messages, mustCompileRendererBenchmarkTemplate(b, content))
|
||||||
|
}
|
||||||
|
return compiledRenderingBenchmarkCase{
|
||||||
|
name: name,
|
||||||
|
renderer: NewCompiledRenderer(rendererBenchmarkLookup{
|
||||||
|
identity: rendererBenchmarkIdentity{id: definition.ID, version: definition.Version},
|
||||||
|
templates: compiled,
|
||||||
|
}),
|
||||||
|
definition: definition, inputs: inputs, vars: vars, ctx: context.Background(),
|
||||||
|
outputBytes: outputBytes, check: check,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func mustCompileRendererBenchmarkTemplate(b *testing.B, source string) *template.Template {
|
||||||
|
b.Helper()
|
||||||
|
compiled, err := CompileTemplate("benchmark", source)
|
||||||
|
if err != nil {
|
||||||
|
b.Fatalf("compile benchmark template: %v", err)
|
||||||
|
}
|
||||||
|
return compiled
|
||||||
|
}
|
||||||
|
|
||||||
|
func renderedMessageLengths(rendered *domain.RenderedPrompt) []int {
|
||||||
|
lengths := make([]int, len(rendered.Messages))
|
||||||
|
for i, message := range rendered.Messages {
|
||||||
|
lengths[i] = len(message.Content)
|
||||||
|
}
|
||||||
|
return lengths
|
||||||
|
}
|
||||||
@@ -1,7 +1,6 @@
|
|||||||
package prompt
|
package prompt
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
@@ -126,7 +125,7 @@ func (r *goRenderer) Render(ctx context.Context, definition *domain.PromptDefini
|
|||||||
if err := ctx.Err(); err != nil {
|
if err := ctx.Err(); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
var buf bytes.Buffer
|
var buf strings.Builder
|
||||||
executeErr := tmpl.Execute(&buf, vars)
|
executeErr := tmpl.Execute(&buf, vars)
|
||||||
if err := ctx.Err(); err != nil {
|
if err := ctx.Err(); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
@@ -235,7 +234,7 @@ func executeSessionTemplate(ctx context.Context, tmpl *template.Template, vars m
|
|||||||
if err := ctx.Err(); err != nil {
|
if err := ctx.Err(); err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
var buf bytes.Buffer
|
var buf strings.Builder
|
||||||
executeErr := tmpl.Execute(&buf, vars)
|
executeErr := tmpl.Execute(&buf, vars)
|
||||||
if err := ctx.Err(); err != nil {
|
if err := ctx.Err(); err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
|
|||||||
@@ -4,8 +4,11 @@ import (
|
|||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
|
"fmt"
|
||||||
"strings"
|
"strings"
|
||||||
|
"sync"
|
||||||
"testing"
|
"testing"
|
||||||
|
"text/template"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||||
)
|
)
|
||||||
@@ -501,6 +504,77 @@ func TestGoRendererArtifactTextLifecycle(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestCompiledRendererKeepsConcurrentRequestsIndependent(t *testing.T) {
|
||||||
|
const (
|
||||||
|
promptID = "concurrent-render"
|
||||||
|
version = "v1"
|
||||||
|
)
|
||||||
|
sessionTemplate, err := CompileTemplate("session", "session-{{.request}}")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("compile session template: %v", err)
|
||||||
|
}
|
||||||
|
messageTemplate, err := CompileTemplate("message", `request={{.request}};document={{input "document"}}`)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("compile message template: %v", err)
|
||||||
|
}
|
||||||
|
renderer := NewCompiledRenderer(compiledRendererTestLookup{
|
||||||
|
id: promptID,
|
||||||
|
version: version,
|
||||||
|
templates: CompiledTemplates{
|
||||||
|
Session: sessionTemplate,
|
||||||
|
Messages: []*template.Template{messageTemplate},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
definition := &domain.PromptDefinition{
|
||||||
|
ID: promptID, Version: version, SessionID: "session-{{.request}}",
|
||||||
|
Inputs: []domain.PromptInput{{Name: "document"}},
|
||||||
|
Templates: []domain.PromptMessageTemplate{{Role: "user", Content: `request={{.request}};document={{input "document"}}`}},
|
||||||
|
}
|
||||||
|
|
||||||
|
const requestCount = 24
|
||||||
|
errs := make(chan error, requestCount)
|
||||||
|
var wait sync.WaitGroup
|
||||||
|
wait.Add(requestCount)
|
||||||
|
for i := 0; i < requestCount; i++ {
|
||||||
|
go func(i int) {
|
||||||
|
defer wait.Done()
|
||||||
|
request := fmt.Sprintf("request-%d", i)
|
||||||
|
document := fmt.Sprintf("document-%d", i)
|
||||||
|
rendered, err := renderer.Render(context.Background(), definition,
|
||||||
|
map[string]*domain.Artifact{"document": {Body: []byte(document)}},
|
||||||
|
map[string]string{"request": request},
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
errs <- fmt.Errorf("render %s: %w", request, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if rendered.SessionID != "session-"+request || len(rendered.Messages) != 1 ||
|
||||||
|
rendered.Messages[0].Content != "request="+request+";document="+document {
|
||||||
|
errs <- fmt.Errorf("render %s returned session %q and messages %#v", request, rendered.SessionID, rendered.Messages)
|
||||||
|
}
|
||||||
|
}(i)
|
||||||
|
}
|
||||||
|
wait.Wait()
|
||||||
|
close(errs)
|
||||||
|
for err := range errs {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type compiledRendererTestLookup struct {
|
||||||
|
id, version string
|
||||||
|
templates CompiledTemplates
|
||||||
|
}
|
||||||
|
|
||||||
|
func (lookup compiledRendererTestLookup) GetCompiledTemplates(id, version string) (CompiledTemplates, bool) {
|
||||||
|
if id != lookup.id || version != lookup.version {
|
||||||
|
return CompiledTemplates{}, false
|
||||||
|
}
|
||||||
|
return lookup.templates, true
|
||||||
|
}
|
||||||
|
|
||||||
|
// BenchmarkGoRendererArtifactReferences measures NewGoRenderer's per-call
|
||||||
|
// template-parsing fallback rather than the captured compiled path.
|
||||||
func BenchmarkGoRendererArtifactReferences(b *testing.B) {
|
func BenchmarkGoRendererArtifactReferences(b *testing.B) {
|
||||||
body := bytes.Repeat([]byte("document content "), (artifactTextChunkSize*4)/len("document content "))
|
body := bytes.Repeat([]byte("document content "), (artifactTextChunkSize*4)/len("document content "))
|
||||||
inputs := map[string]*domain.Artifact{
|
inputs := map[string]*domain.Artifact{
|
||||||
|
|||||||
978
internal/prompt/requirements.go
Normal file
978
internal/prompt/requirements.go
Normal file
@@ -0,0 +1,978 @@
|
|||||||
|
package prompt
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"sort"
|
||||||
|
"text/template"
|
||||||
|
"text/template/parse"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||||
|
)
|
||||||
|
|
||||||
|
type requirementReachability uint8
|
||||||
|
|
||||||
|
const (
|
||||||
|
requirementDefinite requirementReachability = iota
|
||||||
|
requirementPossible
|
||||||
|
requirementInactive
|
||||||
|
)
|
||||||
|
|
||||||
|
type dotIdentity uint8
|
||||||
|
|
||||||
|
const (
|
||||||
|
dotRoot dotIdentity = iota
|
||||||
|
dotNonRoot
|
||||||
|
dotUnknown
|
||||||
|
)
|
||||||
|
|
||||||
|
type inputEvidence struct {
|
||||||
|
name string
|
||||||
|
kind domain.RequirementKind
|
||||||
|
declared bool
|
||||||
|
applicable bool
|
||||||
|
contentType *string
|
||||||
|
description *string
|
||||||
|
explanation string
|
||||||
|
}
|
||||||
|
|
||||||
|
type variableEvidence struct {
|
||||||
|
name string
|
||||||
|
kind domain.RequirementKind
|
||||||
|
applicable bool
|
||||||
|
explanation string
|
||||||
|
}
|
||||||
|
|
||||||
|
type requirementIssueKey struct {
|
||||||
|
code domain.RequirementIssueCode
|
||||||
|
inputName string
|
||||||
|
variableName string
|
||||||
|
}
|
||||||
|
|
||||||
|
type conditionEvaluation struct {
|
||||||
|
value any
|
||||||
|
known bool
|
||||||
|
unsupported bool
|
||||||
|
inputNameUnknown bool
|
||||||
|
missingVariables map[string]struct{}
|
||||||
|
contentInputs map[string]struct{}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (evaluation conditionEvaluation) truth() bool {
|
||||||
|
truth, ok := template.IsTrue(evaluation.value)
|
||||||
|
return evaluation.known && ok && truth
|
||||||
|
}
|
||||||
|
|
||||||
|
func (walker *requirementWalker) evaluateCondition(pipe *parse.PipeNode, reach requirementReachability, dot dotIdentity) conditionEvaluation {
|
||||||
|
if pipe == nil {
|
||||||
|
return conditionEvaluation{unsupported: true}
|
||||||
|
}
|
||||||
|
if len(pipe.Cmds) == 1 && len(pipe.Decl) == 0 && !pipe.IsAssign {
|
||||||
|
return walker.evaluateCommand(pipe.Cmds[0], reach, dot)
|
||||||
|
}
|
||||||
|
if name, ok := directInputPipeline(pipe); ok {
|
||||||
|
return walker.evaluateInputName(name, reach, dot)
|
||||||
|
}
|
||||||
|
before := walker.dependencies
|
||||||
|
visitReach := requirementPossible
|
||||||
|
if reach == requirementInactive {
|
||||||
|
visitReach = requirementInactive
|
||||||
|
}
|
||||||
|
if err := walker.visitPipe(pipe, visitReach, dot); err != nil {
|
||||||
|
return conditionEvaluation{unsupported: true}
|
||||||
|
}
|
||||||
|
return conditionEvaluation{unsupported: walker.dependencies != before}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (walker *requirementWalker) evaluateCommand(command *parse.CommandNode, reach requirementReachability, dot dotIdentity) conditionEvaluation {
|
||||||
|
if command == nil || len(command.Args) == 0 {
|
||||||
|
return conditionEvaluation{unsupported: true}
|
||||||
|
}
|
||||||
|
identifier, isIdentifier := command.Args[0].(*parse.IdentifierNode)
|
||||||
|
if isIdentifier {
|
||||||
|
switch identifier.Ident {
|
||||||
|
case "input":
|
||||||
|
if reach == requirementInactive {
|
||||||
|
if len(command.Args) == 2 {
|
||||||
|
if name, ok := literalStringValue(command.Args[1]); ok {
|
||||||
|
walker.recordInput(name, requirementInactive)
|
||||||
|
return conditionEvaluation{}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, argument := range command.Args[1:] {
|
||||||
|
_ = walker.visit(argument, requirementInactive, dot)
|
||||||
|
}
|
||||||
|
return conditionEvaluation{}
|
||||||
|
}
|
||||||
|
if len(command.Args) == 2 {
|
||||||
|
if name, ok := command.Args[1].(*parse.StringNode); ok {
|
||||||
|
walker.recordInput(name.Text, reach)
|
||||||
|
return conditionEvaluation{contentInputs: map[string]struct{}{name.Text: {}}}
|
||||||
|
}
|
||||||
|
return walker.evaluateInputName(command.Args[1], reach, dot)
|
||||||
|
}
|
||||||
|
for _, argument := range command.Args[1:] {
|
||||||
|
_ = walker.visit(argument, requirementPossible, dot)
|
||||||
|
}
|
||||||
|
return walker.unresolvedInputName(conditionEvaluation{unsupported: true})
|
||||||
|
case "index":
|
||||||
|
if len(command.Args) == 3 {
|
||||||
|
if key, ok := command.Args[2].(*parse.StringNode); ok && isRootMap(command.Args[1], dot, walker.rootKnown) {
|
||||||
|
walker.recordOptionalVariable(key.Text, reach)
|
||||||
|
value, supplied := walker.vars[key.Text]
|
||||||
|
if !supplied {
|
||||||
|
value = ""
|
||||||
|
}
|
||||||
|
return conditionEvaluation{value: value, known: true}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
case "not":
|
||||||
|
if len(command.Args) == 2 {
|
||||||
|
evaluation := walker.evaluateValue(command.Args[1], reach, dot)
|
||||||
|
if evaluation.known {
|
||||||
|
truth, ok := template.IsTrue(evaluation.value)
|
||||||
|
if ok {
|
||||||
|
evaluation.value = !truth
|
||||||
|
} else {
|
||||||
|
evaluation.known = false
|
||||||
|
evaluation.unsupported = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return evaluation
|
||||||
|
}
|
||||||
|
case "eq", "ne":
|
||||||
|
if len(command.Args) == 3 {
|
||||||
|
left := walker.evaluateValue(command.Args[1], reach, dot)
|
||||||
|
right := walker.evaluateValue(command.Args[2], reach, dot)
|
||||||
|
result := mergeConditionEvaluations(left, right)
|
||||||
|
if left.known && right.known {
|
||||||
|
leftString, leftOK := left.value.(string)
|
||||||
|
rightString, rightOK := right.value.(string)
|
||||||
|
if leftOK && rightOK {
|
||||||
|
equal := leftString == rightString
|
||||||
|
if identifier.Ident == "ne" {
|
||||||
|
equal = !equal
|
||||||
|
}
|
||||||
|
result.value = equal
|
||||||
|
result.known = true
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
result.unsupported = true
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !isIdentifier && len(command.Args) == 1 {
|
||||||
|
return walker.evaluateValue(command.Args[0], reach, dot)
|
||||||
|
}
|
||||||
|
if reach == requirementInactive {
|
||||||
|
_ = walker.visitCommand(command, reach, dot)
|
||||||
|
return conditionEvaluation{}
|
||||||
|
}
|
||||||
|
before := walker.dependencies
|
||||||
|
_ = walker.visitCommand(command, requirementPossible, dot)
|
||||||
|
return conditionEvaluation{unsupported: walker.dependencies != before}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (walker *requirementWalker) evaluateValue(node parse.Node, reach requirementReachability, dot dotIdentity) conditionEvaluation {
|
||||||
|
switch node := node.(type) {
|
||||||
|
case *parse.DotNode:
|
||||||
|
if dot == dotRoot {
|
||||||
|
return conditionEvaluation{value: walker.vars, known: true}
|
||||||
|
}
|
||||||
|
if dot == dotUnknown {
|
||||||
|
walker.unresolvedVariableScope(reach)
|
||||||
|
}
|
||||||
|
return conditionEvaluation{unsupported: true}
|
||||||
|
case *parse.StringNode:
|
||||||
|
return conditionEvaluation{value: node.Text, known: true}
|
||||||
|
case *parse.BoolNode:
|
||||||
|
return conditionEvaluation{value: node.True, known: true}
|
||||||
|
case *parse.NilNode:
|
||||||
|
return conditionEvaluation{value: nil, known: true}
|
||||||
|
case *parse.FieldNode:
|
||||||
|
if dot != dotRoot || len(node.Ident) == 0 {
|
||||||
|
if dot == dotUnknown {
|
||||||
|
walker.unresolvedVariableScope(reach)
|
||||||
|
}
|
||||||
|
return conditionEvaluation{unsupported: true}
|
||||||
|
}
|
||||||
|
name := node.Ident[0]
|
||||||
|
walker.recordVariable(name, reach)
|
||||||
|
if len(node.Ident) > 1 {
|
||||||
|
walker.addIssue("", name, "A chained field access is outside the direct-key scope.")
|
||||||
|
return conditionEvaluation{unsupported: true}
|
||||||
|
}
|
||||||
|
value, supplied := walker.vars[name]
|
||||||
|
if !supplied {
|
||||||
|
return conditionEvaluation{missingVariables: map[string]struct{}{name: {}}}
|
||||||
|
}
|
||||||
|
return conditionEvaluation{value: value, known: true}
|
||||||
|
case *parse.VariableNode:
|
||||||
|
if len(node.Ident) == 1 && node.Ident[0] == "$" && walker.rootKnown {
|
||||||
|
return conditionEvaluation{value: walker.vars, known: true}
|
||||||
|
}
|
||||||
|
if len(node.Ident) < 2 || node.Ident[0] != "$" || !walker.rootKnown {
|
||||||
|
walker.visitVariable(node, reach)
|
||||||
|
return conditionEvaluation{unsupported: true}
|
||||||
|
}
|
||||||
|
name := node.Ident[1]
|
||||||
|
walker.recordVariable(name, reach)
|
||||||
|
if len(node.Ident) > 2 {
|
||||||
|
walker.addIssue("", name, "A chained field access is outside the direct-key scope.")
|
||||||
|
return conditionEvaluation{unsupported: true}
|
||||||
|
}
|
||||||
|
value, supplied := walker.vars[name]
|
||||||
|
if !supplied {
|
||||||
|
return conditionEvaluation{missingVariables: map[string]struct{}{name: {}}}
|
||||||
|
}
|
||||||
|
return conditionEvaluation{value: value, known: true}
|
||||||
|
case *parse.CommandNode:
|
||||||
|
return walker.evaluateCommand(node, reach, dot)
|
||||||
|
case *parse.PipeNode:
|
||||||
|
return walker.evaluateCondition(node, reach, dot)
|
||||||
|
case *parse.ChainNode:
|
||||||
|
before := walker.dependencies
|
||||||
|
visitReach := requirementPossible
|
||||||
|
if reach == requirementInactive {
|
||||||
|
visitReach = requirementInactive
|
||||||
|
}
|
||||||
|
_ = walker.visit(node.Node, visitReach, dot)
|
||||||
|
return conditionEvaluation{unsupported: walker.dependencies != before}
|
||||||
|
default:
|
||||||
|
return conditionEvaluation{unsupported: true}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (walker *requirementWalker) evaluateInputName(node parse.Node, reach requirementReachability, dot dotIdentity) conditionEvaluation {
|
||||||
|
evaluation := walker.evaluateValue(node, reach, dot)
|
||||||
|
if evaluation.known {
|
||||||
|
if name, ok := evaluation.value.(string); ok {
|
||||||
|
walker.recordInput(name, reach)
|
||||||
|
return conditionEvaluation{contentInputs: map[string]struct{}{name: {}}}
|
||||||
|
}
|
||||||
|
evaluation.unsupported = true
|
||||||
|
}
|
||||||
|
// Value analysis has already visited recognizable references, including
|
||||||
|
// the conservative fallback for unsupported expressions. Do not rescan
|
||||||
|
// the same subtree: nested computed names would multiply that work.
|
||||||
|
return walker.unresolvedInputName(evaluation)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (walker *requirementWalker) unresolvedInputName(evaluation conditionEvaluation) conditionEvaluation {
|
||||||
|
for name := range evaluation.missingVariables {
|
||||||
|
walker.addIssueCode(domain.RequirementIssueMissingVariable, "", name, "A missing variable prevents resolving an input name.")
|
||||||
|
}
|
||||||
|
for name := range evaluation.contentInputs {
|
||||||
|
walker.addIssueCode(domain.RequirementIssueInputContent, name, "", "Captured input content affects an input name.")
|
||||||
|
}
|
||||||
|
walker.addIssueCode(domain.RequirementIssueDynamicInputName, "", "", "An input helper name is not statically available.")
|
||||||
|
walker.unknownInputEffect = true
|
||||||
|
evaluation.inputNameUnknown = true
|
||||||
|
return evaluation
|
||||||
|
}
|
||||||
|
|
||||||
|
func directInputPipeline(pipe *parse.PipeNode) (parse.Node, bool) {
|
||||||
|
if pipe == nil || len(pipe.Cmds) != 2 || len(pipe.Decl) != 0 || pipe.IsAssign {
|
||||||
|
return nil, false
|
||||||
|
}
|
||||||
|
input := pipe.Cmds[1]
|
||||||
|
if len(input.Args) != 1 {
|
||||||
|
return nil, false
|
||||||
|
}
|
||||||
|
identifier, ok := input.Args[0].(*parse.IdentifierNode)
|
||||||
|
if !ok || identifier.Ident != "input" {
|
||||||
|
return nil, false
|
||||||
|
}
|
||||||
|
value := pipe.Cmds[0]
|
||||||
|
if len(value.Args) > 0 {
|
||||||
|
valueIdentifier, isValueIdentifier := value.Args[0].(*parse.IdentifierNode)
|
||||||
|
if isValueIdentifier && valueIdentifier.Ident == "input" && len(value.Args) == 2 {
|
||||||
|
return value, true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(value.Args) == 1 {
|
||||||
|
return value.Args[0], true
|
||||||
|
}
|
||||||
|
if len(value.Args) == 3 {
|
||||||
|
function, ok := value.Args[0].(*parse.IdentifierNode)
|
||||||
|
if ok && function.Ident == "index" {
|
||||||
|
return value, true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil, false
|
||||||
|
}
|
||||||
|
|
||||||
|
func literalStringValue(node parse.Node) (string, bool) {
|
||||||
|
switch node := node.(type) {
|
||||||
|
case *parse.StringNode:
|
||||||
|
return node.Text, true
|
||||||
|
case *parse.PipeNode:
|
||||||
|
if len(node.Cmds) == 1 && len(node.Cmds[0].Args) == 1 && len(node.Decl) == 0 && !node.IsAssign {
|
||||||
|
return literalStringValue(node.Cmds[0].Args[0])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
|
||||||
|
func mergeConditionEvaluations(left, right conditionEvaluation) conditionEvaluation {
|
||||||
|
result := conditionEvaluation{
|
||||||
|
unsupported: left.unsupported || right.unsupported,
|
||||||
|
inputNameUnknown: left.inputNameUnknown || right.inputNameUnknown,
|
||||||
|
missingVariables: make(map[string]struct{}, len(left.missingVariables)+len(right.missingVariables)),
|
||||||
|
contentInputs: make(map[string]struct{}, len(left.contentInputs)+len(right.contentInputs)),
|
||||||
|
}
|
||||||
|
for name := range left.missingVariables {
|
||||||
|
result.missingVariables[name] = struct{}{}
|
||||||
|
}
|
||||||
|
for name := range right.missingVariables {
|
||||||
|
result.missingVariables[name] = struct{}{}
|
||||||
|
}
|
||||||
|
for name := range left.contentInputs {
|
||||||
|
result.contentInputs[name] = struct{}{}
|
||||||
|
}
|
||||||
|
for name := range right.contentInputs {
|
||||||
|
result.contentInputs[name] = struct{}{}
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
|
func branchReach(parent requirementReachability, known, truth, selected bool) requirementReachability {
|
||||||
|
if parent == requirementInactive {
|
||||||
|
return requirementInactive
|
||||||
|
}
|
||||||
|
if !known {
|
||||||
|
return requirementPossible
|
||||||
|
}
|
||||||
|
if truth == selected {
|
||||||
|
if parent == requirementPossible {
|
||||||
|
return requirementPossible
|
||||||
|
}
|
||||||
|
return requirementDefinite
|
||||||
|
}
|
||||||
|
return requirementInactive
|
||||||
|
}
|
||||||
|
|
||||||
|
type requirementWalker struct {
|
||||||
|
ctx context.Context
|
||||||
|
vars map[string]string
|
||||||
|
inputs map[string]*inputEvidence
|
||||||
|
variables map[string]*variableEvidence
|
||||||
|
issues map[requirementIssueKey]domain.RequirementIssue
|
||||||
|
complete bool
|
||||||
|
rootKnown bool
|
||||||
|
unknownInputEffect bool
|
||||||
|
unknownVariableEffect bool
|
||||||
|
dependencies int
|
||||||
|
templateSet *template.Template
|
||||||
|
visitedTrees map[*parse.Tree]bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// AnalyzeRequirements collects the prompt dependencies recognizable from its
|
||||||
|
// captured syntax. Branch bodies remain conditional until their conditions are
|
||||||
|
// classified; unsupported dependency-bearing syntax makes the result partial.
|
||||||
|
func AnalyzeRequirements(
|
||||||
|
ctx context.Context,
|
||||||
|
definition *domain.PromptDefinition,
|
||||||
|
snapshot RequirementSnapshot,
|
||||||
|
vars map[string]string,
|
||||||
|
sessionBypassed bool,
|
||||||
|
) (domain.RequirementInspection, error) {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return domain.RequirementInspection{}, err
|
||||||
|
}
|
||||||
|
result := domain.RequirementInspection{Complete: true}
|
||||||
|
if definition != nil {
|
||||||
|
result.PromptID = definition.ID
|
||||||
|
result.PromptVersion = definition.Version
|
||||||
|
}
|
||||||
|
walker := &requirementWalker{
|
||||||
|
ctx: ctx,
|
||||||
|
vars: vars,
|
||||||
|
inputs: make(map[string]*inputEvidence, len(snapshot.Inputs)),
|
||||||
|
variables: make(map[string]*variableEvidence),
|
||||||
|
issues: make(map[requirementIssueKey]domain.RequirementIssue),
|
||||||
|
complete: true,
|
||||||
|
rootKnown: true,
|
||||||
|
}
|
||||||
|
for _, input := range snapshot.Inputs {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return domain.RequirementInspection{}, err
|
||||||
|
}
|
||||||
|
kind := domain.RequirementOptional
|
||||||
|
if input.Required {
|
||||||
|
kind = domain.RequirementRequired
|
||||||
|
}
|
||||||
|
walker.inputs[input.Name] = &inputEvidence{
|
||||||
|
name: input.Name,
|
||||||
|
kind: kind,
|
||||||
|
declared: true,
|
||||||
|
contentType: copyString(input.ContentType),
|
||||||
|
description: copyString(input.Description),
|
||||||
|
}
|
||||||
|
if kind == domain.RequirementOptional {
|
||||||
|
walker.inputs[input.Name].explanation = "Declared as an optional input."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, message := range snapshot.Templates.Messages {
|
||||||
|
if err := walker.walkTemplate(message); err != nil {
|
||||||
|
return domain.RequirementInspection{}, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !sessionBypassed && snapshot.Templates.Session != nil {
|
||||||
|
if err := walker.walkTemplate(snapshot.Templates.Session); err != nil {
|
||||||
|
return domain.RequirementInspection{}, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return domain.RequirementInspection{}, err
|
||||||
|
}
|
||||||
|
return walker.result(result), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (walker *requirementWalker) walkTemplate(compiled *template.Template) error {
|
||||||
|
if err := walker.ctx.Err(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if compiled == nil || compiled.Tree == nil || compiled.Tree.Root == nil {
|
||||||
|
walker.addIssue("", "", "A captured template has no parsed root.")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
previousSet := walker.templateSet
|
||||||
|
previousVisited := walker.visitedTrees
|
||||||
|
walker.templateSet = compiled
|
||||||
|
walker.visitedTrees = map[*parse.Tree]bool{compiled.Tree: true}
|
||||||
|
walker.rootKnown = true
|
||||||
|
err := walker.visit(compiled.Tree.Root, requirementDefinite, dotRoot)
|
||||||
|
walker.templateSet = previousSet
|
||||||
|
walker.visitedTrees = previousVisited
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (walker *requirementWalker) visit(node parse.Node, reach requirementReachability, dot dotIdentity) error {
|
||||||
|
if err := walker.ctx.Err(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if node == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
switch node := node.(type) {
|
||||||
|
case *parse.ListNode:
|
||||||
|
if node == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
for _, child := range node.Nodes {
|
||||||
|
if err := walker.visit(child, reach, dot); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
case *parse.ActionNode:
|
||||||
|
return walker.visit(node.Pipe, reach, dot)
|
||||||
|
case *parse.IfNode:
|
||||||
|
if reach == requirementInactive {
|
||||||
|
if err := walker.visit(node.Pipe, requirementInactive, dot); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := walker.visit(node.List, requirementInactive, dot); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return walker.visit(node.ElseList, requirementInactive, dot)
|
||||||
|
}
|
||||||
|
condition := walker.evaluateCondition(node.Pipe, reach, dot)
|
||||||
|
before := walker.dependencies
|
||||||
|
thenReach := branchReach(reach, condition.known, condition.truth(), true)
|
||||||
|
elseReach := branchReach(reach, condition.known, condition.truth(), false)
|
||||||
|
if err := walker.visit(node.List, thenReach, dot); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := walker.visit(node.ElseList, elseReach, dot); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if !condition.known && walker.dependencies != before {
|
||||||
|
walker.recordConditionLimitation(condition)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
case *parse.WithNode:
|
||||||
|
if reach == requirementInactive {
|
||||||
|
if err := walker.visit(node.Pipe, requirementInactive, dot); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
bodyDot := walker.withDotIdentity(node.Pipe, dot, conditionEvaluation{})
|
||||||
|
if err := walker.visit(node.List, requirementInactive, bodyDot); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return walker.visit(node.ElseList, requirementInactive, dot)
|
||||||
|
}
|
||||||
|
condition := walker.evaluateCondition(node.Pipe, reach, dot)
|
||||||
|
before := walker.dependencies
|
||||||
|
thenReach := branchReach(reach, condition.known, condition.truth(), true)
|
||||||
|
elseReach := branchReach(reach, condition.known, condition.truth(), false)
|
||||||
|
bodyDot := walker.withDotIdentity(node.Pipe, dot, condition)
|
||||||
|
if err := walker.visit(node.List, thenReach, bodyDot); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := walker.visit(node.ElseList, elseReach, dot); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if !condition.known && walker.dependencies != before {
|
||||||
|
walker.recordConditionLimitation(condition)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
case *parse.RangeNode:
|
||||||
|
if err := walker.visit(node.Pipe, reach, dot); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
before := walker.dependencies
|
||||||
|
bodyReach := requirementPossible
|
||||||
|
if reach == requirementInactive {
|
||||||
|
bodyReach = requirementInactive
|
||||||
|
}
|
||||||
|
if err := walker.visit(node.List, bodyReach, dotNonRoot); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := walker.visit(node.ElseList, bodyReach, dot); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if reach != requirementInactive && walker.dependencies != before {
|
||||||
|
walker.addIssue("", "", "Dependencies inside a range are only conservatively enumerated.")
|
||||||
|
}
|
||||||
|
case *parse.TemplateNode:
|
||||||
|
pipeReach := requirementPossible
|
||||||
|
if reach == requirementInactive {
|
||||||
|
pipeReach = requirementInactive
|
||||||
|
}
|
||||||
|
if err := walker.visit(node.Pipe, pipeReach, dot); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if reach == requirementInactive {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
called := walker.templateSet.Lookup(node.Name)
|
||||||
|
if called == nil || called.Tree == nil || called.Tree.Root == nil {
|
||||||
|
walker.addIssue("", "", "An invoked named template is unavailable for syntax inspection.")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if walker.visitedTrees[called.Tree] {
|
||||||
|
walker.addIssue("", "", "An invoked named template is scanned conservatively.")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
walker.visitedTrees[called.Tree] = true
|
||||||
|
rootKnown := walker.rootKnown
|
||||||
|
walker.rootKnown = false
|
||||||
|
walker.unknownVariableEffect = true
|
||||||
|
err := walker.visit(called.Tree.Root, requirementPossible, dotNonRoot)
|
||||||
|
walker.rootKnown = rootKnown
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
walker.addIssue("", "", "An invoked named template is scanned conservatively.")
|
||||||
|
case *parse.PipeNode:
|
||||||
|
return walker.visitPipe(node, reach, dot)
|
||||||
|
case *parse.CommandNode:
|
||||||
|
return walker.visitCommand(node, reach, dot)
|
||||||
|
case *parse.FieldNode:
|
||||||
|
if dot == dotRoot && len(node.Ident) > 0 {
|
||||||
|
walker.recordVariable(node.Ident[0], reach)
|
||||||
|
if len(node.Ident) > 1 && reach != requirementInactive {
|
||||||
|
walker.addIssue("", node.Ident[0], "A chained field access is outside the direct-key scope.")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if dot == dotUnknown {
|
||||||
|
walker.unresolvedVariableScope(reach)
|
||||||
|
}
|
||||||
|
case *parse.DotNode:
|
||||||
|
if dot == dotUnknown {
|
||||||
|
walker.unresolvedVariableScope(reach)
|
||||||
|
}
|
||||||
|
case *parse.VariableNode:
|
||||||
|
walker.visitVariable(node, reach)
|
||||||
|
case *parse.ChainNode:
|
||||||
|
before := walker.dependencies
|
||||||
|
chainReach := requirementPossible
|
||||||
|
if reach == requirementInactive {
|
||||||
|
chainReach = requirementInactive
|
||||||
|
}
|
||||||
|
if err := walker.visit(node.Node, chainReach, dot); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if reach != requirementInactive && walker.dependencies != before {
|
||||||
|
walker.addIssue("", "", "A chained expression is not interpreted.")
|
||||||
|
}
|
||||||
|
case *parse.BreakNode, *parse.ContinueNode:
|
||||||
|
if reach != requirementInactive {
|
||||||
|
walker.addIssue("", "", "Loop control is outside the supported syntax scope.")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (walker *requirementWalker) visitPipe(pipe *parse.PipeNode, reach requirementReachability, dot dotIdentity) error {
|
||||||
|
if pipe == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if len(pipe.Cmds) == 1 && len(pipe.Decl) == 0 && !pipe.IsAssign {
|
||||||
|
return walker.visitCommand(pipe.Cmds[0], reach, dot)
|
||||||
|
}
|
||||||
|
if name, ok := directInputPipeline(pipe); ok {
|
||||||
|
if reach == requirementInactive {
|
||||||
|
if literal, known := literalStringValue(name); known {
|
||||||
|
walker.recordInput(literal, requirementInactive)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return walker.visit(name, requirementInactive, dot)
|
||||||
|
}
|
||||||
|
walker.evaluateInputName(name, reach, dot)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
before := walker.dependencies
|
||||||
|
commandReach := requirementPossible
|
||||||
|
if reach == requirementInactive {
|
||||||
|
commandReach = requirementInactive
|
||||||
|
}
|
||||||
|
for _, command := range pipe.Cmds {
|
||||||
|
if err := walker.visitCommand(command, commandReach, dot); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if reach != requirementInactive && walker.dependencies != before {
|
||||||
|
walker.addIssue("", "", "A pipeline or variable assignment is not interpreted.")
|
||||||
|
}
|
||||||
|
if reach != requirementInactive {
|
||||||
|
for _, declaration := range pipe.Decl {
|
||||||
|
if len(declaration.Ident) > 0 && declaration.Ident[0] == "$" {
|
||||||
|
walker.rootKnown = false
|
||||||
|
walker.unknownVariableEffect = true
|
||||||
|
walker.addIssue("", "", "The root template binding is reassigned.")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (walker *requirementWalker) visitCommand(command *parse.CommandNode, reach requirementReachability, dot dotIdentity) error {
|
||||||
|
if command == nil || len(command.Args) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
identifier, isIdentifier := command.Args[0].(*parse.IdentifierNode)
|
||||||
|
if isIdentifier && identifier.Ident == "input" {
|
||||||
|
if reach == requirementInactive {
|
||||||
|
if len(command.Args) == 2 {
|
||||||
|
if name, ok := literalStringValue(command.Args[1]); ok {
|
||||||
|
walker.recordInput(name, reach)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, argument := range command.Args[1:] {
|
||||||
|
if err := walker.visit(argument, requirementInactive, dot); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if len(command.Args) == 2 {
|
||||||
|
if name, ok := command.Args[1].(*parse.StringNode); ok {
|
||||||
|
walker.recordInput(name.Text, reach)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
walker.evaluateInputName(command.Args[1], reach, dot)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
for _, argument := range command.Args[1:] {
|
||||||
|
if err := walker.visit(argument, requirementPossible, dot); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
walker.unresolvedInputName(conditionEvaluation{unsupported: true})
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if isIdentifier && identifier.Ident == "index" && len(command.Args) == 3 {
|
||||||
|
if _, ok := command.Args[2].(*parse.StringNode); ok && isRootMap(command.Args[1], dot, walker.rootKnown) {
|
||||||
|
key := command.Args[2].(*parse.StringNode).Text
|
||||||
|
walker.recordOptionalVariable(key, reach)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
// A local map may alias the caller root even though its value is not
|
||||||
|
// tracked. A lookup on it must not vanish from complete discovery.
|
||||||
|
if variable, ok := unwrapSingleValue(command.Args[1]).(*parse.VariableNode); ok && len(variable.Ident) == 1 {
|
||||||
|
walker.unresolvedVariableScope(reach)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !isIdentifier && len(command.Args) == 1 {
|
||||||
|
return walker.visit(command.Args[0], reach, dot)
|
||||||
|
}
|
||||||
|
before := walker.dependencies
|
||||||
|
for index, argument := range command.Args {
|
||||||
|
if isIdentifier && index == 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
argumentReach := requirementPossible
|
||||||
|
if reach == requirementInactive {
|
||||||
|
argumentReach = requirementInactive
|
||||||
|
}
|
||||||
|
if err := walker.visit(argument, argumentReach, dot); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if reach != requirementInactive && walker.dependencies != before {
|
||||||
|
walker.addIssue("", "", "A function call or computed expression is not interpreted.")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (walker *requirementWalker) visitVariable(variable *parse.VariableNode, reach requirementReachability) {
|
||||||
|
if len(variable.Ident) < 2 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if variable.Ident[0] != "$" {
|
||||||
|
walker.unresolvedVariableScope(reach)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if !walker.rootKnown {
|
||||||
|
if reach != requirementInactive {
|
||||||
|
walker.unknownVariableEffect = true
|
||||||
|
walker.addIssue("", "", "The root template binding is not known.")
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
walker.recordVariable(variable.Ident[1], reach)
|
||||||
|
if len(variable.Ident) > 2 && reach != requirementInactive {
|
||||||
|
walker.addIssue("", variable.Ident[1], "A chained field access is outside the direct-key scope.")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (walker *requirementWalker) unresolvedVariableScope(reach requirementReachability) {
|
||||||
|
if reach != requirementInactive {
|
||||||
|
walker.unknownVariableEffect = true
|
||||||
|
walker.addIssue("", "", "A caller-variable scope is not statically known.")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (walker *requirementWalker) withDotIdentity(pipe *parse.PipeNode, dot dotIdentity, condition conditionEvaluation) dotIdentity {
|
||||||
|
if pipe != nil && len(pipe.Cmds) == 1 && len(pipe.Decl) == 0 && !pipe.IsAssign && len(pipe.Cmds[0].Args) == 1 &&
|
||||||
|
isRootMap(pipe.Cmds[0].Args[0], dot, walker.rootKnown) {
|
||||||
|
return dotRoot
|
||||||
|
}
|
||||||
|
if condition.known {
|
||||||
|
if _, root := condition.value.(map[string]string); root {
|
||||||
|
return dotRoot
|
||||||
|
}
|
||||||
|
return dotNonRoot
|
||||||
|
}
|
||||||
|
if condition.inputNameUnknown ||
|
||||||
|
!condition.unsupported && (len(condition.missingVariables) > 0 || len(condition.contentInputs) > 0) {
|
||||||
|
// Supported caller values and input bodies are strings; supported
|
||||||
|
// comparisons and negation return booleans, never the root map.
|
||||||
|
return dotNonRoot
|
||||||
|
}
|
||||||
|
return dotUnknown
|
||||||
|
}
|
||||||
|
|
||||||
|
func isRootMap(node parse.Node, dot dotIdentity, rootKnown bool) bool {
|
||||||
|
switch node := unwrapSingleValue(node).(type) {
|
||||||
|
case *parse.DotNode:
|
||||||
|
return dot == dotRoot
|
||||||
|
case *parse.VariableNode:
|
||||||
|
return rootKnown && len(node.Ident) == 1 && node.Ident[0] == "$"
|
||||||
|
default:
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func unwrapSingleValue(node parse.Node) parse.Node {
|
||||||
|
for {
|
||||||
|
pipe, ok := node.(*parse.PipeNode)
|
||||||
|
if !ok || len(pipe.Cmds) != 1 || len(pipe.Decl) != 0 || pipe.IsAssign || len(pipe.Cmds[0].Args) != 1 {
|
||||||
|
return node
|
||||||
|
}
|
||||||
|
node = pipe.Cmds[0].Args[0]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (walker *requirementWalker) recordInput(name string, reach requirementReachability) {
|
||||||
|
evidence := walker.inputs[name]
|
||||||
|
if evidence == nil {
|
||||||
|
evidence = &inputEvidence{
|
||||||
|
name: name, kind: domain.RequirementOptional,
|
||||||
|
explanation: "Declared as an optional input.",
|
||||||
|
}
|
||||||
|
walker.inputs[name] = evidence
|
||||||
|
}
|
||||||
|
if reach == requirementInactive {
|
||||||
|
if !evidence.applicable && evidence.kind != domain.RequirementRequired {
|
||||||
|
evidence.explanation = "This input is referenced only in a skipped template path."
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
evidence.applicable = true
|
||||||
|
walker.promoteInput(evidence, kindForReach(reach))
|
||||||
|
if evidence.kind == domain.RequirementRequired {
|
||||||
|
evidence.explanation = ""
|
||||||
|
} else if reach == requirementPossible {
|
||||||
|
evidence.explanation = "This reference is not known to be reached on every execution path."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if reach != requirementInactive {
|
||||||
|
walker.dependencies++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (walker *requirementWalker) recordVariable(name string, reach requirementReachability) {
|
||||||
|
evidence := walker.variables[name]
|
||||||
|
if evidence == nil {
|
||||||
|
evidence = &variableEvidence{name: name, kind: domain.RequirementOptional}
|
||||||
|
walker.variables[name] = evidence
|
||||||
|
}
|
||||||
|
if reach == requirementInactive {
|
||||||
|
if !evidence.applicable {
|
||||||
|
evidence.explanation = "This variable is referenced only in a skipped template path."
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
evidence.applicable = true
|
||||||
|
walker.promoteVariable(evidence, kindForReach(reach))
|
||||||
|
if evidence.kind == domain.RequirementRequired {
|
||||||
|
evidence.explanation = ""
|
||||||
|
} else if reach == requirementPossible {
|
||||||
|
evidence.explanation = "This reference is not known to be reached on every execution path."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if reach != requirementInactive {
|
||||||
|
walker.dependencies++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (walker *requirementWalker) recordOptionalVariable(name string, reach requirementReachability) {
|
||||||
|
evidence := walker.variables[name]
|
||||||
|
if evidence == nil {
|
||||||
|
evidence = &variableEvidence{name: name, kind: domain.RequirementOptional}
|
||||||
|
walker.variables[name] = evidence
|
||||||
|
}
|
||||||
|
if reach == requirementInactive {
|
||||||
|
if !evidence.applicable {
|
||||||
|
evidence.explanation = "This variable is referenced only in a skipped template path."
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
evidence.applicable = true
|
||||||
|
}
|
||||||
|
if evidence.kind == domain.RequirementOptional && reach != requirementInactive {
|
||||||
|
evidence.explanation = "This key is read through an optional map lookup."
|
||||||
|
}
|
||||||
|
if reach != requirementInactive {
|
||||||
|
walker.dependencies++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (walker *requirementWalker) promoteInput(evidence *inputEvidence, kind domain.RequirementKind) {
|
||||||
|
if requirementRank(kind) > requirementRank(evidence.kind) {
|
||||||
|
evidence.kind = kind
|
||||||
|
}
|
||||||
|
if evidence.kind == domain.RequirementRequired {
|
||||||
|
evidence.explanation = ""
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (walker *requirementWalker) promoteVariable(evidence *variableEvidence, kind domain.RequirementKind) {
|
||||||
|
if requirementRank(kind) > requirementRank(evidence.kind) {
|
||||||
|
evidence.kind = kind
|
||||||
|
}
|
||||||
|
if evidence.kind == domain.RequirementRequired {
|
||||||
|
evidence.explanation = ""
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func kindForReach(reach requirementReachability) domain.RequirementKind {
|
||||||
|
if reach == requirementPossible {
|
||||||
|
return domain.RequirementConditional
|
||||||
|
}
|
||||||
|
return domain.RequirementRequired
|
||||||
|
}
|
||||||
|
|
||||||
|
func requirementRank(kind domain.RequirementKind) int {
|
||||||
|
switch kind {
|
||||||
|
case domain.RequirementRequired:
|
||||||
|
return 3
|
||||||
|
case domain.RequirementConditional:
|
||||||
|
return 2
|
||||||
|
default:
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (walker *requirementWalker) addIssue(inputName, variableName, description string) {
|
||||||
|
walker.addIssueCode(domain.RequirementIssueUnsupportedConstruct, inputName, variableName, description)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (walker *requirementWalker) addIssueCode(code domain.RequirementIssueCode, inputName, variableName, description string) {
|
||||||
|
walker.complete = false
|
||||||
|
issue := domain.RequirementIssue{
|
||||||
|
Code: code,
|
||||||
|
InputName: inputName,
|
||||||
|
VariableName: variableName,
|
||||||
|
Description: description,
|
||||||
|
}
|
||||||
|
key := requirementIssueKey{code: issue.Code, inputName: inputName, variableName: variableName}
|
||||||
|
walker.issues[key] = issue
|
||||||
|
}
|
||||||
|
|
||||||
|
func (walker *requirementWalker) recordConditionLimitation(condition conditionEvaluation) {
|
||||||
|
for name := range condition.missingVariables {
|
||||||
|
walker.addIssueCode(domain.RequirementIssueMissingVariable, "", name, "A missing variable prevents selecting a dependency-bearing branch.")
|
||||||
|
}
|
||||||
|
for name := range condition.contentInputs {
|
||||||
|
walker.addIssueCode(domain.RequirementIssueInputContent, name, "", "Captured input content affects branch selection.")
|
||||||
|
}
|
||||||
|
if condition.unsupported && !condition.inputNameUnknown ||
|
||||||
|
(len(condition.missingVariables) == 0 && len(condition.contentInputs) == 0 && !condition.inputNameUnknown) {
|
||||||
|
walker.addIssue("", "", "A dependency-bearing branch condition is not resolved by the supported syntax analysis.")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (walker *requirementWalker) result(result domain.RequirementInspection) domain.RequirementInspection {
|
||||||
|
result.Complete = walker.complete
|
||||||
|
result.Inputs = make([]domain.InputRequirement, 0, len(walker.inputs))
|
||||||
|
for _, evidence := range walker.inputs {
|
||||||
|
if walker.unknownInputEffect && evidence.kind != domain.RequirementRequired {
|
||||||
|
evidence.kind = domain.RequirementConditional
|
||||||
|
evidence.explanation = "An unresolved input name could refer to this input."
|
||||||
|
}
|
||||||
|
result.Inputs = append(result.Inputs, domain.InputRequirement{
|
||||||
|
Name: evidence.name, Kind: evidence.kind, Declared: evidence.declared,
|
||||||
|
ContentType: copyString(evidence.contentType), Description: copyString(evidence.description),
|
||||||
|
Explanation: evidence.explanation,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
sort.Slice(result.Inputs, func(i, j int) bool { return result.Inputs[i].Name < result.Inputs[j].Name })
|
||||||
|
result.Variables = make([]domain.VariableRequirement, 0, len(walker.variables))
|
||||||
|
for _, evidence := range walker.variables {
|
||||||
|
if walker.unknownVariableEffect && evidence.kind != domain.RequirementRequired {
|
||||||
|
evidence.kind = domain.RequirementConditional
|
||||||
|
evidence.explanation = "An unresolved caller-variable scope could refer to this variable."
|
||||||
|
}
|
||||||
|
_, supplied := walker.vars[evidence.name]
|
||||||
|
result.Variables = append(result.Variables, domain.VariableRequirement{
|
||||||
|
Name: evidence.name, Kind: evidence.kind, Supplied: supplied, Explanation: evidence.explanation,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
sort.Slice(result.Variables, func(i, j int) bool { return result.Variables[i].Name < result.Variables[j].Name })
|
||||||
|
result.Issues = make([]domain.RequirementIssue, 0, len(walker.issues))
|
||||||
|
for _, issue := range walker.issues {
|
||||||
|
result.Issues = append(result.Issues, issue)
|
||||||
|
}
|
||||||
|
sort.Slice(result.Issues, func(i, j int) bool {
|
||||||
|
left, right := result.Issues[i], result.Issues[j]
|
||||||
|
if left.Code != right.Code {
|
||||||
|
return left.Code < right.Code
|
||||||
|
}
|
||||||
|
if left.InputName != right.InputName {
|
||||||
|
return left.InputName < right.InputName
|
||||||
|
}
|
||||||
|
return left.VariableName < right.VariableName
|
||||||
|
})
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
|
func copyString(value *string) *string {
|
||||||
|
if value == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
copy := *value
|
||||||
|
return ©
|
||||||
|
}
|
||||||
671
internal/prompt/requirements_test.go
Normal file
671
internal/prompt/requirements_test.go
Normal file
@@ -0,0 +1,671 @@
|
|||||||
|
package prompt
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"text/template"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestAnalyzeRequirementsDoesNotHideCallerMapScopes(t *testing.T) {
|
||||||
|
for _, source := range []string{
|
||||||
|
`{{with .}}{{.audience}}{{end}}`,
|
||||||
|
`{{with $}}{{.audience}}{{end}}`,
|
||||||
|
`{{with (.)}}{{.audience}}{{end}}`,
|
||||||
|
`{{$alias := .}}{{with $alias}}{{.audience}}{{end}}`,
|
||||||
|
`{{with .}}{{index . "audience"}}{{end}}`,
|
||||||
|
`{{$alias := .}}{{index $alias "audience"}}`,
|
||||||
|
`{{$alias := .}}{{index ($alias) "audience"}}`,
|
||||||
|
`{{$alias := .}}{{$alias.audience}}`,
|
||||||
|
} {
|
||||||
|
t.Run(source, func(t *testing.T) {
|
||||||
|
compiled := compileRequirementTemplate(t, "scope", source)
|
||||||
|
vars := map[string]string{"audience": "visible"}
|
||||||
|
var rendered strings.Builder
|
||||||
|
if err := compiled.Execute(&rendered, vars); err != nil || rendered.String() != "visible" {
|
||||||
|
t.Fatalf("caller-map reference did not render: %q, %v", rendered.String(), err)
|
||||||
|
}
|
||||||
|
result, err := AnalyzeRequirements(context.Background(), &domain.PromptDefinition{}, RequirementSnapshot{
|
||||||
|
Templates: CompiledTemplates{Messages: []*template.Template{compiled}},
|
||||||
|
}, vars, false)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if result.Complete {
|
||||||
|
if len(result.Variables) != 1 || result.Variables[0].Name != "audience" {
|
||||||
|
t.Fatalf("complete discovery lost the caller key: %+v", result)
|
||||||
|
}
|
||||||
|
} else if len(result.Issues) == 0 {
|
||||||
|
t.Fatalf("partial discovery did not explain its limitation: %+v", result)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAnalyzeRequirementsUnknownScopesAffectOptionalEntriesOnlyWhenActive(t *testing.T) {
|
||||||
|
for _, access := range []string{`{{with $alias}}{{.audience}}{{end}}`, `{{index $alias "audience"}}`, `{{index ($alias) "audience"}}`, `{{$alias.audience}}`} {
|
||||||
|
for _, active := range []bool{false, true} {
|
||||||
|
source := `{{if false}}{{.optional}}{{end}}{{$alias := .}}` + access
|
||||||
|
if !active {
|
||||||
|
source = `{{if false}}` + source + `{{end}}`
|
||||||
|
}
|
||||||
|
compiled := compileRequirementTemplate(t, "unknown-scope", source)
|
||||||
|
result, err := AnalyzeRequirements(context.Background(), &domain.PromptDefinition{}, RequirementSnapshot{
|
||||||
|
Templates: CompiledTemplates{Messages: []*template.Template{compiled}},
|
||||||
|
}, map[string]string{"audience": "visible"}, false)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
wantKind := domain.RequirementOptional
|
||||||
|
if active {
|
||||||
|
wantKind = domain.RequirementConditional
|
||||||
|
}
|
||||||
|
if result.Complete == active || len(result.Variables) != 1 || result.Variables[0].Name != "optional" || result.Variables[0].Kind != wantKind {
|
||||||
|
t.Fatalf("active=%v, source=%s, result=%+v", active, source, result)
|
||||||
|
}
|
||||||
|
if active && (len(result.Issues) == 0 || result.Issues[0].Code != domain.RequirementIssueUnsupportedConstruct) {
|
||||||
|
t.Fatalf("active unknown scope lacks a structured issue: %+v", result)
|
||||||
|
}
|
||||||
|
if !active && len(result.Issues) != 0 {
|
||||||
|
t.Fatalf("skipped unknown scope introduced uncertainty: %+v", result)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRequirementWalkerDoesNotRescanNestedUnsupportedNames(t *testing.T) {
|
||||||
|
for _, depth := range []int{6, 12} {
|
||||||
|
expression := ".selector"
|
||||||
|
for level := 0; level < depth; level++ {
|
||||||
|
expression = `input (printf "%s" (` + expression + `))`
|
||||||
|
}
|
||||||
|
source := "{{" + expression + "}}"
|
||||||
|
compiled := compileRequirementTemplate(t, "computed-name", source)
|
||||||
|
walker := &requirementWalker{
|
||||||
|
ctx: context.Background(), vars: map[string]string{"selector": "binding"},
|
||||||
|
inputs: make(map[string]*inputEvidence), variables: make(map[string]*variableEvidence),
|
||||||
|
issues: make(map[requirementIssueKey]domain.RequirementIssue), complete: true, rootKnown: true,
|
||||||
|
}
|
||||||
|
if err := walker.walkTemplate(compiled); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// A generous source-relative bound detects exponential rescanning, not
|
||||||
|
// an exact visit count or a production analysis limit.
|
||||||
|
if walker.dependencies > len(source)*8 {
|
||||||
|
t.Fatalf("depth=%d: %d accesses for %d bytes of source", depth, walker.dependencies, len(source))
|
||||||
|
}
|
||||||
|
result := walker.result(domain.RequirementInspection{})
|
||||||
|
if result.Complete || len(result.Variables) != 1 || result.Variables[0].Name != "selector" ||
|
||||||
|
result.Variables[0].Kind != domain.RequirementConditional || len(result.Issues) == 0 {
|
||||||
|
t.Fatalf("nested unsupported names lost discovery metadata: %+v", result)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAnalyzeRequirementsCollectsDeclaredAndLiteralInputs(t *testing.T) {
|
||||||
|
description := "retained metadata"
|
||||||
|
definition := &domain.PromptDefinition{ID: "capture", Version: "v1"}
|
||||||
|
snapshot := RequirementSnapshot{
|
||||||
|
Inputs: []InputMetadata{
|
||||||
|
{Name: "optional", ContentType: stringPointerForTest("text/plain"), Description: &description},
|
||||||
|
{Name: "unused", Required: true},
|
||||||
|
{Name: "transcript", Required: true},
|
||||||
|
},
|
||||||
|
Templates: CompiledTemplates{Messages: []*template.Template{
|
||||||
|
compileRequirementTemplate(t, "message", `{{input "transcript"}} {{input "extra"}} {{input "extra"}}`),
|
||||||
|
}},
|
||||||
|
}
|
||||||
|
result, err := AnalyzeRequirements(context.Background(), definition, snapshot, nil, false)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if result.PromptID != "capture" || result.PromptVersion != "v1" || !result.Complete || len(result.Issues) != 0 {
|
||||||
|
t.Fatalf("unexpected result identity or completeness: %+v", result)
|
||||||
|
}
|
||||||
|
wantNames := []string{"extra", "optional", "transcript", "unused"}
|
||||||
|
if got := requirementInputNames(result.Inputs); !reflect.DeepEqual(got, wantNames) {
|
||||||
|
t.Fatalf("input names = %v, want %v", got, wantNames)
|
||||||
|
}
|
||||||
|
if got := result.Inputs[0]; got.Kind != domain.RequirementRequired || got.Declared || got.ContentType != nil || got.Description != nil {
|
||||||
|
t.Fatalf("undeclared literal input was not required independently: %+v", got)
|
||||||
|
}
|
||||||
|
if got := result.Inputs[1]; got.Kind != domain.RequirementOptional || !got.Declared || got.ContentType == nil || *got.ContentType != "text/plain" ||
|
||||||
|
got.Description == nil || *got.Description != description || got.Explanation == "" {
|
||||||
|
t.Fatalf("optional declaration metadata was not retained: %+v", got)
|
||||||
|
}
|
||||||
|
if result.Inputs[2].Kind != domain.RequirementRequired || !result.Inputs[2].Declared || result.Inputs[3].Kind != domain.RequirementRequired {
|
||||||
|
t.Fatalf("required declarations or literal use were lost: %+v", result.Inputs)
|
||||||
|
}
|
||||||
|
*result.Inputs[1].Description = "caller mutation"
|
||||||
|
if *snapshot.Inputs[0].Description != description {
|
||||||
|
t.Fatal("result metadata pointer aliases the captured snapshot")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAnalyzeRequirementsDistinguishesDirectAndOptionalCallerKeys(t *testing.T) {
|
||||||
|
result, err := AnalyzeRequirements(context.Background(), &domain.PromptDefinition{}, RequirementSnapshot{
|
||||||
|
Templates: CompiledTemplates{Messages: []*template.Template{
|
||||||
|
compileRequirementTemplate(t, "message", `{{.missing}} {{$.present}} {{index . "lookup"}}`),
|
||||||
|
}},
|
||||||
|
}, map[string]string{"present": ""}, false)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !result.Complete || len(result.Issues) != 0 {
|
||||||
|
t.Fatalf("plain root and literal lookup syntax should be complete: %+v", result)
|
||||||
|
}
|
||||||
|
wantNames := []string{"lookup", "missing", "present"}
|
||||||
|
if got := requirementVariableNames(result.Variables); !reflect.DeepEqual(got, wantNames) {
|
||||||
|
t.Fatalf("variable names = %v, want %v", got, wantNames)
|
||||||
|
}
|
||||||
|
if result.Variables[0].Kind != domain.RequirementOptional || result.Variables[0].Supplied || result.Variables[0].Explanation == "" {
|
||||||
|
t.Errorf("literal index lookup should be optional and absent: %+v", result.Variables[0])
|
||||||
|
}
|
||||||
|
if result.Variables[1].Kind != domain.RequirementRequired || result.Variables[1].Supplied {
|
||||||
|
t.Errorf("missing direct variable should be required but not supplied: %+v", result.Variables[1])
|
||||||
|
}
|
||||||
|
if result.Variables[2].Kind != domain.RequirementRequired || !result.Variables[2].Supplied {
|
||||||
|
t.Errorf("empty supplied direct variable was not detected by presence: %+v", result.Variables[2])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAnalyzeRequirementsSeparatesRootAndLocalTemplateValues(t *testing.T) {
|
||||||
|
result, err := AnalyzeRequirements(context.Background(), &domain.PromptDefinition{}, RequirementSnapshot{
|
||||||
|
Templates: CompiledTemplates{Messages: []*template.Template{
|
||||||
|
compileRequirementTemplate(t, "message", `{{$local := "local"}}{{with .context}}{{$local}} {{.child}} {{$.root}}{{end}}`),
|
||||||
|
}},
|
||||||
|
}, nil, false)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
wantNames := []string{"context", "root"}
|
||||||
|
if got := requirementVariableNames(result.Variables); !reflect.DeepEqual(got, wantNames) {
|
||||||
|
t.Fatalf("root variable names = %v, want %v", got, wantNames)
|
||||||
|
}
|
||||||
|
if result.Variables[0].Kind != domain.RequirementRequired || result.Variables[1].Kind != domain.RequirementConditional || result.Complete {
|
||||||
|
t.Fatalf("with scope and unresolved branch should be conservative: %+v", result)
|
||||||
|
}
|
||||||
|
if len(result.Issues) != 1 || result.Issues[0].Code != domain.RequirementIssueMissingVariable || result.Issues[0].VariableName != "context" {
|
||||||
|
t.Fatalf("conditional root use should have one associated limitation: %+v", result.Issues)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAnalyzeRequirementsMarksBranchAndUnsupportedScopeReferences(t *testing.T) {
|
||||||
|
result, err := AnalyzeRequirements(context.Background(), &domain.PromptDefinition{}, RequirementSnapshot{
|
||||||
|
Templates: CompiledTemplates{Messages: []*template.Template{
|
||||||
|
compileRequirementTemplate(t, "message", `{{if .enabled}}{{input "maybe"}}{{end}}{{range .items}}{{input "inside"}} {{.local}} {{$.root}}{{end}}`),
|
||||||
|
}},
|
||||||
|
}, nil, false)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
wantInputs := []string{"inside", "maybe"}
|
||||||
|
wantVars := []string{"enabled", "items", "root"}
|
||||||
|
if got := requirementInputNames(result.Inputs); !reflect.DeepEqual(got, wantInputs) {
|
||||||
|
t.Fatalf("input names = %v, want %v", got, wantInputs)
|
||||||
|
}
|
||||||
|
if got := requirementVariableNames(result.Variables); !reflect.DeepEqual(got, wantVars) {
|
||||||
|
t.Fatalf("variable names = %v, want %v", got, wantVars)
|
||||||
|
}
|
||||||
|
if result.Complete || len(result.Issues) < 2 {
|
||||||
|
t.Fatalf("branch/range limitations were not reported: %+v", result)
|
||||||
|
}
|
||||||
|
for _, input := range result.Inputs {
|
||||||
|
if input.Kind != domain.RequirementConditional {
|
||||||
|
t.Errorf("input %q should be conservative conditional: %+v", input.Name, input)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if result.Variables[0].Kind != domain.RequirementRequired || result.Variables[1].Kind != domain.RequirementRequired ||
|
||||||
|
result.Variables[2].Kind != domain.RequirementConditional {
|
||||||
|
t.Errorf("root and loop-scope variables were misclassified: %+v", result.Variables)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAnalyzeRequirementsExcludesUnusedDefinitionsAndBypassedSession(t *testing.T) {
|
||||||
|
message := compileRequirementTemplate(t, "message", `{{define "unused"}}{{input "unused"}}{{end}}main`)
|
||||||
|
session := compileRequirementTemplate(t, "session", `{{input "session_only"}}`)
|
||||||
|
result, err := AnalyzeRequirements(context.Background(), &domain.PromptDefinition{}, RequirementSnapshot{
|
||||||
|
Templates: CompiledTemplates{Messages: []*template.Template{message}, Session: session},
|
||||||
|
}, nil, true)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !result.Complete || len(result.Inputs) != 0 {
|
||||||
|
t.Fatalf("unused definitions or bypassed session were included: %+v", result)
|
||||||
|
}
|
||||||
|
result, err = AnalyzeRequirements(context.Background(), &domain.PromptDefinition{}, RequirementSnapshot{
|
||||||
|
Templates: CompiledTemplates{Messages: []*template.Template{message}, Session: session},
|
||||||
|
}, nil, false)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(result.Inputs) != 1 || result.Inputs[0].Name != "session_only" || result.Inputs[0].Kind != domain.RequirementRequired {
|
||||||
|
t.Fatalf("applicable session template was not visited: %+v", result)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAnalyzeRequirementsScansInvokedDefinitionsConservatively(t *testing.T) {
|
||||||
|
message := compileRequirementTemplate(t, "message", `{{define "unused"}}{{input "unused"}}{{end}}{{define "called"}}{{input "called"}} {{.local}}{{end}}main {{template "called" .}}`)
|
||||||
|
result, err := AnalyzeRequirements(context.Background(), &domain.PromptDefinition{}, RequirementSnapshot{
|
||||||
|
Templates: CompiledTemplates{Messages: []*template.Template{message}},
|
||||||
|
}, nil, false)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if result.Complete || len(result.Inputs) != 1 || result.Inputs[0].Name != "called" ||
|
||||||
|
result.Inputs[0].Kind != domain.RequirementConditional || len(result.Variables) != 0 {
|
||||||
|
t.Fatalf("called definition was not scanned conservatively: %+v", result)
|
||||||
|
}
|
||||||
|
if len(result.Issues) < 1 || result.Issues[0].Code != domain.RequirementIssueUnsupportedConstruct {
|
||||||
|
t.Fatalf("named-template limitation was not reported: %+v", result.Issues)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAnalyzeRequirementsClassifiesKnownConditions(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
condition string
|
||||||
|
vars map[string]string
|
||||||
|
wantActive string
|
||||||
|
wantSkipped string
|
||||||
|
}{
|
||||||
|
{name: "string false is truthy", condition: `.flag`, vars: map[string]string{"flag": "false"}, wantActive: "active", wantSkipped: "inactive"},
|
||||||
|
{name: "empty direct value is false", condition: `.flag`, vars: map[string]string{"flag": ""}, wantActive: "inactive", wantSkipped: "active"},
|
||||||
|
{name: "simple equality", condition: `eq .mode "active"`, vars: map[string]string{"mode": "active"}, wantActive: "active", wantSkipped: "inactive"},
|
||||||
|
{name: "simple inequality", condition: `ne .mode "active"`, vars: map[string]string{"mode": "other"}, wantActive: "active", wantSkipped: "inactive"},
|
||||||
|
{name: "not direct value", condition: `not .flag`, vars: map[string]string{"flag": ""}, wantActive: "active", wantSkipped: "inactive"},
|
||||||
|
{name: "literal true", condition: `true`, wantActive: "active", wantSkipped: "inactive"},
|
||||||
|
{name: "literal false", condition: `false`, wantActive: "inactive", wantSkipped: "active"},
|
||||||
|
{name: "missing index is empty", condition: `index . "toggle"`, wantActive: "inactive", wantSkipped: "active"},
|
||||||
|
}
|
||||||
|
for _, test := range tests {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
definition := &domain.PromptDefinition{Templates: []domain.PromptMessageTemplate{{
|
||||||
|
Role: "user", Content: `{{if ` + test.condition + `}}{{input "active"}}{{else}}{{input "inactive"}}{{end}}`,
|
||||||
|
}}}
|
||||||
|
snapshot := RequirementSnapshot{Templates: CompiledTemplates{Messages: []*template.Template{
|
||||||
|
compileRequirementTemplate(t, "message", definition.Templates[0].Content),
|
||||||
|
}}}
|
||||||
|
result, err := AnalyzeRequirements(context.Background(), definition, snapshot, test.vars, false)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !result.Complete || len(result.Issues) != 0 {
|
||||||
|
t.Fatalf("known condition should produce a complete result: %+v", result)
|
||||||
|
}
|
||||||
|
inputs := make(map[string]domain.InputRequirement, len(result.Inputs))
|
||||||
|
for _, input := range result.Inputs {
|
||||||
|
inputs[input.Name] = input
|
||||||
|
}
|
||||||
|
if inputs[test.wantActive].Kind != domain.RequirementRequired || inputs[test.wantSkipped].Kind != domain.RequirementOptional || inputs[test.wantSkipped].Explanation == "" {
|
||||||
|
t.Fatalf("selected/skipped inputs were misclassified: %+v", result.Inputs)
|
||||||
|
}
|
||||||
|
if test.condition == `index . "toggle"` {
|
||||||
|
if len(result.Variables) != 1 || result.Variables[0].Name != "toggle" || result.Variables[0].Kind != domain.RequirementOptional || result.Variables[0].Supplied {
|
||||||
|
t.Fatalf("missing index key was not retained as optional: %+v", result.Variables)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
artifacts := map[string]*domain.Artifact{
|
||||||
|
"active": {Name: "active", Body: []byte("active")},
|
||||||
|
"inactive": {Name: "inactive", Body: []byte("inactive")},
|
||||||
|
}
|
||||||
|
rendered, err := NewGoRenderer().Render(context.Background(), definition, artifacts, test.vars)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
selected := "active"
|
||||||
|
if inputs[selected].Kind != domain.RequirementRequired {
|
||||||
|
selected = "inactive"
|
||||||
|
}
|
||||||
|
if len(rendered.Messages) != 1 || rendered.Messages[0].Content != selected {
|
||||||
|
t.Fatalf("analysis and renderer disagree: result=%+v rendered=%+v", result, rendered)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAnalyzeRequirementsHandlesWithDotAndNestedReachability(t *testing.T) {
|
||||||
|
definition := &domain.PromptDefinition{Templates: []domain.PromptMessageTemplate{{
|
||||||
|
Role: "user", Content: `{{with .context}}{{input "inside"}} {{.child}} {{$.root}}{{else}}{{input "empty"}}{{end}}`,
|
||||||
|
}}}
|
||||||
|
snapshot := RequirementSnapshot{Templates: CompiledTemplates{Messages: []*template.Template{
|
||||||
|
compileRequirementTemplate(t, "message", definition.Templates[0].Content),
|
||||||
|
}}}
|
||||||
|
result, err := AnalyzeRequirements(context.Background(), definition, snapshot, map[string]string{"context": "value", "root": "root"}, false)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !result.Complete || len(result.Inputs) != 2 || len(result.Variables) != 2 {
|
||||||
|
t.Fatalf("known with condition should have a complete inventory: %+v", result)
|
||||||
|
}
|
||||||
|
if result.Inputs[0].Name != "empty" || result.Inputs[0].Kind != domain.RequirementOptional ||
|
||||||
|
result.Inputs[1].Name != "inside" || result.Inputs[1].Kind != domain.RequirementRequired {
|
||||||
|
t.Fatalf("with body and else reachability is incorrect: %+v", result.Inputs)
|
||||||
|
}
|
||||||
|
if result.Variables[0].Name != "context" || result.Variables[0].Kind != domain.RequirementRequired ||
|
||||||
|
result.Variables[1].Name != "root" || result.Variables[1].Kind != domain.RequirementRequired {
|
||||||
|
t.Fatalf("with dot was confused with the root caller map: %+v", result.Variables)
|
||||||
|
}
|
||||||
|
|
||||||
|
nested := compileRequirementTemplate(t, "nested", `{{if .outer}}{{if .inner}}{{input "inner"}}{{else}}{{input "inner_else"}}{{end}}{{else}}{{input "outer_else"}}{{end}}`)
|
||||||
|
nestedResult, err := AnalyzeRequirements(context.Background(), &domain.PromptDefinition{}, RequirementSnapshot{
|
||||||
|
Templates: CompiledTemplates{Messages: []*template.Template{nested}},
|
||||||
|
}, map[string]string{"outer": "yes", "inner": "yes"}, false)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !nestedResult.Complete || nestedResult.Inputs[0].Name != "inner" || nestedResult.Inputs[0].Kind != domain.RequirementRequired ||
|
||||||
|
nestedResult.Inputs[1].Name != "inner_else" || nestedResult.Inputs[1].Kind != domain.RequirementOptional ||
|
||||||
|
nestedResult.Inputs[2].Name != "outer_else" || nestedResult.Inputs[2].Kind != domain.RequirementOptional {
|
||||||
|
t.Fatalf("known nested conditions lost selected or inactive reachability: %+v", nestedResult)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAnalyzeRequirementsReportsUnresolvedDecisionAndContentConditions(t *testing.T) {
|
||||||
|
missing := compileRequirementTemplate(t, "missing", `{{if eq .mode "active"}}{{input "yes"}}{{else}}{{input "no"}}{{end}}`)
|
||||||
|
missingResult, err := AnalyzeRequirements(context.Background(), &domain.PromptDefinition{}, RequirementSnapshot{
|
||||||
|
Templates: CompiledTemplates{Messages: []*template.Template{missing}},
|
||||||
|
}, nil, false)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if missingResult.Complete || len(missingResult.Issues) != 1 || missingResult.Issues[0].Code != domain.RequirementIssueMissingVariable ||
|
||||||
|
missingResult.Issues[0].VariableName != "mode" || missingResult.Inputs[0].Kind != domain.RequirementConditional ||
|
||||||
|
missingResult.Inputs[1].Kind != domain.RequirementConditional {
|
||||||
|
t.Fatalf("missing decision value did not produce conditional dependencies: %+v", missingResult)
|
||||||
|
}
|
||||||
|
|
||||||
|
content := compileRequirementTemplate(t, "content", `{{if input "selector"}}{{input "yes"}}{{else}}{{input "no"}}{{end}}`)
|
||||||
|
contentResult, err := AnalyzeRequirements(context.Background(), &domain.PromptDefinition{}, RequirementSnapshot{
|
||||||
|
Templates: CompiledTemplates{Messages: []*template.Template{content}},
|
||||||
|
}, nil, false)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
contentInputs := make(map[string]domain.InputRequirement, len(contentResult.Inputs))
|
||||||
|
for _, input := range contentResult.Inputs {
|
||||||
|
contentInputs[input.Name] = input
|
||||||
|
}
|
||||||
|
if contentResult.Complete || len(contentResult.Issues) != 1 || contentResult.Issues[0].Code != domain.RequirementIssueInputContent ||
|
||||||
|
contentResult.Issues[0].InputName != "selector" || contentInputs["selector"].Kind != domain.RequirementRequired ||
|
||||||
|
contentInputs["yes"].Kind != domain.RequirementConditional || contentInputs["no"].Kind != domain.RequirementConditional {
|
||||||
|
t.Fatalf("content-dependent condition did not preserve known dependencies: %+v", contentResult)
|
||||||
|
}
|
||||||
|
|
||||||
|
textOnly := compileRequirementTemplate(t, "text-only", `{{if .missing}}plain text{{else}}also plain text{{end}}`)
|
||||||
|
textResult, err := AnalyzeRequirements(context.Background(), &domain.PromptDefinition{}, RequirementSnapshot{
|
||||||
|
Templates: CompiledTemplates{Messages: []*template.Template{textOnly}},
|
||||||
|
}, nil, false)
|
||||||
|
if err != nil || !textResult.Complete || len(textResult.Issues) != 0 {
|
||||||
|
t.Fatalf("unresolved text-only branch should not affect completeness: %+v, %v", textResult, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAnalyzeRequirementsRetainsInactiveEntriesAndPromotesActiveUses(t *testing.T) {
|
||||||
|
description := "declared for a reason"
|
||||||
|
definition := &domain.PromptDefinition{Templates: []domain.PromptMessageTemplate{{
|
||||||
|
Role: "user", Content: `{{if false}}{{input "shared"}} {{.shared_var}} {{input "declared"}}{{input .dynamic}}{{and .left .right}}{{end}}{{input "shared"}} {{.shared_var}}`,
|
||||||
|
}}}
|
||||||
|
snapshot := RequirementSnapshot{
|
||||||
|
Inputs: []InputMetadata{{Name: "declared", Required: true, Description: &description}},
|
||||||
|
Templates: CompiledTemplates{Messages: []*template.Template{
|
||||||
|
compileRequirementTemplate(t, "message", definition.Templates[0].Content),
|
||||||
|
}},
|
||||||
|
}
|
||||||
|
result, err := AnalyzeRequirements(context.Background(), definition, snapshot, map[string]string{"shared_var": "present"}, false)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !result.Complete || len(result.Issues) != 0 {
|
||||||
|
t.Fatalf("skipped unsupported expressions introduced uncertainty: %+v", result)
|
||||||
|
}
|
||||||
|
inputs := make(map[string]domain.InputRequirement, len(result.Inputs))
|
||||||
|
for _, input := range result.Inputs {
|
||||||
|
inputs[input.Name] = input
|
||||||
|
}
|
||||||
|
if len(inputs) != 2 || inputs["shared"].Kind != domain.RequirementRequired || inputs["shared"].Explanation != "" ||
|
||||||
|
inputs["declared"].Kind != domain.RequirementRequired || inputs["declared"].Description == nil || *inputs["declared"].Description != description {
|
||||||
|
t.Fatalf("active or declared requirements did not override inactivity: %+v", result.Inputs)
|
||||||
|
}
|
||||||
|
variables := make(map[string]domain.VariableRequirement, len(result.Variables))
|
||||||
|
for _, variable := range result.Variables {
|
||||||
|
variables[variable.Name] = variable
|
||||||
|
}
|
||||||
|
for _, name := range []string{"dynamic", "left", "right", "shared_var"} {
|
||||||
|
if _, ok := variables[name]; !ok {
|
||||||
|
t.Errorf("inactive caller key %q was not inventoried", name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if variables["shared_var"].Kind != domain.RequirementRequired || variables["shared_var"].Explanation != "" ||
|
||||||
|
variables["dynamic"].Kind != domain.RequirementOptional || variables["dynamic"].Explanation == "" {
|
||||||
|
t.Fatalf("inactive caller metadata was not promoted or explained: %+v", result.Variables)
|
||||||
|
}
|
||||||
|
|
||||||
|
allOptional := compileRequirementTemplate(t, "optional", `{{index . "lookup"}}`)
|
||||||
|
optionalResult, err := AnalyzeRequirements(context.Background(), &domain.PromptDefinition{}, RequirementSnapshot{
|
||||||
|
Inputs: []InputMetadata{{Name: "optional"}}, Templates: CompiledTemplates{Messages: []*template.Template{allOptional}},
|
||||||
|
}, nil, false)
|
||||||
|
if err != nil || !optionalResult.Complete || len(optionalResult.Issues) != 0 || optionalResult.Inputs[0].Kind != domain.RequirementOptional ||
|
||||||
|
optionalResult.Variables[0].Kind != domain.RequirementOptional {
|
||||||
|
t.Fatalf("all-optional inventory should be complete without artifacts: %+v, %v", optionalResult, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAnalyzeRequirementsResolvesSimpleInputNames(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
expression string
|
||||||
|
vars map[string]string
|
||||||
|
kind domain.RequirementKind
|
||||||
|
supplied bool
|
||||||
|
}{
|
||||||
|
{name: "direct dot", expression: `{{input .name}}`, vars: map[string]string{"name": " chosen "}, kind: domain.RequirementRequired, supplied: true},
|
||||||
|
{name: "explicit root", expression: `{{input $.name}}`, vars: map[string]string{"name": "chosen"}, kind: domain.RequirementRequired, supplied: true},
|
||||||
|
{name: "parenthesized direct", expression: `{{input (.name)}}`, vars: map[string]string{"name": "chosen"}, kind: domain.RequirementRequired, supplied: true},
|
||||||
|
{name: "literal lookup", expression: `{{input (index . "name")}}`, vars: map[string]string{"name": "chosen"}, kind: domain.RequirementOptional, supplied: true},
|
||||||
|
{name: "direct pipeline", expression: `{{.name | input}}`, vars: map[string]string{"name": "chosen"}, kind: domain.RequirementRequired, supplied: true},
|
||||||
|
{name: "empty supplied value", expression: `{{input .name}}`, vars: map[string]string{"name": ""}, kind: domain.RequirementRequired, supplied: true},
|
||||||
|
{name: "missing lookup has empty zero value", expression: `{{input (index . "name")}}`, vars: nil, kind: domain.RequirementOptional, supplied: false},
|
||||||
|
}
|
||||||
|
for _, test := range tests {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
definition := &domain.PromptDefinition{Templates: []domain.PromptMessageTemplate{{Role: "user", Content: test.expression}}}
|
||||||
|
snapshot := RequirementSnapshot{Templates: CompiledTemplates{Messages: []*template.Template{
|
||||||
|
compileRequirementTemplate(t, "message", test.expression),
|
||||||
|
}}}
|
||||||
|
result, err := AnalyzeRequirements(context.Background(), definition, snapshot, test.vars, false)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
inputName := "chosen"
|
||||||
|
if test.name == "direct dot" {
|
||||||
|
inputName = " chosen "
|
||||||
|
}
|
||||||
|
if test.name == "empty supplied value" || test.name == "missing lookup has empty zero value" {
|
||||||
|
inputName = ""
|
||||||
|
}
|
||||||
|
if !result.Complete || len(result.Issues) != 0 || len(result.Inputs) != 1 || result.Inputs[0].Name != inputName ||
|
||||||
|
result.Inputs[0].Kind != domain.RequirementRequired {
|
||||||
|
t.Fatalf("simple input name was not resolved exactly: %+v", result)
|
||||||
|
}
|
||||||
|
if len(result.Variables) != 1 || result.Variables[0].Name != "name" || result.Variables[0].Kind != test.kind || result.Variables[0].Supplied != test.supplied {
|
||||||
|
t.Fatalf("name-source caller key classification is incorrect: %+v", result.Variables)
|
||||||
|
}
|
||||||
|
artifacts := map[string]*domain.Artifact{inputName: {Name: inputName, Body: []byte("resolved")}}
|
||||||
|
rendered, err := NewGoRenderer().Render(context.Background(), definition, artifacts, test.vars)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(rendered.Messages) != 1 || rendered.Messages[0].Content != "resolved" {
|
||||||
|
t.Fatalf("analyzed input name differs from renderer: %+v", rendered)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAnalyzeRequirementsReportsUnresolvedInputNamesAndPromotesUnknownEffects(t *testing.T) {
|
||||||
|
definition := &domain.PromptDefinition{Templates: []domain.PromptMessageTemplate{{
|
||||||
|
Role: "user", Content: `{{input .missing}}`,
|
||||||
|
}}}
|
||||||
|
result, err := AnalyzeRequirements(context.Background(), definition, RequirementSnapshot{
|
||||||
|
Inputs: []InputMetadata{{Name: "optional"}, {Name: "required", Required: true}},
|
||||||
|
Templates: CompiledTemplates{Messages: []*template.Template{
|
||||||
|
compileRequirementTemplate(t, "message", definition.Templates[0].Content),
|
||||||
|
}},
|
||||||
|
}, nil, false)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if result.Complete || len(result.Issues) != 2 || result.Issues[0].Code != domain.RequirementIssueDynamicInputName ||
|
||||||
|
result.Issues[1].Code != domain.RequirementIssueMissingVariable {
|
||||||
|
t.Fatalf("missing dynamic name did not report its limitations: %+v", result)
|
||||||
|
}
|
||||||
|
if result.Inputs[0].Name != "optional" || result.Inputs[0].Kind != domain.RequirementConditional || result.Inputs[0].Explanation == "" ||
|
||||||
|
result.Inputs[1].Name != "required" || result.Inputs[1].Kind != domain.RequirementRequired || len(result.Variables) != 1 ||
|
||||||
|
result.Variables[0].Name != "missing" || result.Variables[0].Kind != domain.RequirementRequired {
|
||||||
|
t.Fatalf("unknown name effects lost known classifications: %+v", result)
|
||||||
|
}
|
||||||
|
|
||||||
|
computed := compileRequirementTemplate(t, "computed", `{{input (printf "%s" .prefix)}}`)
|
||||||
|
computedResult, err := AnalyzeRequirements(context.Background(), &domain.PromptDefinition{}, RequirementSnapshot{
|
||||||
|
Inputs: []InputMetadata{{Name: "declared"}}, Templates: CompiledTemplates{Messages: []*template.Template{computed}},
|
||||||
|
}, map[string]string{"prefix": "SENSITIVE-VALUE"}, false)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if computedResult.Complete || len(computedResult.Inputs) != 1 || computedResult.Inputs[0].Name != "declared" ||
|
||||||
|
computedResult.Inputs[0].Kind != domain.RequirementConditional || len(computedResult.Variables) != 1 ||
|
||||||
|
computedResult.Variables[0].Name != "prefix" || computedResult.Variables[0].Kind != domain.RequirementConditional {
|
||||||
|
t.Fatalf("computed-name uncertainty did not affect optional entries: %+v", computedResult)
|
||||||
|
}
|
||||||
|
for _, issue := range computedResult.Issues {
|
||||||
|
if strings.Contains(issue.Description, "SENSITIVE-VALUE") || issue.InputName == "SENSITIVE-VALUE" || issue.VariableName == "SENSITIVE-VALUE" {
|
||||||
|
t.Fatalf("input-name source value leaked into diagnostics: %+v", issue)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, input := range computedResult.Inputs {
|
||||||
|
if strings.Contains(input.Explanation, "SENSITIVE-VALUE") {
|
||||||
|
t.Fatalf("input-name source value leaked into an explanation: %+v", input)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, variable := range computedResult.Variables {
|
||||||
|
if strings.Contains(variable.Explanation, "SENSITIVE-VALUE") {
|
||||||
|
t.Fatalf("input-name source value leaked into an explanation: %+v", variable)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, expression := range []string{`{{input (input "selector")}}`, `{{input "selector" | input}}`} {
|
||||||
|
t.Run(expression, func(t *testing.T) {
|
||||||
|
contentName := compileRequirementTemplate(t, "content-name", expression)
|
||||||
|
contentResult, err := AnalyzeRequirements(context.Background(), &domain.PromptDefinition{}, RequirementSnapshot{
|
||||||
|
Templates: CompiledTemplates{Messages: []*template.Template{contentName}},
|
||||||
|
}, nil, false)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if contentResult.Complete || len(contentResult.Issues) != 2 ||
|
||||||
|
contentResult.Issues[0].Code != domain.RequirementIssueDynamicInputName ||
|
||||||
|
contentResult.Issues[1].Code != domain.RequirementIssueInputContent || contentResult.Issues[1].InputName != "selector" || len(contentResult.Inputs) != 1 ||
|
||||||
|
contentResult.Inputs[0].Name != "selector" || contentResult.Inputs[0].Kind != domain.RequirementRequired {
|
||||||
|
t.Fatalf("input-body-derived name did not preserve the known source dependency: %+v", contentResult)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAnalyzeRequirementsDoesNotResolveSkippedComputedInputNames(t *testing.T) {
|
||||||
|
compiled := compileRequirementTemplate(t, "inactive-name", `{{if false}}{{input .name}}{{input (printf "%s" .other)}}{{"piped-literal" | input}}{{template "missing" .arg}}{{end}}`)
|
||||||
|
result, err := AnalyzeRequirements(context.Background(), &domain.PromptDefinition{}, RequirementSnapshot{
|
||||||
|
Templates: CompiledTemplates{Messages: []*template.Template{compiled}},
|
||||||
|
}, nil, false)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !result.Complete || len(result.Issues) != 0 || len(result.Inputs) != 1 || result.Inputs[0].Name != "piped-literal" ||
|
||||||
|
result.Inputs[0].Kind != domain.RequirementOptional || result.Inputs[0].Explanation == "" || len(result.Variables) != 3 {
|
||||||
|
t.Fatalf("skipped computed expressions introduced uncertainty or names: %+v", result)
|
||||||
|
}
|
||||||
|
for _, variable := range result.Variables {
|
||||||
|
if variable.Kind != domain.RequirementOptional || variable.Explanation == "" {
|
||||||
|
t.Errorf("skipped variable %q was not retained as inactive optional: %+v", variable.Name, variable)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAnalyzeRequirementsPromotesOptionalVariablesWhenRootScopeIsUnknown(t *testing.T) {
|
||||||
|
compiled := compileRequirementTemplate(t, "scope", `{{index . "optional"}} {{.required}} {{template "called" .}}{{define "called"}}{{$.hidden}}{{end}}`)
|
||||||
|
result, err := AnalyzeRequirements(context.Background(), &domain.PromptDefinition{}, RequirementSnapshot{
|
||||||
|
Templates: CompiledTemplates{Messages: []*template.Template{compiled}},
|
||||||
|
}, map[string]string{"required": "value"}, false)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if result.Complete || len(result.Variables) != 2 || result.Variables[0].Name != "optional" ||
|
||||||
|
result.Variables[0].Kind != domain.RequirementConditional || result.Variables[0].Explanation == "" ||
|
||||||
|
result.Variables[1].Name != "required" || result.Variables[1].Kind != domain.RequirementRequired {
|
||||||
|
t.Fatalf("unknown caller scope did not affect only optional known entries: %+v", result)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAnalyzeRequirementsLeavesAssignmentDerivedNamesPartial(t *testing.T) {
|
||||||
|
compiled := compileRequirementTemplate(t, "assignment", `{{$alias := .source}}{{input $alias}}`)
|
||||||
|
result, err := AnalyzeRequirements(context.Background(), &domain.PromptDefinition{}, RequirementSnapshot{
|
||||||
|
Inputs: []InputMetadata{{Name: "known-optional"}}, Templates: CompiledTemplates{Messages: []*template.Template{compiled}},
|
||||||
|
}, nil, false)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if result.Complete || len(result.Issues) == 0 || len(result.Inputs) != 1 ||
|
||||||
|
result.Inputs[0].Name != "known-optional" || result.Inputs[0].Kind != domain.RequirementConditional ||
|
||||||
|
len(result.Variables) != 1 || result.Variables[0].Name != "source" || result.Variables[0].Kind != domain.RequirementConditional {
|
||||||
|
t.Fatalf("assignment-derived input name was interpreted or lost: %+v", result)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAnalyzeRequirementsSupportsEmptyTemplatesAndCancellation(t *testing.T) {
|
||||||
|
result, err := AnalyzeRequirements(context.Background(), &domain.PromptDefinition{}, RequirementSnapshot{
|
||||||
|
Templates: CompiledTemplates{Messages: []*template.Template{compileRequirementTemplate(t, "message", "plain text")}},
|
||||||
|
}, nil, false)
|
||||||
|
if err != nil || !result.Complete || len(result.Inputs) != 0 || len(result.Variables) != 0 {
|
||||||
|
t.Fatalf("input-free template should have a complete empty inventory: %+v, %v", result, err)
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
cancel()
|
||||||
|
if _, err := AnalyzeRequirements(ctx, &domain.PromptDefinition{}, RequirementSnapshot{}, nil, false); !errors.Is(err, context.Canceled) {
|
||||||
|
t.Fatalf("canceled context returned %v, want context.Canceled", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func compileRequirementTemplate(t *testing.T, name, source string) *template.Template {
|
||||||
|
t.Helper()
|
||||||
|
compiled, err := CompileTemplate(name, source)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("compile template: %v", err)
|
||||||
|
}
|
||||||
|
return compiled
|
||||||
|
}
|
||||||
|
|
||||||
|
func requirementInputNames(inputs []domain.InputRequirement) []string {
|
||||||
|
names := make([]string, len(inputs))
|
||||||
|
for index, input := range inputs {
|
||||||
|
names[index] = input.Name
|
||||||
|
}
|
||||||
|
return names
|
||||||
|
}
|
||||||
|
|
||||||
|
func requirementVariableNames(variables []domain.VariableRequirement) []string {
|
||||||
|
names := make([]string, len(variables))
|
||||||
|
for index, variable := range variables {
|
||||||
|
names[index] = variable.Name
|
||||||
|
}
|
||||||
|
return names
|
||||||
|
}
|
||||||
|
|
||||||
|
func stringPointerForTest(value string) *string {
|
||||||
|
return &value
|
||||||
|
}
|
||||||
@@ -9,6 +9,22 @@ type CompiledTemplates struct {
|
|||||||
Messages []*template.Template
|
Messages []*template.Template
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// InputMetadata is the captured declaration metadata for one prompt input.
|
||||||
|
// Nil metadata pointers mean the corresponding field was absent or null.
|
||||||
|
type InputMetadata struct {
|
||||||
|
Name string
|
||||||
|
Required bool
|
||||||
|
ContentType *string
|
||||||
|
Description *string
|
||||||
|
}
|
||||||
|
|
||||||
|
// RequirementSnapshot provides the captured syntax and declarations needed
|
||||||
|
// for requirement inspection without reopening prompt resources.
|
||||||
|
type RequirementSnapshot struct {
|
||||||
|
Templates CompiledTemplates
|
||||||
|
Inputs []InputMetadata
|
||||||
|
}
|
||||||
|
|
||||||
// TemplateLookup provides parsed templates for a captured prompt identity.
|
// TemplateLookup provides parsed templates for a captured prompt identity.
|
||||||
type TemplateLookup interface {
|
type TemplateLookup interface {
|
||||||
GetCompiledTemplates(id, version string) (CompiledTemplates, bool)
|
GetCompiledTemplates(id, version string) (CompiledTemplates, bool)
|
||||||
|
|||||||
@@ -50,10 +50,12 @@ func (r *Runner) PrepareExecution(ctx context.Context, req domain.RunRequest) (*
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
executionSnapshot, err := clonePreparedRun(operation.run)
|
executionSnapshot := operation.run
|
||||||
|
extraParams, err := jsonvalue.CopyMap(executionSnapshot.EffectiveModelParams.ExtraParams)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("%w: failed to copy prepared execution: %v", ErrInvalidRequest, err)
|
return nil, fmt.Errorf("%w: failed to copy prepared execution: %v", ErrInvalidRequest, err)
|
||||||
}
|
}
|
||||||
|
executionSnapshot.EffectiveModelParams.ExtraParams = extraParams
|
||||||
|
|
||||||
details, err := clonePreparedRun(executionSnapshot)
|
details, err := clonePreparedRun(executionSnapshot)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -631,6 +631,124 @@ func TestRunnerPreparedExecutionOwnershipUseAndDiscard(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRunnerPreparedExecutionDetachesTransferredSnapshotSources(t *testing.T) {
|
||||||
|
newExtras := func(value string) map[string]any {
|
||||||
|
return map[string]any{"provider_options": map[string]any{
|
||||||
|
"provider": map[string]any{"settings": map[string]any{"labels": []any{value}}},
|
||||||
|
}}
|
||||||
|
}
|
||||||
|
extraLabel := func(extraParams map[string]any) string {
|
||||||
|
return extraParams["provider_options"].(map[string]any)["provider"].(map[string]any)["settings"].(map[string]any)["labels"].([]any)[0].(string)
|
||||||
|
}
|
||||||
|
mutateExtra := func(extraParams map[string]any, value string) {
|
||||||
|
extraParams["provider_options"].(map[string]any)["provider"].(map[string]any)["settings"].(map[string]any)["labels"].([]any)[0] = value
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, test := range []struct {
|
||||||
|
name string
|
||||||
|
requestOverride bool
|
||||||
|
}{
|
||||||
|
{name: "profile target"},
|
||||||
|
{name: "request override", requestOverride: true},
|
||||||
|
} {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
profileExtras := newExtras("profile source")
|
||||||
|
requestExtras := newExtras("request source")
|
||||||
|
profile := defaultExecutionProfile()
|
||||||
|
profile.ResolvedTarget.ExtraParams = profileExtras
|
||||||
|
|
||||||
|
schemaDocument := map[string]any{
|
||||||
|
"type": "object",
|
||||||
|
"properties": map[string]any{"value": map[string]any{"type": "string"}},
|
||||||
|
}
|
||||||
|
definition := promptDef(domain.FormatJSON, domain.ValidationJSONSchema, 0)
|
||||||
|
definition.Validation.SchemaPath = "schema.json"
|
||||||
|
control := &domain.CacheControl{Type: domain.CacheControlEphemeral, TTL: "1h"}
|
||||||
|
rendered := &domain.RenderedPrompt{Messages: []domain.RenderedMessage{{
|
||||||
|
Role: domain.RoleUser, Content: "captured rendering", CacheControl: control,
|
||||||
|
}}}
|
||||||
|
renderer := &fakeRenderer{rendered: rendered}
|
||||||
|
sourceArtifact := &domain.Artifact{Body: []byte("captured input"), Hash: "captured-input-hash"}
|
||||||
|
reader := &fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{"source": sourceArtifact}}
|
||||||
|
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: `{"value":"ok"}`}}
|
||||||
|
validator := &recordingValidationPreparer{
|
||||||
|
plan: &recordingPreparedValidation{schemaDocument: schemaDocument},
|
||||||
|
}
|
||||||
|
runner := NewRunner(
|
||||||
|
&fakePromptRepo{def: definition},
|
||||||
|
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": profile}},
|
||||||
|
reader, renderer, llmClient, validator, nil,
|
||||||
|
)
|
||||||
|
request := domain.RunRequest{
|
||||||
|
PromptID: "p", PromptVersion: "1", ProfileID: "exec",
|
||||||
|
Inputs: map[string]domain.ArtifactRef{
|
||||||
|
"transcript": {Type: domain.ArtifactRefFile, URI: "source"},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
wantLabel := "profile source"
|
||||||
|
if test.requestOverride {
|
||||||
|
request.Execution = &domain.ExecutionTargetOverride{ExtraParams: requestExtras}
|
||||||
|
wantLabel = "request source"
|
||||||
|
}
|
||||||
|
|
||||||
|
prepared, err := runner.PrepareExecution(context.Background(), request)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("prepare execution: %v", err)
|
||||||
|
}
|
||||||
|
mutateExtra(profileExtras, "mutated profile source")
|
||||||
|
mutateExtra(requestExtras, "mutated request source")
|
||||||
|
rendered.Messages[0].Content = "mutated renderer source"
|
||||||
|
control.TTL = "mutated renderer cache metadata"
|
||||||
|
schemaDocument["properties"].(map[string]any)["value"].(map[string]any)["type"] = "integer"
|
||||||
|
sourceArtifact.Hash = "mutated source hash"
|
||||||
|
|
||||||
|
details := prepared.Details()
|
||||||
|
if details == nil || len(details.Messages) != 1 || details.Messages[0].Content != "captured rendering" ||
|
||||||
|
details.Messages[0].CacheControl == nil || details.Messages[0].CacheControl.TTL != "1h" ||
|
||||||
|
details.InputHashes["transcript"] != "captured-input-hash" ||
|
||||||
|
extraLabel(details.EffectiveModelParams.ExtraParams) != wantLabel ||
|
||||||
|
details.StructuredOutput == nil || details.StructuredOutput.JSONSchema == nil ||
|
||||||
|
details.StructuredOutput.JSONSchema.Schema.(map[string]any)["properties"].(map[string]any)["value"].(map[string]any)["type"] != "string" {
|
||||||
|
t.Fatalf("prepared details retained a source alias: %#v", details)
|
||||||
|
}
|
||||||
|
|
||||||
|
result, err := runner.RunPrepared(context.Background(), prepared)
|
||||||
|
if err != nil || result == nil || !result.Validation.IsValid || result.InputHashes["transcript"] != "captured-input-hash" {
|
||||||
|
t.Fatalf("run prepared snapshot=(%+v, %v)", result, err)
|
||||||
|
}
|
||||||
|
if len(llmClient.lastReq.Prompt.Messages) != 1 || llmClient.lastReq.StructuredOutput == nil ||
|
||||||
|
llmClient.lastReq.StructuredOutput.JSONSchema == nil ||
|
||||||
|
extraLabel(llmClient.lastReq.Target.ExtraParams) != wantLabel ||
|
||||||
|
llmClient.lastReq.Prompt.Messages[0].Content != "captured rendering" ||
|
||||||
|
llmClient.lastReq.Prompt.Messages[0].CacheControl == nil || llmClient.lastReq.Prompt.Messages[0].CacheControl.TTL != "1h" ||
|
||||||
|
llmClient.lastReq.StructuredOutput.JSONSchema.Schema.(map[string]any)["properties"].(map[string]any)["value"].(map[string]any)["type"] != "string" {
|
||||||
|
t.Fatalf("execution snapshot retained a source alias: %#v", llmClient.lastReq)
|
||||||
|
}
|
||||||
|
if completedDetails := prepared.Details(); completedDetails == nil ||
|
||||||
|
len(completedDetails.Messages) != 1 || completedDetails.Messages[0].Content != "captured rendering" ||
|
||||||
|
extraLabel(completedDetails.EffectiveModelParams.ExtraParams) != wantLabel {
|
||||||
|
t.Fatalf("completed details changed after source mutation: %#v", completedDetails)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunnerPrepareExecutionClassifiesExtraCopyFailureAsInvalidRequest(t *testing.T) {
|
||||||
|
profile := defaultExecutionProfile()
|
||||||
|
profile.ResolvedTarget.ExtraParams = map[string]any{"unsupported": func() {}}
|
||||||
|
runner := NewRunner(
|
||||||
|
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
|
||||||
|
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": profile}},
|
||||||
|
defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil, nil,
|
||||||
|
)
|
||||||
|
prepared, err := runner.PrepareExecution(context.Background(), domain.RunRequest{
|
||||||
|
PromptID: "p", ProfileID: "exec",
|
||||||
|
})
|
||||||
|
if prepared != nil || !errors.Is(err, ErrInvalidRequest) {
|
||||||
|
t.Fatalf("prepared execution copy failure=(%v, %v), want ErrInvalidRequest", prepared, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestRunnerPreparedExecutionWithoutValidatorSkipsValidation(t *testing.T) {
|
func TestRunnerPreparedExecutionWithoutValidatorSkipsValidation(t *testing.T) {
|
||||||
runner := NewRunner(
|
runner := NewRunner(
|
||||||
&fakePromptRepo{def: promptDef(domain.FormatJSON, domain.ValidationJSON, 0)},
|
&fakePromptRepo{def: promptDef(domain.FormatJSON, domain.ValidationJSON, 0)},
|
||||||
|
|||||||
237
internal/usecase/prepared_snapshot_benchmark_test.go
Normal file
237
internal/usecase/prepared_snapshot_benchmark_test.go
Normal file
@@ -0,0 +1,237 @@
|
|||||||
|
package usecase
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"reflect"
|
||||||
|
"runtime"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/validate"
|
||||||
|
)
|
||||||
|
|
||||||
|
var preparedSnapshotHandleSink *PreparedExecution
|
||||||
|
|
||||||
|
const preparedSnapshotRetainedBatchSize = 4
|
||||||
|
|
||||||
|
type preparedSnapshotBenchmarkClient struct{}
|
||||||
|
|
||||||
|
func (preparedSnapshotBenchmarkClient) Generate(
|
||||||
|
context.Context,
|
||||||
|
domain.GenerateRequest,
|
||||||
|
) (*domain.GenerateResponse, error) {
|
||||||
|
return &domain.GenerateResponse{Content: "ok"}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type preparedSnapshotBenchmarkValidation struct{ schema any }
|
||||||
|
|
||||||
|
func (validation *preparedSnapshotBenchmarkValidation) Validate(
|
||||||
|
context.Context,
|
||||||
|
*domain.Artifact,
|
||||||
|
) (domain.ValidationResult, error) {
|
||||||
|
return domain.ValidationResult{Status: domain.ValidationPassed, IsValid: true}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (validation *preparedSnapshotBenchmarkValidation) SchemaDocument() any {
|
||||||
|
return validation.schema
|
||||||
|
}
|
||||||
|
|
||||||
|
type preparedSnapshotBenchmarkValidator struct{ schema any }
|
||||||
|
|
||||||
|
func (validator preparedSnapshotBenchmarkValidator) Validate(
|
||||||
|
context.Context,
|
||||||
|
*domain.Artifact,
|
||||||
|
domain.OutputContract,
|
||||||
|
) (domain.ValidationResult, error) {
|
||||||
|
return domain.ValidationResult{Status: domain.ValidationPassed, IsValid: true}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (validator preparedSnapshotBenchmarkValidator) PrepareValidation(
|
||||||
|
context.Context,
|
||||||
|
domain.OutputContract,
|
||||||
|
) (validate.PreparedValidation, error) {
|
||||||
|
return &preparedSnapshotBenchmarkValidation{schema: validator.schema}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func preparedSnapshotBenchmarkSchema(properties int) map[string]any {
|
||||||
|
fields := make(map[string]any, properties)
|
||||||
|
for index := range properties {
|
||||||
|
fields[fmt.Sprintf("property-%03d", index)] = map[string]any{"type": "string"}
|
||||||
|
}
|
||||||
|
return map[string]any{
|
||||||
|
"type": "object", "properties": fields, "additionalProperties": false,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func preparedSnapshotBenchmarkExtras(entries int) map[string]any {
|
||||||
|
providers := make(map[string]any, entries)
|
||||||
|
for index := range entries {
|
||||||
|
providers[fmt.Sprintf("provider-%03d", index)] = map[string]any{
|
||||||
|
"enabled": true,
|
||||||
|
"labels": []any{fmt.Sprintf("tier-%d", index%4), fmt.Sprintf("region-%d", index%3)},
|
||||||
|
"limits": map[string]any{"burst": index + 1, "period": "minute"},
|
||||||
|
"routes": []any{map[string]any{"weight": index + 1, "fallback": true}},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return map[string]any{"provider_options": providers}
|
||||||
|
}
|
||||||
|
|
||||||
|
func newPreparedSnapshotBenchmarkRunner(
|
||||||
|
mutable bool,
|
||||||
|
) (*Runner, domain.RunRequest, any) {
|
||||||
|
messageCount := 1
|
||||||
|
extraEntries := 0
|
||||||
|
schemaProperties := 1
|
||||||
|
if mutable {
|
||||||
|
messageCount = 8
|
||||||
|
extraEntries = 64
|
||||||
|
schemaProperties = 256
|
||||||
|
}
|
||||||
|
|
||||||
|
messages := make([]domain.RenderedMessage, messageCount)
|
||||||
|
for index := range messages {
|
||||||
|
messages[index] = domain.RenderedMessage{
|
||||||
|
Role: domain.RoleUser, Content: fmt.Sprintf("rendered message %d", index),
|
||||||
|
CacheControl: &domain.CacheControl{Type: domain.CacheControlEphemeral, TTL: "1h"},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{
|
||||||
|
SessionID: "snapshot-session", Messages: messages,
|
||||||
|
}}
|
||||||
|
|
||||||
|
profile := defaultExecutionProfile()
|
||||||
|
if extraEntries > 0 {
|
||||||
|
profile.ResolvedTarget.ExtraParams = preparedSnapshotBenchmarkExtras(extraEntries)
|
||||||
|
}
|
||||||
|
profileRepository := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": profile}}
|
||||||
|
definition := promptDef(domain.FormatJSON, domain.ValidationJSONSchema, 0)
|
||||||
|
definition.Validation.SchemaPath = "snapshot-schema.json"
|
||||||
|
schema := preparedSnapshotBenchmarkSchema(schemaProperties)
|
||||||
|
validator := preparedSnapshotBenchmarkValidator{schema: schema}
|
||||||
|
runner := NewRunner(
|
||||||
|
&fakePromptRepo{def: definition},
|
||||||
|
profileRepository,
|
||||||
|
staticArtifactReader{},
|
||||||
|
renderer,
|
||||||
|
preparedSnapshotBenchmarkClient{},
|
||||||
|
validator,
|
||||||
|
nil,
|
||||||
|
)
|
||||||
|
request := domain.RunRequest{
|
||||||
|
PromptID: "p", PromptVersion: "1", ProfileID: "exec",
|
||||||
|
Inputs: singleInputRef(), Vars: map[string]string{"name": "Ada"},
|
||||||
|
AppendedMessages: []domain.RenderedMessage{{
|
||||||
|
Role: domain.RoleAssistant, Content: "appended message",
|
||||||
|
CacheControl: &domain.CacheControl{Type: domain.CacheControlEphemeral, TTL: "1h"},
|
||||||
|
}},
|
||||||
|
}
|
||||||
|
return runner, request, schema
|
||||||
|
}
|
||||||
|
|
||||||
|
func checkPreparedSnapshotBenchmarkFixture(b *testing.B, runner *Runner, request domain.RunRequest, schema any) {
|
||||||
|
b.Helper()
|
||||||
|
prepared, err := runner.PrepareExecution(context.Background(), request)
|
||||||
|
if err != nil {
|
||||||
|
b.Fatal(err)
|
||||||
|
}
|
||||||
|
details := prepared.Details()
|
||||||
|
if details == nil || details.PromptID != request.PromptID || len(details.Messages) < 2 ||
|
||||||
|
details.Messages[0].Content != "rendered message 0" || details.Messages[len(details.Messages)-1].Content != "appended message" ||
|
||||||
|
details.InputHashes["transcript"] == "" || details.StructuredOutput == nil || details.StructuredOutput.JSONSchema == nil {
|
||||||
|
prepared.Discard()
|
||||||
|
b.Fatalf("prepared snapshot fixture is incomplete: %#v", details)
|
||||||
|
}
|
||||||
|
if got := details.StructuredOutput.JSONSchema.Schema; !reflect.DeepEqual(got, schema) {
|
||||||
|
prepared.Discard()
|
||||||
|
b.Fatalf("prepared schema differs from fixture: %#v", got)
|
||||||
|
}
|
||||||
|
result, err := runner.RunPrepared(context.Background(), prepared)
|
||||||
|
if err != nil || result == nil || result.RawOutput != "ok" || !result.Validation.IsValid {
|
||||||
|
b.Fatalf("completed fixture=(%+v, %v)", result, err)
|
||||||
|
}
|
||||||
|
completedDetails := prepared.Details()
|
||||||
|
if completedDetails == nil || completedDetails.Messages[0].Content != "rendered message 0" ||
|
||||||
|
!reflect.DeepEqual(completedDetails.StructuredOutput.JSONSchema.Schema, schema) {
|
||||||
|
b.Fatalf("completed handle did not retain its details: %#v", completedDetails)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func BenchmarkPreparedSnapshot(b *testing.B) {
|
||||||
|
for _, fixture := range []struct {
|
||||||
|
name string
|
||||||
|
mutable bool
|
||||||
|
}{
|
||||||
|
{name: "small"},
|
||||||
|
{name: "mutable_payload", mutable: true},
|
||||||
|
} {
|
||||||
|
b.Run(fixture.name, func(b *testing.B) {
|
||||||
|
runner, request, schema := newPreparedSnapshotBenchmarkRunner(fixture.mutable)
|
||||||
|
checkPreparedSnapshotBenchmarkFixture(b, runner, request, schema)
|
||||||
|
b.ReportAllocs()
|
||||||
|
b.ResetTimer()
|
||||||
|
for b.Loop() {
|
||||||
|
prepared, err := runner.PrepareExecution(context.Background(), request)
|
||||||
|
if err != nil {
|
||||||
|
b.Fatal(err)
|
||||||
|
}
|
||||||
|
prepared.Discard()
|
||||||
|
preparedSnapshotHandleSink = prepared
|
||||||
|
}
|
||||||
|
b.StopTimer()
|
||||||
|
preparedSnapshotHandleSink = nil
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func BenchmarkRetainedPreparedHandles(b *testing.B) {
|
||||||
|
for _, state := range []string{"ready", "completed", "discarded"} {
|
||||||
|
b.Run(state, func(b *testing.B) {
|
||||||
|
runner, request, schema := newPreparedSnapshotBenchmarkRunner(true)
|
||||||
|
checkPreparedSnapshotBenchmarkFixture(b, runner, request, schema)
|
||||||
|
retained := make([]*PreparedExecution, preparedSnapshotRetainedBatchSize)
|
||||||
|
runtime.GC()
|
||||||
|
var setupMemory runtime.MemStats
|
||||||
|
runtime.ReadMemStats(&setupMemory)
|
||||||
|
runtime.KeepAlive(runner)
|
||||||
|
runtime.KeepAlive(request)
|
||||||
|
runtime.KeepAlive(retained)
|
||||||
|
|
||||||
|
retainHandle := func() *PreparedExecution {
|
||||||
|
prepared, err := runner.PrepareExecution(context.Background(), request)
|
||||||
|
if err != nil {
|
||||||
|
b.Fatal(err)
|
||||||
|
}
|
||||||
|
switch state {
|
||||||
|
case "completed":
|
||||||
|
result, err := runner.RunPrepared(context.Background(), prepared)
|
||||||
|
if err != nil || result == nil || result.RawOutput != "ok" || !result.Validation.IsValid {
|
||||||
|
b.Fatalf("complete retained handle=(%+v, %v)", result, err)
|
||||||
|
}
|
||||||
|
case "discarded":
|
||||||
|
prepared.Discard()
|
||||||
|
}
|
||||||
|
return prepared
|
||||||
|
}
|
||||||
|
|
||||||
|
b.ReportAllocs()
|
||||||
|
b.ResetTimer()
|
||||||
|
for iteration := 0; iteration < b.N; iteration++ {
|
||||||
|
retained[iteration%len(retained)] = retainHandle()
|
||||||
|
}
|
||||||
|
b.StopTimer()
|
||||||
|
for index, prepared := range retained {
|
||||||
|
if prepared == nil {
|
||||||
|
retained[index] = retainHandle()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
runtime.GC()
|
||||||
|
var retainedMemory runtime.MemStats
|
||||||
|
runtime.ReadMemStats(&retainedMemory)
|
||||||
|
b.ReportMetric(float64(int64(retainedMemory.HeapAlloc)-int64(setupMemory.HeapAlloc)), "retained-B")
|
||||||
|
runtime.KeepAlive(runner)
|
||||||
|
runtime.KeepAlive(request)
|
||||||
|
runtime.KeepAlive(retained)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -15,16 +15,27 @@ type resolvedProfileSelection struct {
|
|||||||
profile *domain.ExecutionProfile
|
profile *domain.ExecutionProfile
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type profileIdentityRepository interface {
|
||||||
|
GetProfileIdentity(context.Context, string) (string, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *Runner) profileLookupID(profileID string) (string, error) {
|
||||||
|
if strings.TrimSpace(profileID) == "" {
|
||||||
|
return "", fmt.Errorf("%w: profile id is required", ErrInvalidRequest)
|
||||||
|
}
|
||||||
|
if r == nil || r.profiles == nil {
|
||||||
|
return "", fmt.Errorf("%w: profile repository is not configured", ErrProfileLoad)
|
||||||
|
}
|
||||||
|
return profileID, nil
|
||||||
|
}
|
||||||
|
|
||||||
func (r *Runner) resolveProfileSelection(
|
func (r *Runner) resolveProfileSelection(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
profileID string,
|
profileID string,
|
||||||
) (*resolvedProfileSelection, error) {
|
) (*resolvedProfileSelection, error) {
|
||||||
lookupID := profileID
|
lookupID, err := r.profileLookupID(profileID)
|
||||||
if strings.TrimSpace(lookupID) == "" {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("%w: profile id is required", ErrInvalidRequest)
|
return nil, err
|
||||||
}
|
|
||||||
if r == nil || r.profiles == nil {
|
|
||||||
return nil, fmt.Errorf("%w: profile repository is not configured", ErrProfileLoad)
|
|
||||||
}
|
}
|
||||||
selectedProfile, err := r.profiles.GetProfile(ctx, lookupID)
|
selectedProfile, err := r.profiles.GetProfile(ctx, lookupID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -45,6 +56,25 @@ func (r *Runner) resolveProfileSelection(
|
|||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (r *Runner) resolveProfileIdentity(ctx context.Context, profileID string) (string, error) {
|
||||||
|
lookupID, err := r.profileLookupID(profileID)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
if indexed, ok := r.profiles.(profileIdentityRepository); ok {
|
||||||
|
identity, err := indexed.GetProfileIdentity(ctx, lookupID)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("%w: %w", ErrProfileLoad, err)
|
||||||
|
}
|
||||||
|
return identity, nil
|
||||||
|
}
|
||||||
|
selection, err := r.resolveProfileSelection(ctx, lookupID)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
return selection.id, nil
|
||||||
|
}
|
||||||
|
|
||||||
func normalizeResolvedExecutionTarget(target domain.ExecutionTarget) (domain.ExecutionTarget, error) {
|
func normalizeResolvedExecutionTarget(target domain.ExecutionTarget) (domain.ExecutionTarget, error) {
|
||||||
endpoint, err := domain.NormalizeOpenAICompatibleBaseEndpoint(target.Endpoint)
|
endpoint, err := domain.NormalizeOpenAICompatibleBaseEndpoint(target.Endpoint)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -13,6 +13,10 @@ type resolvedPromptDefinition struct {
|
|||||||
hash string
|
hash string
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type promptInspectionRepository interface {
|
||||||
|
GetPromptInspection(context.Context, string, string) (*domain.PromptInspection, error)
|
||||||
|
}
|
||||||
|
|
||||||
func (r *Runner) resolvePromptDefinition(
|
func (r *Runner) resolvePromptDefinition(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
promptID string,
|
promptID string,
|
||||||
@@ -51,6 +55,43 @@ func (r *Runner) resolvePromptDefinition(
|
|||||||
return &resolvedPromptDefinition{definition: definition, hash: hash}, nil
|
return &resolvedPromptDefinition{definition: definition, hash: hash}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (r *Runner) resolvePromptInspection(
|
||||||
|
ctx context.Context,
|
||||||
|
promptID string,
|
||||||
|
promptVersion string,
|
||||||
|
) (*domain.PromptInspection, error) {
|
||||||
|
if strings.TrimSpace(promptID) == "" {
|
||||||
|
return nil, fmt.Errorf("%w: prompt id is required", ErrInvalidRequest)
|
||||||
|
}
|
||||||
|
if r == nil || r.promptDefs == nil {
|
||||||
|
return nil, fmt.Errorf("%w: prompt repository is not configured", ErrPromptLoad)
|
||||||
|
}
|
||||||
|
if indexed, ok := r.promptDefs.(promptInspectionRepository); ok {
|
||||||
|
inspection, err := indexed.GetPromptInspection(ctx, promptID, promptVersion)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("%w: %w", ErrPromptLoad, err)
|
||||||
|
}
|
||||||
|
if inspection == nil {
|
||||||
|
return nil, fmt.Errorf("%w: prompt repository returned nil inspection", ErrPromptLoad)
|
||||||
|
}
|
||||||
|
return inspection, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
selection, err := r.resolvePromptDefinition(ctx, promptID, promptVersion)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
definition := selection.definition
|
||||||
|
return &domain.PromptInspection{
|
||||||
|
PromptID: definition.ID,
|
||||||
|
PromptVersion: definition.Version,
|
||||||
|
PromptHash: selection.hash,
|
||||||
|
DefaultProfileID: definition.DefaultProfile,
|
||||||
|
Inputs: append([]domain.PromptInput(nil), definition.Inputs...),
|
||||||
|
OutputContract: definition.Validation,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
// InspectPrompt resolves one explicit prompt without execution work.
|
// InspectPrompt resolves one explicit prompt without execution work.
|
||||||
func (r *Runner) InspectPrompt(
|
func (r *Runner) InspectPrompt(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
@@ -66,19 +107,5 @@ func (r *Runner) InspectPrompt(
|
|||||||
default:
|
default:
|
||||||
}
|
}
|
||||||
|
|
||||||
selection, err := r.resolvePromptDefinition(ctx, promptID, promptVersion)
|
return r.resolvePromptInspection(ctx, promptID, promptVersion)
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
inputs := make([]domain.PromptInput, len(selection.definition.Inputs))
|
|
||||||
copy(inputs, selection.definition.Inputs)
|
|
||||||
|
|
||||||
return &domain.PromptInspection{
|
|
||||||
PromptID: selection.definition.ID,
|
|
||||||
PromptVersion: selection.definition.Version,
|
|
||||||
PromptHash: selection.hash,
|
|
||||||
DefaultProfileID: selection.definition.DefaultProfile,
|
|
||||||
Inputs: inputs,
|
|
||||||
OutputContract: selection.definition.Validation,
|
|
||||||
}, nil
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -18,6 +18,13 @@ type inspectionPromptRepository struct {
|
|||||||
version string
|
version string
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type promptMetadataRepository struct {
|
||||||
|
inspection domain.PromptInspection
|
||||||
|
inspectionErr error
|
||||||
|
lookupID string
|
||||||
|
lookupVersion string
|
||||||
|
}
|
||||||
|
|
||||||
func (r *inspectionPromptRepository) GetPromptDefinition(
|
func (r *inspectionPromptRepository) GetPromptDefinition(
|
||||||
_ context.Context,
|
_ context.Context,
|
||||||
id string,
|
id string,
|
||||||
@@ -32,6 +39,22 @@ func (r *inspectionPromptRepository) GetPromptDefinition(
|
|||||||
return r.definition, nil
|
return r.definition, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (r *promptMetadataRepository) GetPromptDefinition(context.Context, string, string) (*domain.PromptDefinition, error) {
|
||||||
|
return nil, errors.New("full prompt definition should not be requested")
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *promptMetadataRepository) GetPromptInspection(
|
||||||
|
_ context.Context,
|
||||||
|
id string,
|
||||||
|
version string,
|
||||||
|
) (*domain.PromptInspection, error) {
|
||||||
|
r.lookupID = id
|
||||||
|
r.lookupVersion = version
|
||||||
|
inspection := r.inspection
|
||||||
|
inspection.Inputs = append([]domain.PromptInput(nil), inspection.Inputs...)
|
||||||
|
return &inspection, r.inspectionErr
|
||||||
|
}
|
||||||
|
|
||||||
func TestRunnerInspectPromptResolvesOneDefinitionWithoutExecutionCollaborators(t *testing.T) {
|
func TestRunnerInspectPromptResolvesOneDefinitionWithoutExecutionCollaborators(t *testing.T) {
|
||||||
definition := &domain.PromptDefinition{
|
definition := &domain.PromptDefinition{
|
||||||
ID: "normalized.prompt",
|
ID: "normalized.prompt",
|
||||||
@@ -81,6 +104,42 @@ func TestRunnerInspectPromptResolvesOneDefinitionWithoutExecutionCollaborators(t
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRunnerInspectPromptUsesOwnedMetadataWhenRepositorySupportsIt(t *testing.T) {
|
||||||
|
repository := &promptMetadataRepository{inspection: domain.PromptInspection{
|
||||||
|
PromptID: "captured.prompt", PromptVersion: "v2", PromptHash: "cached-prompt-hash",
|
||||||
|
DefaultProfileID: "partial-profile",
|
||||||
|
Inputs: []domain.PromptInput{{Name: "document", Required: true, ContentType: "text/plain", Description: "Captured."}},
|
||||||
|
OutputContract: domain.OutputContract{Format: domain.FormatJSON, ValidationMode: domain.ValidationJSON},
|
||||||
|
}}
|
||||||
|
runner := &Runner{promptDefs: repository}
|
||||||
|
inspection, err := runner.InspectPrompt(context.Background(), " requested-id ", " requested-version ")
|
||||||
|
if err != nil || repository.lookupID != " requested-id " || repository.lookupVersion != " requested-version " ||
|
||||||
|
inspection.PromptID != "captured.prompt" || inspection.PromptVersion != "v2" ||
|
||||||
|
inspection.PromptHash != "cached-prompt-hash" || inspection.DefaultProfileID != "partial-profile" ||
|
||||||
|
!reflect.DeepEqual(inspection.Inputs, repository.inspection.Inputs) || inspection.OutputContract != repository.inspection.OutputContract {
|
||||||
|
t.Fatalf("metadata inspection=(%+v, %v), lookup=(%q,%q)", inspection, err, repository.lookupID, repository.lookupVersion)
|
||||||
|
}
|
||||||
|
inspection.Inputs[0].Name = "caller mutation"
|
||||||
|
inspection.OutputContract.SchemaPath = "caller mutation"
|
||||||
|
second, err := runner.InspectPrompt(context.Background(), "requested-id", "requested-version")
|
||||||
|
if err != nil || second.Inputs[0].Name != "document" || second.OutputContract.SchemaPath != "" {
|
||||||
|
t.Fatalf("metadata inspection was not caller-owned: (%+v, %v)", second, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunnerInspectPromptUsesCapturedHashWhenMetadataCapabilityIsAbsent(t *testing.T) {
|
||||||
|
repository := &requirementInspectionPromptRepository{
|
||||||
|
definition: &domain.PromptDefinition{
|
||||||
|
ID: "captured.prompt", Version: "v1", Inputs: []domain.PromptInput{{Name: "document"}},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
inspection, err := (&Runner{promptDefs: repository}).InspectPrompt(context.Background(), " requested-id ", " requested-version ")
|
||||||
|
if err != nil || inspection.PromptID != "captured.prompt" || inspection.PromptVersion != "v1" ||
|
||||||
|
inspection.PromptHash != "captured-hash" || inspection.Inputs[0].Name != "document" {
|
||||||
|
t.Fatalf("captured-hash fallback inspection=(%+v, %v)", inspection, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestRunnerInspectPromptClassifiesFailuresWithoutRepositoryWorkAfterCancellation(t *testing.T) {
|
func TestRunnerInspectPromptClassifiesFailuresWithoutRepositoryWorkAfterCancellation(t *testing.T) {
|
||||||
t.Run("blank ID", func(t *testing.T) {
|
t.Run("blank ID", func(t *testing.T) {
|
||||||
repository := &inspectionPromptRepository{}
|
repository := &inspectionPromptRepository{}
|
||||||
|
|||||||
80
internal/usecase/requirements_inspection.go
Normal file
80
internal/usecase/requirements_inspection.go
Normal file
@@ -0,0 +1,80 @@
|
|||||||
|
package usecase
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/prompt"
|
||||||
|
)
|
||||||
|
|
||||||
|
// RequirementInspectionRequest selects a captured prompt and supplies the
|
||||||
|
// context used for dependency classification.
|
||||||
|
type RequirementInspectionRequest struct {
|
||||||
|
PromptID string
|
||||||
|
PromptVersion string
|
||||||
|
ProfileID string
|
||||||
|
Vars map[string]string
|
||||||
|
SessionID string
|
||||||
|
}
|
||||||
|
|
||||||
|
type requirementSnapshotRepository interface {
|
||||||
|
GetRequirementSnapshot(id, version string) (prompt.RequirementSnapshot, bool)
|
||||||
|
}
|
||||||
|
|
||||||
|
// InspectRequirements discovers prompt inputs and caller variables without
|
||||||
|
// loading artifacts or performing execution preparation.
|
||||||
|
func (r *Runner) InspectRequirements(
|
||||||
|
ctx context.Context,
|
||||||
|
req RequirementInspectionRequest,
|
||||||
|
) (*domain.RequirementInspection, error) {
|
||||||
|
if strings.TrimSpace(req.PromptID) == "" {
|
||||||
|
return nil, fmt.Errorf("%w: prompt id is required", ErrInvalidRequest)
|
||||||
|
}
|
||||||
|
directSessionID, err := domain.NormalizeSessionID(req.SessionID)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("%w: session_id: %v", ErrInvalidRequest, err)
|
||||||
|
}
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, fmt.Errorf("%w: %w", ErrPromptLoad, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
selection, err := r.resolvePromptInspection(ctx, req.PromptID, req.PromptVersion)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, fmt.Errorf("%w: %w", ErrProfileLoad, err)
|
||||||
|
}
|
||||||
|
selectedProfileID := req.ProfileID
|
||||||
|
if strings.TrimSpace(selectedProfileID) == "" {
|
||||||
|
selectedProfileID = selection.DefaultProfileID
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(selectedProfileID) == "" {
|
||||||
|
return nil, fmt.Errorf("%w: %w: profile id is required either in request or prompt default_profile", ErrInvalidRequest, ErrProfileRequired)
|
||||||
|
}
|
||||||
|
profileIdentity, err := r.resolveProfileIdentity(ctx, selectedProfileID)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, fmt.Errorf("%w: %w", ErrPromptLoad, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
repository, ok := r.promptDefs.(requirementSnapshotRepository)
|
||||||
|
if !ok {
|
||||||
|
return nil, fmt.Errorf("%w: prompt repository does not provide captured requirement snapshots", ErrPromptLoad)
|
||||||
|
}
|
||||||
|
snapshot, ok := repository.GetRequirementSnapshot(selection.PromptID, selection.PromptVersion)
|
||||||
|
if !ok {
|
||||||
|
return nil, fmt.Errorf("%w: captured requirement snapshot is unavailable", ErrPromptLoad)
|
||||||
|
}
|
||||||
|
definition := &domain.PromptDefinition{ID: selection.PromptID, Version: selection.PromptVersion}
|
||||||
|
inspection, err := prompt.AnalyzeRequirements(ctx, definition, snapshot, req.Vars, directSessionID != "")
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("%w: %w", ErrPromptLoad, err)
|
||||||
|
}
|
||||||
|
inspection.ProfileID = profileIdentity
|
||||||
|
return &inspection, nil
|
||||||
|
}
|
||||||
372
internal/usecase/requirements_inspection_test.go
Normal file
372
internal/usecase/requirements_inspection_test.go
Normal file
@@ -0,0 +1,372 @@
|
|||||||
|
package usecase
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"testing"
|
||||||
|
"text/template"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/prompt"
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/promptdef"
|
||||||
|
)
|
||||||
|
|
||||||
|
type requirementInspectionPromptRepository struct {
|
||||||
|
definition *domain.PromptDefinition
|
||||||
|
snapshot prompt.RequirementSnapshot
|
||||||
|
lookupErr error
|
||||||
|
snapshotFound bool
|
||||||
|
lookupCalls int
|
||||||
|
snapshotCalls int
|
||||||
|
lookupID string
|
||||||
|
lookupVersion string
|
||||||
|
snapshotID string
|
||||||
|
snapshotVersion string
|
||||||
|
}
|
||||||
|
|
||||||
|
type requirementPromptMetadataRepository struct {
|
||||||
|
*requirementInspectionPromptRepository
|
||||||
|
inspection domain.PromptInspection
|
||||||
|
lookupID string
|
||||||
|
lookupVersion string
|
||||||
|
}
|
||||||
|
|
||||||
|
type inspectionProfileIdentityRepository struct {
|
||||||
|
*inspectionProfileRepository
|
||||||
|
identity string
|
||||||
|
identityErr error
|
||||||
|
identityID string
|
||||||
|
identityCalls int
|
||||||
|
}
|
||||||
|
|
||||||
|
func (repository *inspectionProfileIdentityRepository) GetProfileIdentity(
|
||||||
|
_ context.Context,
|
||||||
|
id string,
|
||||||
|
) (string, error) {
|
||||||
|
repository.identityCalls++
|
||||||
|
repository.identityID = id
|
||||||
|
return repository.identity, repository.identityErr
|
||||||
|
}
|
||||||
|
|
||||||
|
func (repository *requirementInspectionPromptRepository) GetPromptDefinition(
|
||||||
|
context.Context,
|
||||||
|
string,
|
||||||
|
string,
|
||||||
|
) (*domain.PromptDefinition, error) {
|
||||||
|
return repository.definition, repository.lookupErr
|
||||||
|
}
|
||||||
|
|
||||||
|
func (repository *requirementPromptMetadataRepository) GetPromptInspection(
|
||||||
|
_ context.Context,
|
||||||
|
id string,
|
||||||
|
version string,
|
||||||
|
) (*domain.PromptInspection, error) {
|
||||||
|
repository.lookupID = id
|
||||||
|
repository.lookupVersion = version
|
||||||
|
inspection := repository.inspection
|
||||||
|
inspection.Inputs = append([]domain.PromptInput(nil), inspection.Inputs...)
|
||||||
|
return &inspection, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (repository *requirementInspectionPromptRepository) GetPromptSnapshot(
|
||||||
|
_ context.Context,
|
||||||
|
id string,
|
||||||
|
version string,
|
||||||
|
) (*domain.PromptDefinition, string, error) {
|
||||||
|
repository.lookupCalls++
|
||||||
|
repository.lookupID = id
|
||||||
|
repository.lookupVersion = version
|
||||||
|
return repository.definition, "captured-hash", repository.lookupErr
|
||||||
|
}
|
||||||
|
|
||||||
|
func (repository *requirementInspectionPromptRepository) GetRequirementSnapshot(
|
||||||
|
id string,
|
||||||
|
version string,
|
||||||
|
) (prompt.RequirementSnapshot, bool) {
|
||||||
|
repository.snapshotCalls++
|
||||||
|
repository.snapshotID = id
|
||||||
|
repository.snapshotVersion = version
|
||||||
|
return repository.snapshot, repository.snapshotFound
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunnerInspectRequirementsUsesCapturedSelectionWithoutExecutionWork(t *testing.T) {
|
||||||
|
message, err := prompt.CompileTemplate("message", `{{input "message-input"}} {{.caller}}`)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
session, err := prompt.CompileTemplate("session", `{{input "session-only"}}`)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
definition := &domain.PromptDefinition{ID: "selected", Version: "v2", DefaultProfile: "default-profile"}
|
||||||
|
repository := &requirementInspectionPromptRepository{
|
||||||
|
definition: definition,
|
||||||
|
snapshotFound: true,
|
||||||
|
snapshot: prompt.RequirementSnapshot{Templates: prompt.CompiledTemplates{
|
||||||
|
Messages: []*template.Template{message}, Session: session,
|
||||||
|
}},
|
||||||
|
}
|
||||||
|
profiles := &inspectionProfileRepository{profile: &domain.ExecutionProfile{
|
||||||
|
ID: "partial-profile", ResolvedTarget: &domain.ExecutionTarget{Endpoint: ""},
|
||||||
|
}}
|
||||||
|
runner := &Runner{promptDefs: repository, profiles: profiles}
|
||||||
|
inspection, err := runner.InspectRequirements(context.Background(), RequirementInspectionRequest{
|
||||||
|
PromptID: " requested ", PromptVersion: " exact-version ", ProfileID: "partial-profile",
|
||||||
|
Vars: map[string]string{"caller": ""}, SessionID: " direct-session ",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("inspect requirements: %v", err)
|
||||||
|
}
|
||||||
|
if repository.lookupCalls != 1 || repository.lookupID != " requested " || repository.lookupVersion != " exact-version " ||
|
||||||
|
repository.snapshotCalls != 1 || repository.snapshotID != definition.ID || repository.snapshotVersion != definition.Version {
|
||||||
|
t.Fatalf("selection lookup=(%d,%q,%q), snapshot=(%d,%q,%q)", repository.lookupCalls, repository.lookupID,
|
||||||
|
repository.lookupVersion, repository.snapshotCalls, repository.snapshotID, repository.snapshotVersion)
|
||||||
|
}
|
||||||
|
if profiles.calls != 1 || profiles.id != "partial-profile" || inspection.PromptID != definition.ID ||
|
||||||
|
inspection.PromptVersion != definition.Version || inspection.ProfileID != "partial-profile" || !inspection.Complete ||
|
||||||
|
len(inspection.Inputs) != 1 || inspection.Inputs[0].Name != "message-input" || len(inspection.Variables) != 1 ||
|
||||||
|
inspection.Variables[0].Name != "caller" || !inspection.Variables[0].Supplied {
|
||||||
|
t.Fatalf("unexpected requirement inspection: %+v", inspection)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunnerInspectRequirementsUsesPromptMetadataAndExactRequirementSnapshot(t *testing.T) {
|
||||||
|
message, err := prompt.CompileTemplate("message", `{{input "message-input"}} {{.caller}}`)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
session, err := prompt.CompileTemplate("session", `{{input "session-only"}}`)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
baseRepository := &requirementInspectionPromptRepository{
|
||||||
|
snapshotFound: true,
|
||||||
|
snapshot: prompt.RequirementSnapshot{Templates: prompt.CompiledTemplates{
|
||||||
|
Messages: []*template.Template{message}, Session: session,
|
||||||
|
}},
|
||||||
|
}
|
||||||
|
promptRepository := &requirementPromptMetadataRepository{
|
||||||
|
requirementInspectionPromptRepository: baseRepository,
|
||||||
|
inspection: domain.PromptInspection{
|
||||||
|
PromptID: "captured-id", PromptVersion: "captured-version", PromptHash: "captured-hash",
|
||||||
|
DefaultProfileID: "default-profile",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
profiles := &inspectionProfileIdentityRepository{
|
||||||
|
inspectionProfileRepository: &inspectionProfileRepository{},
|
||||||
|
identity: "default-profile",
|
||||||
|
}
|
||||||
|
inspection, err := (&Runner{promptDefs: promptRepository, profiles: profiles}).InspectRequirements(
|
||||||
|
context.Background(), RequirementInspectionRequest{
|
||||||
|
PromptID: " requested-id ", PromptVersion: " requested-version ",
|
||||||
|
Vars: map[string]string{"caller": "Ada"}, SessionID: " direct-session ",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
if err != nil || promptRepository.lookupID != " requested-id " || promptRepository.lookupVersion != " requested-version " ||
|
||||||
|
baseRepository.snapshotID != "captured-id" || baseRepository.snapshotVersion != "captured-version" ||
|
||||||
|
profiles.identityID != "default-profile" || inspection.PromptID != "captured-id" ||
|
||||||
|
inspection.PromptVersion != "captured-version" || inspection.ProfileID != "default-profile" ||
|
||||||
|
!inspection.Complete || len(inspection.Inputs) != 1 || inspection.Inputs[0].Name != "message-input" ||
|
||||||
|
len(inspection.Variables) != 1 || inspection.Variables[0].Name != "caller" || !inspection.Variables[0].Supplied {
|
||||||
|
t.Fatalf("metadata-based requirement inspection=(%+v, %v), prompt lookup=(%q,%q), snapshot=(%q,%q), profile=%q",
|
||||||
|
inspection, err, promptRepository.lookupID, promptRepository.lookupVersion,
|
||||||
|
baseRepository.snapshotID, baseRepository.snapshotVersion, profiles.identityID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunnerInspectRequirementsUsesProfileIdentityCapabilityOrFallback(t *testing.T) {
|
||||||
|
message, err := prompt.CompileTemplate("message", `{{.name}}`)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
definition := &domain.PromptDefinition{ID: "selected", Version: "v1"}
|
||||||
|
makePromptRepository := func() *requirementInspectionPromptRepository {
|
||||||
|
return &requirementInspectionPromptRepository{
|
||||||
|
definition: definition, snapshotFound: true,
|
||||||
|
snapshot: prompt.RequirementSnapshot{Templates: prompt.CompiledTemplates{Messages: []*template.Template{message}}},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
requestedID := " literal-profile "
|
||||||
|
|
||||||
|
t.Run("optional capability", func(t *testing.T) {
|
||||||
|
profiles := &inspectionProfileIdentityRepository{
|
||||||
|
inspectionProfileRepository: &inspectionProfileRepository{profile: &domain.ExecutionProfile{
|
||||||
|
ID: "repository-profile", ResolvedTarget: &domain.ExecutionTarget{},
|
||||||
|
}},
|
||||||
|
identity: requestedID,
|
||||||
|
}
|
||||||
|
inspection, err := (&Runner{promptDefs: makePromptRepository(), profiles: profiles}).InspectRequirements(
|
||||||
|
context.Background(), RequirementInspectionRequest{
|
||||||
|
PromptID: "selected", ProfileID: requestedID, Vars: map[string]string{"name": "Ada"},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
if err != nil || inspection.ProfileID != requestedID || profiles.identityCalls != 1 ||
|
||||||
|
profiles.identityID != requestedID || profiles.calls != 0 {
|
||||||
|
t.Fatalf("identity capability inspection=(%+v, %v), capability calls=%d id=%q full lookups=%d",
|
||||||
|
inspection, err, profiles.identityCalls, profiles.identityID, profiles.calls)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("full-profile fallback", func(t *testing.T) {
|
||||||
|
profiles := &inspectionProfileRepository{profile: &domain.ExecutionProfile{
|
||||||
|
ID: "repository-profile", ResolvedTarget: &domain.ExecutionTarget{},
|
||||||
|
}}
|
||||||
|
inspection, err := (&Runner{promptDefs: makePromptRepository(), profiles: profiles}).InspectRequirements(
|
||||||
|
context.Background(), RequirementInspectionRequest{
|
||||||
|
PromptID: "selected", ProfileID: requestedID, Vars: map[string]string{"name": "Ada"},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
if err != nil || inspection.ProfileID != requestedID || profiles.calls != 1 || profiles.id != requestedID {
|
||||||
|
t.Fatalf("fallback inspection=(%+v, %v), full lookup=(%d,%q)", inspection, err, profiles.calls, profiles.id)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("capability failure", func(t *testing.T) {
|
||||||
|
lookupErr := errors.New("profile identity lookup failed")
|
||||||
|
profiles := &inspectionProfileIdentityRepository{
|
||||||
|
inspectionProfileRepository: &inspectionProfileRepository{},
|
||||||
|
identityErr: lookupErr,
|
||||||
|
}
|
||||||
|
_, err := (&Runner{promptDefs: makePromptRepository(), profiles: profiles}).InspectRequirements(
|
||||||
|
context.Background(), RequirementInspectionRequest{PromptID: "selected", ProfileID: requestedID},
|
||||||
|
)
|
||||||
|
if !errors.Is(err, ErrProfileLoad) || !errors.Is(err, lookupErr) || profiles.identityCalls != 1 || profiles.calls != 0 {
|
||||||
|
t.Fatalf("capability failure=(%v), capability calls=%d full lookups=%d", err, profiles.identityCalls, profiles.calls)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunnerInspectRequirementsUsesPromptDefaultAndIncludesSessionWhenNotBypassed(t *testing.T) {
|
||||||
|
message, err := prompt.CompileTemplate("message", `{{input "message-input"}}`)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
session, err := prompt.CompileTemplate("session", `{{input "session-input"}}`)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
definition := &domain.PromptDefinition{ID: "selected", Version: "v1", DefaultProfile: "default-profile"}
|
||||||
|
repository := &requirementInspectionPromptRepository{
|
||||||
|
definition: definition, snapshotFound: true,
|
||||||
|
snapshot: prompt.RequirementSnapshot{Templates: prompt.CompiledTemplates{Messages: []*template.Template{message}, Session: session}},
|
||||||
|
}
|
||||||
|
profiles := &inspectionProfileRepository{profile: &domain.ExecutionProfile{
|
||||||
|
ID: "default-profile", ResolvedTarget: &domain.ExecutionTarget{},
|
||||||
|
}}
|
||||||
|
inspection, err := (&Runner{promptDefs: repository, profiles: profiles}).InspectRequirements(
|
||||||
|
context.Background(), RequirementInspectionRequest{PromptID: "selected"},
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("inspect requirements: %v", err)
|
||||||
|
}
|
||||||
|
if profiles.id != "default-profile" || inspection.ProfileID != "default-profile" || len(inspection.Inputs) != 2 ||
|
||||||
|
inspection.Inputs[0].Name != "message-input" || inspection.Inputs[1].Name != "session-input" {
|
||||||
|
t.Fatalf("prompt-default profile or session template was not selected: %+v", inspection)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunnerInspectRequirementsReturnsIncompleteDiscoveryAsSuccess(t *testing.T) {
|
||||||
|
compiled, err := prompt.CompileTemplate("message", `{{if .missing}}{{input "left"}}{{else}}{{input "right"}}{{end}}`)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
repository := &requirementInspectionPromptRepository{
|
||||||
|
definition: &domain.PromptDefinition{ID: "selected", Version: "v1", DefaultProfile: "profile"},
|
||||||
|
snapshotFound: true,
|
||||||
|
snapshot: prompt.RequirementSnapshot{Templates: prompt.CompiledTemplates{Messages: []*template.Template{compiled}}},
|
||||||
|
}
|
||||||
|
profiles := &inspectionProfileRepository{profile: &domain.ExecutionProfile{ID: "profile", ResolvedTarget: &domain.ExecutionTarget{}}}
|
||||||
|
inspection, err := (&Runner{promptDefs: repository, profiles: profiles}).InspectRequirements(
|
||||||
|
context.Background(), RequirementInspectionRequest{PromptID: "selected"},
|
||||||
|
)
|
||||||
|
if err != nil || inspection == nil || inspection.Complete || len(inspection.Issues) != 1 ||
|
||||||
|
inspection.Issues[0].Code != domain.RequirementIssueMissingVariable {
|
||||||
|
t.Fatalf("incomplete discovery returned (%+v, %v), want successful partial result", inspection, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunnerInspectRequirementsClassifiesSelectionAndSnapshotFailures(t *testing.T) {
|
||||||
|
t.Run("blank prompt", func(t *testing.T) {
|
||||||
|
repository := &requirementInspectionPromptRepository{}
|
||||||
|
_, err := (&Runner{promptDefs: repository}).InspectRequirements(context.Background(), RequirementInspectionRequest{PromptID: " \t "})
|
||||||
|
if !errors.Is(err, ErrInvalidRequest) || repository.lookupCalls != 0 {
|
||||||
|
t.Fatalf("blank prompt error=%v lookups=%d", err, repository.lookupCalls)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("invalid session", func(t *testing.T) {
|
||||||
|
repository := &requirementInspectionPromptRepository{}
|
||||||
|
_, err := (&Runner{promptDefs: repository}).InspectRequirements(context.Background(), RequirementInspectionRequest{
|
||||||
|
PromptID: "selected", SessionID: string([]byte{0xff}),
|
||||||
|
})
|
||||||
|
if !errors.Is(err, ErrInvalidRequest) || repository.lookupCalls != 0 {
|
||||||
|
t.Fatalf("invalid session error=%v lookups=%d", err, repository.lookupCalls)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("cancellation before lookup", func(t *testing.T) {
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
cancel()
|
||||||
|
repository := &requirementInspectionPromptRepository{}
|
||||||
|
_, err := (&Runner{promptDefs: repository}).InspectRequirements(ctx, RequirementInspectionRequest{PromptID: "selected"})
|
||||||
|
if !errors.Is(err, ErrPromptLoad) || !errors.Is(err, context.Canceled) || repository.lookupCalls != 0 {
|
||||||
|
t.Fatalf("canceled inspection=%v lookups=%d", err, repository.lookupCalls)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("no profile selection", func(t *testing.T) {
|
||||||
|
repository := &requirementInspectionPromptRepository{definition: &domain.PromptDefinition{ID: "selected"}}
|
||||||
|
_, err := (&Runner{promptDefs: repository}).InspectRequirements(context.Background(), RequirementInspectionRequest{PromptID: "selected"})
|
||||||
|
if !errors.Is(err, ErrInvalidRequest) || !errors.Is(err, ErrProfileRequired) {
|
||||||
|
t.Fatalf("missing profile error=%v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("ambiguous prompt selection", func(t *testing.T) {
|
||||||
|
ambiguous := errors.New("ambiguous omitted version")
|
||||||
|
repository := &requirementInspectionPromptRepository{lookupErr: ambiguous}
|
||||||
|
_, err := (&Runner{promptDefs: repository}).InspectRequirements(context.Background(), RequirementInspectionRequest{PromptID: "selected"})
|
||||||
|
if !errors.Is(err, ErrPromptLoad) || !errors.Is(err, ambiguous) || repository.lookupVersion != "" {
|
||||||
|
t.Fatalf("ambiguous selection error=%v version=%q", err, repository.lookupVersion)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("captured snapshot unavailable", func(t *testing.T) {
|
||||||
|
repository := &requirementInspectionPromptRepository{definition: &domain.PromptDefinition{ID: "selected", DefaultProfile: "profile"}}
|
||||||
|
profiles := &inspectionProfileRepository{profile: &domain.ExecutionProfile{ID: "profile", ResolvedTarget: &domain.ExecutionTarget{}}}
|
||||||
|
_, err := (&Runner{promptDefs: repository, profiles: profiles}).InspectRequirements(context.Background(), RequirementInspectionRequest{PromptID: "selected"})
|
||||||
|
if !errors.Is(err, ErrPromptLoad) || repository.snapshotCalls != 1 {
|
||||||
|
t.Fatalf("missing snapshot error=%v calls=%d", err, repository.snapshotCalls)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("repository lacks snapshot capability", func(t *testing.T) {
|
||||||
|
_, err := (&Runner{promptDefs: &inspectionPromptRepository{definition: &domain.PromptDefinition{ID: "selected", DefaultProfile: "profile"}},
|
||||||
|
profiles: &inspectionProfileRepository{profile: &domain.ExecutionProfile{ID: "profile", ResolvedTarget: &domain.ExecutionTarget{}}},
|
||||||
|
}).InspectRequirements(context.Background(), RequirementInspectionRequest{PromptID: "selected"})
|
||||||
|
if !errors.Is(err, ErrPromptLoad) {
|
||||||
|
t.Fatalf("missing capability error=%v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
if _, err := (*Runner)(nil).InspectRequirements(context.Background(), RequirementInspectionRequest{PromptID: "selected"}); !errors.Is(err, ErrPromptLoad) {
|
||||||
|
t.Fatalf("nil runner error=%v", err)
|
||||||
|
}
|
||||||
|
if _, err := (&Runner{}).InspectRequirements(context.Background(), RequirementInspectionRequest{PromptID: "selected"}); !errors.Is(err, ErrPromptLoad) {
|
||||||
|
t.Fatalf("nil repository error=%v", err)
|
||||||
|
}
|
||||||
|
if _, err := (&Runner{promptDefs: &requirementInspectionPromptRepository{definition: &domain.PromptDefinition{ID: "selected"}}}).InspectRequirements(
|
||||||
|
context.Background(), RequirementInspectionRequest{PromptID: "selected", ProfileID: "explicit"},
|
||||||
|
); !errors.Is(err, ErrProfileLoad) {
|
||||||
|
t.Fatalf("missing profile error=%v", err)
|
||||||
|
}
|
||||||
|
if _, err := (&Runner{promptDefs: &requirementInspectionPromptRepository{definition: &domain.PromptDefinition{ID: "selected", DefaultProfile: "profile"}},
|
||||||
|
profiles: &inspectionProfileRepository{profile: &domain.ExecutionProfile{ID: "profile"}},
|
||||||
|
}).InspectRequirements(context.Background(), RequirementInspectionRequest{PromptID: "selected"}); !errors.Is(err, ErrProfileLoad) {
|
||||||
|
t.Fatalf("unresolved profile error=%v", err)
|
||||||
|
}
|
||||||
|
if _, err := (&Runner{promptDefs: &requirementInspectionPromptRepository{lookupErr: promptdef.ErrPromptDefinitionNotFound}}).InspectRequirements(
|
||||||
|
context.Background(), RequirementInspectionRequest{PromptID: "missing"},
|
||||||
|
); !errors.Is(err, ErrPromptLoad) || !errors.Is(err, promptdef.ErrPromptDefinitionNotFound) {
|
||||||
|
t.Fatalf("missing prompt error=%v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -188,7 +188,7 @@ func validateArtifact(ctx context.Context, artifact *domain.Artifact, contract d
|
|||||||
res.IsValid = true
|
res.IsValid = true
|
||||||
return res, nil
|
return res, nil
|
||||||
case domain.ValidationBasic:
|
case domain.ValidationBasic:
|
||||||
empty := strings.TrimSpace(string(artifact.Body)) == ""
|
empty := len(bytes.TrimSpace(artifact.Body)) == 0
|
||||||
if err := ctx.Err(); err != nil {
|
if err := ctx.Err(); err != nil {
|
||||||
return domain.ValidationResult{}, err
|
return domain.ValidationResult{}, err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
package validate
|
package validate
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"net/url"
|
"net/url"
|
||||||
@@ -35,32 +36,51 @@ func TestStandardValidatorNoneSkipped(t *testing.T) {
|
|||||||
|
|
||||||
func TestStandardValidatorBasicSuccess(t *testing.T) {
|
func TestStandardValidatorBasicSuccess(t *testing.T) {
|
||||||
v := NewStandardValidator("")
|
v := NewStandardValidator("")
|
||||||
|
for _, tc := range []struct {
|
||||||
res, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte("hello")}, domain.OutputContract{
|
name string
|
||||||
ValidationMode: domain.ValidationBasic,
|
body []byte
|
||||||
})
|
}{
|
||||||
if err != nil {
|
{name: "text", body: []byte("hello")},
|
||||||
t.Fatalf("expected no error, got %v", err)
|
{name: "invalid UTF-8 byte", body: []byte{0xff}},
|
||||||
}
|
} {
|
||||||
if res.Status != domain.ValidationPassed || !res.IsValid {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
t.Fatalf("expected passed/valid, got status=%q valid=%v", res.Status, res.IsValid)
|
res, err := v.Validate(context.Background(), &domain.Artifact{Body: tc.body}, domain.OutputContract{
|
||||||
|
ValidationMode: domain.ValidationBasic,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected no error, got %v", err)
|
||||||
|
}
|
||||||
|
if res.Status != domain.ValidationPassed || !res.IsValid {
|
||||||
|
t.Fatalf("expected passed/valid, got status=%q valid=%v", res.Status, res.IsValid)
|
||||||
|
}
|
||||||
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestStandardValidatorBasicFailureEmpty(t *testing.T) {
|
func TestStandardValidatorBasicFailureEmpty(t *testing.T) {
|
||||||
v := NewStandardValidator("")
|
v := NewStandardValidator("")
|
||||||
|
for _, tc := range []struct {
|
||||||
res, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte(" \n\t ")}, domain.OutputContract{
|
name string
|
||||||
ValidationMode: domain.ValidationBasic,
|
body []byte
|
||||||
})
|
}{
|
||||||
if err != nil {
|
{name: "empty bytes", body: []byte{}},
|
||||||
t.Fatalf("expected no error, got %v", err)
|
{name: "ASCII whitespace", body: []byte(" \n\t ")},
|
||||||
}
|
{name: "Unicode whitespace", body: []byte("\u2003\u00a0\u3000")},
|
||||||
if res.Status != domain.ValidationFailed || res.IsValid {
|
} {
|
||||||
t.Fatalf("expected failed/invalid, got status=%q valid=%v", res.Status, res.IsValid)
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
}
|
res, err := v.Validate(context.Background(), &domain.Artifact{Body: tc.body}, domain.OutputContract{
|
||||||
if len(res.Errors) == 0 {
|
ValidationMode: domain.ValidationBasic,
|
||||||
t.Fatal("expected validation errors")
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected no error, got %v", err)
|
||||||
|
}
|
||||||
|
if res.Status != domain.ValidationFailed || res.IsValid {
|
||||||
|
t.Fatalf("expected failed/invalid, got status=%q valid=%v", res.Status, res.IsValid)
|
||||||
|
}
|
||||||
|
if len(res.Errors) == 0 {
|
||||||
|
t.Fatal("expected validation errors")
|
||||||
|
}
|
||||||
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -917,6 +937,49 @@ func BenchmarkJSONValidation(b *testing.B) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
var basicValidationBenchmarkResult domain.ValidationResult
|
||||||
|
|
||||||
|
func BenchmarkBasicValidation(b *testing.B) {
|
||||||
|
largeText := bytes.Repeat([]byte("x"), 1<<20)
|
||||||
|
largeWhitespace := bytes.Repeat([]byte(" \t\n"), (1<<20)/3)
|
||||||
|
unicodeWhitespace := []byte("\u2003\u00a0\u3000")
|
||||||
|
benchmarks := []struct {
|
||||||
|
name string
|
||||||
|
body []byte
|
||||||
|
wantValid bool
|
||||||
|
}{
|
||||||
|
{name: "short_text", body: []byte("A concise response."), wantValid: true},
|
||||||
|
{name: "1MiB_nonempty_text", body: largeText, wantValid: true},
|
||||||
|
{name: "whitespace_only_text", body: largeWhitespace},
|
||||||
|
{name: "unicode_whitespace", body: unicodeWhitespace},
|
||||||
|
}
|
||||||
|
validator := NewStandardValidator("")
|
||||||
|
contract := domain.OutputContract{ValidationMode: domain.ValidationBasic}
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
for _, benchmark := range benchmarks {
|
||||||
|
b.Run(benchmark.name, func(b *testing.B) {
|
||||||
|
artifact := &domain.Artifact{Body: benchmark.body}
|
||||||
|
result, err := validator.Validate(ctx, artifact, contract)
|
||||||
|
if err != nil || result.IsValid != benchmark.wantValid {
|
||||||
|
b.Fatalf("validate basic output: result=%+v error=%v", result, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
b.ReportAllocs()
|
||||||
|
// Nonempty input can be classified from its boundary bytes without
|
||||||
|
// scanning the body, so report latency rather than bytes per second.
|
||||||
|
b.ResetTimer()
|
||||||
|
for range b.N {
|
||||||
|
result, err := validator.Validate(ctx, artifact, contract)
|
||||||
|
if err != nil || result.IsValid != benchmark.wantValid {
|
||||||
|
b.Fatalf("validate basic output: result=%+v error=%v", result, err)
|
||||||
|
}
|
||||||
|
basicValidationBenchmarkResult = result
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestJSONSchemaDialectIsDraft2020(t *testing.T) {
|
func TestJSONSchemaDialectIsDraft2020(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
|
|||||||
67
listing.go
Normal file
67
listing.go
Normal file
@@ -0,0 +1,67 @@
|
|||||||
|
package promptkit
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
)
|
||||||
|
|
||||||
|
// PromptIdentity identifies one prompt ID and exact version in the effective
|
||||||
|
// configuration captured when an Engine is constructed. It has no stable JSON
|
||||||
|
// representation.
|
||||||
|
type PromptIdentity struct {
|
||||||
|
ID string
|
||||||
|
Version string
|
||||||
|
}
|
||||||
|
|
||||||
|
// ListProfiles returns profile IDs in the effective configuration captured
|
||||||
|
// when the engine was constructed. Maintained and consumer profiles are
|
||||||
|
// included; an inherited profile retains its own ID. IDs are preserved exactly
|
||||||
|
// and sorted lexically. The returned slice is caller-owned and safe to mutate
|
||||||
|
// while the engine is used concurrently. Empty inventories succeed.
|
||||||
|
//
|
||||||
|
// Listing reports membership, not execution readiness. It does not reload
|
||||||
|
// configuration or perform resource, credential, provider, or execution work.
|
||||||
|
// Profile IDs are application data and may be sensitive. A nil or zero-value
|
||||||
|
// engine returns an error matching ErrInvalidConfig. Cancellation or deadline
|
||||||
|
// expiration returns the context error and no partial result.
|
||||||
|
func (e *Engine) ListProfiles(ctx context.Context) ([]string, error) {
|
||||||
|
if e == nil || e.internal == nil {
|
||||||
|
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
|
||||||
|
}
|
||||||
|
|
||||||
|
profiles, err := e.internal.ListProfiles(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, mapPublicError(err)
|
||||||
|
}
|
||||||
|
return profiles, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ListPrompts returns every prompt ID and exact version in the effective
|
||||||
|
// configuration captured when the engine was constructed, sorted lexically by
|
||||||
|
// ID and then version. Versions are not interpreted or prioritized. The
|
||||||
|
// returned slice is caller-owned and safe to mutate while the engine is used
|
||||||
|
// concurrently. Empty inventories succeed.
|
||||||
|
//
|
||||||
|
// Listing reports membership, not execution readiness. It does not reload
|
||||||
|
// configuration or perform resource, credential, provider, or execution work.
|
||||||
|
// Prompt identities are application data and may be sensitive. A nil or
|
||||||
|
// zero-value engine returns an error matching ErrInvalidConfig. Cancellation
|
||||||
|
// or deadline expiration returns the context error and no partial result.
|
||||||
|
func (e *Engine) ListPrompts(ctx context.Context) ([]PromptIdentity, error) {
|
||||||
|
if e == nil || e.internal == nil {
|
||||||
|
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
|
||||||
|
}
|
||||||
|
|
||||||
|
identities, err := e.internal.ListPrompts(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, mapPublicError(err)
|
||||||
|
}
|
||||||
|
public := make([]PromptIdentity, len(identities))
|
||||||
|
for i, identity := range identities {
|
||||||
|
public[i] = PromptIdentity{ID: identity.ID, Version: identity.Version}
|
||||||
|
}
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, mapPublicError(err)
|
||||||
|
}
|
||||||
|
return public, nil
|
||||||
|
}
|
||||||
379
listing_contract_test.go
Normal file
379
listing_contract_test.go
Normal file
@@ -0,0 +1,379 @@
|
|||||||
|
package promptkit_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"reflect"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
"testing/fstest"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/ConfigKit"
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestListIdentitiesBasicContract(t *testing.T) {
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
|
||||||
|
Source: configkit.Mapping("listing", map[string]any{
|
||||||
|
"profiles": map[string]any{"configured-profile": map[string]any{"model": "example-model"}},
|
||||||
|
"prompts": map[string]any{"example-prompt": map[string]any{
|
||||||
|
"2": listingContractPrompt(),
|
||||||
|
"10": listingContractPrompt(),
|
||||||
|
}},
|
||||||
|
}),
|
||||||
|
}}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
profiles, err := engine.ListProfiles(context.Background())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !containsListingProfile(profiles, "configured-profile") {
|
||||||
|
t.Fatalf("profile listing %#v does not include configured profile", profiles)
|
||||||
|
}
|
||||||
|
|
||||||
|
prompts, err := engine.ListPrompts(context.Background())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
want := []promptkit.PromptIdentity{
|
||||||
|
{ID: "example-prompt", Version: "10"},
|
||||||
|
{ID: "example-prompt", Version: "2"},
|
||||||
|
}
|
||||||
|
if len(prompts) != len(want) {
|
||||||
|
t.Fatalf("prompt listing = %#v, want %#v", prompts, want)
|
||||||
|
}
|
||||||
|
for i := range want {
|
||||||
|
if prompts[i] != want[i] {
|
||||||
|
t.Fatalf("prompt listing = %#v, want %#v", prompts, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
profile, err := engine.InspectProfile(context.Background(), "configured-profile")
|
||||||
|
if err != nil || profile == nil || profile.ProfileID != "configured-profile" {
|
||||||
|
t.Fatalf("listed profile could not be inspected: (%+v, %v)", profile, err)
|
||||||
|
}
|
||||||
|
prompt, err := engine.InspectPrompt(context.Background(), prompts[0].ID, prompts[0].Version)
|
||||||
|
if err != nil || prompt == nil || prompt.PromptID != prompts[0].ID || prompt.PromptVersion != prompts[0].Version {
|
||||||
|
t.Fatalf("listed prompt could not be inspected: (%+v, %v)", prompt, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestListIdentitiesInvalidEngine(t *testing.T) {
|
||||||
|
var nilEngine *promptkit.Engine
|
||||||
|
var zeroEngine promptkit.Engine
|
||||||
|
for _, engine := range []*promptkit.Engine{nilEngine, &zeroEngine} {
|
||||||
|
if _, err := engine.ListProfiles(context.Background()); !errors.Is(err, promptkit.ErrInvalidConfig) {
|
||||||
|
t.Errorf("ListProfiles on invalid engine: %v", err)
|
||||||
|
}
|
||||||
|
if _, err := engine.ListPrompts(context.Background()); !errors.Is(err, promptkit.ErrInvalidConfig) {
|
||||||
|
t.Errorf("ListPrompts on invalid engine: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestListIdentitiesPreserveContextErrorsWithoutPartialResults(t *testing.T) {
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
|
||||||
|
Source: configkit.Mapping("listing", map[string]any{
|
||||||
|
"profiles": map[string]any{"profile": map[string]any{}},
|
||||||
|
"prompts": map[string]any{"example": map[string]any{"v1": listingContractPrompt()}},
|
||||||
|
}),
|
||||||
|
}}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
canceled, cancel := context.WithCancel(context.Background())
|
||||||
|
cancel()
|
||||||
|
expired, expire := context.WithDeadline(context.Background(), time.Now().Add(-time.Second))
|
||||||
|
defer expire()
|
||||||
|
for _, test := range []struct {
|
||||||
|
name string
|
||||||
|
ctx context.Context
|
||||||
|
want error
|
||||||
|
}{
|
||||||
|
{name: "canceled", ctx: canceled, want: context.Canceled},
|
||||||
|
{name: "expired", ctx: expired, want: context.DeadlineExceeded},
|
||||||
|
} {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
profiles, err := engine.ListProfiles(test.ctx)
|
||||||
|
if !errors.Is(err, test.want) || len(profiles) != 0 {
|
||||||
|
t.Errorf("ListProfiles = %#v, %v; want no results and %v", profiles, err, test.want)
|
||||||
|
}
|
||||||
|
prompts, err := engine.ListPrompts(test.ctx)
|
||||||
|
if !errors.Is(err, test.want) || len(prompts) != 0 {
|
||||||
|
t.Errorf("ListPrompts = %#v, %v; want no results and %v", prompts, err, test.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestListIdentitiesUseEffectiveLayeredMembership(t *testing.T) {
|
||||||
|
underlay := configkit.Input{Source: configkit.Mapping("consumer defaults", map[string]any{
|
||||||
|
"profiles": map[string]any{
|
||||||
|
"base/profile": map[string]any{"endpoint": "https://example.test/v1", "model": "base-model"},
|
||||||
|
"child/profile": map[string]any{"extends": "base/profile"},
|
||||||
|
"overlay/profile": map[string]any{"endpoint": "https://example.test/v1", "model": "old-model"},
|
||||||
|
},
|
||||||
|
"prompts": map[string]any{
|
||||||
|
"shared/prompt": map[string]any{
|
||||||
|
"v1": listingContractPrompt(),
|
||||||
|
"v2": listingContractPrompt(),
|
||||||
|
},
|
||||||
|
"underlay-only": map[string]any{"v1": listingContractPrompt()},
|
||||||
|
},
|
||||||
|
})}
|
||||||
|
main := configkit.Input{Source: configkit.Mapping("application overlay", map[string]any{
|
||||||
|
"profiles": map[string]any{
|
||||||
|
"overlay/profile": map[string]any{"model": "effective-model"},
|
||||||
|
},
|
||||||
|
"prompts": map[string]any{
|
||||||
|
"shared/prompt": map[string]any{"v1": listingContractPrompt()},
|
||||||
|
"main-only": map[string]any{"v3": listingContractPrompt()},
|
||||||
|
},
|
||||||
|
})}
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||||
|
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{underlay}}},
|
||||||
|
Main: &main,
|
||||||
|
}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
profiles, err := engine.ListProfiles(context.Background())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, id := range []string{
|
||||||
|
"base/profile", "child/profile", "overlay/profile",
|
||||||
|
"promptkit-openrouter-standard", "promptkit-openrouter-auto",
|
||||||
|
} {
|
||||||
|
if !containsListingProfile(profiles, id) {
|
||||||
|
t.Errorf("profile listing %#v does not include %q", profiles, id)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, id := range []string{"base/profile", "child/profile", "overlay/profile"} {
|
||||||
|
if got := countListingProfile(profiles, id); got != 1 {
|
||||||
|
t.Errorf("profile %q appears %d times, want 1", id, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
overlay, err := engine.InspectProfile(context.Background(), "overlay/profile")
|
||||||
|
if err != nil || overlay == nil || overlay.EffectiveModelParams.Model != "effective-model" {
|
||||||
|
t.Fatalf("overlay profile did not resolve to its effective definition: (%+v, %v)", overlay, err)
|
||||||
|
}
|
||||||
|
child, err := engine.InspectProfile(context.Background(), "child/profile")
|
||||||
|
if err != nil || child == nil || child.EffectiveModelParams.Model != "base-model" {
|
||||||
|
t.Fatalf("inherited child profile did not resolve its effective definition: (%+v, %v)", child, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
prompts, err := engine.ListPrompts(context.Background())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
wantPrompts := []promptkit.PromptIdentity{
|
||||||
|
{ID: "main-only", Version: "v3"},
|
||||||
|
{ID: "shared/prompt", Version: "v1"},
|
||||||
|
{ID: "shared/prompt", Version: "v2"},
|
||||||
|
{ID: "underlay-only", Version: "v1"},
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(prompts, wantPrompts) {
|
||||||
|
t.Fatalf("effective prompt identities = %#v, want %#v", prompts, wantPrompts)
|
||||||
|
}
|
||||||
|
if got := countListingPrompt(prompts, "shared/prompt", "v1"); got != 1 {
|
||||||
|
t.Fatalf("overlaid prompt identity appears %d times, want 1", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestListIdentitiesEmptyEffectiveConfiguration(t *testing.T) {
|
||||||
|
main := configkit.Input{Source: configkit.Mapping("clear effective identities", map[string]any{
|
||||||
|
"profiles": nil,
|
||||||
|
"prompts": nil,
|
||||||
|
})}
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &main}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
profiles, err := engine.ListProfiles(context.Background())
|
||||||
|
if err != nil || len(profiles) != 0 {
|
||||||
|
t.Fatalf("cleared profile listing = %#v, %v", profiles, err)
|
||||||
|
}
|
||||||
|
prompts, err := engine.ListPrompts(context.Background())
|
||||||
|
if err != nil || len(prompts) != 0 {
|
||||||
|
t.Fatalf("cleared prompt listing = %#v, %v", prompts, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestListIdentitiesAreOwnedAndSafeForConcurrentUse(t *testing.T) {
|
||||||
|
engine, err := newListingContractEngine(t, map[string]any{
|
||||||
|
"profiles": map[string]any{"profile": map[string]any{"model": "partial-model"}},
|
||||||
|
"prompts": map[string]any{"prompt": map[string]any{"v1": listingContractPrompt()}},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
profiles, err := engine.ListProfiles(context.Background())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
prompts, err := engine.ListPrompts(context.Background())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for i := range profiles {
|
||||||
|
if profiles[i] == "profile" {
|
||||||
|
profiles[i] = "caller mutation"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for i := range prompts {
|
||||||
|
if prompts[i].ID == "prompt" {
|
||||||
|
prompts[i].ID = "caller mutation"
|
||||||
|
prompts[i].Version = "caller mutation"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
profiles, err = engine.ListProfiles(context.Background())
|
||||||
|
if err != nil || !containsListingProfile(profiles, "profile") {
|
||||||
|
t.Fatalf("profile result mutation escaped into engine: %#v, %v", profiles, err)
|
||||||
|
}
|
||||||
|
prompts, err = engine.ListPrompts(context.Background())
|
||||||
|
if err != nil || countListingPrompt(prompts, "prompt", "v1") != 1 {
|
||||||
|
t.Fatalf("prompt result mutation escaped into engine: %#v, %v", prompts, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
const workers = 4
|
||||||
|
const iterations = 25
|
||||||
|
start := make(chan struct{})
|
||||||
|
results := make(chan error, workers)
|
||||||
|
var wait sync.WaitGroup
|
||||||
|
wait.Add(workers)
|
||||||
|
for worker := 0; worker < workers; worker++ {
|
||||||
|
go func() {
|
||||||
|
defer wait.Done()
|
||||||
|
<-start
|
||||||
|
for iteration := 0; iteration < iterations; iteration++ {
|
||||||
|
profiles, err := engine.ListProfiles(context.Background())
|
||||||
|
if err != nil {
|
||||||
|
results <- fmt.Errorf("list profiles: %w", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if len(profiles) == 0 {
|
||||||
|
results <- errors.New("profile listing is empty")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
profiles[0] = "worker mutation"
|
||||||
|
|
||||||
|
prompts, err := engine.ListPrompts(context.Background())
|
||||||
|
if err != nil {
|
||||||
|
results <- fmt.Errorf("list prompts: %w", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if len(prompts) == 0 {
|
||||||
|
results <- errors.New("prompt listing is empty")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
prompts[0].ID = "worker mutation"
|
||||||
|
prompts[0].Version = "worker mutation"
|
||||||
|
|
||||||
|
if _, err := engine.InspectProfile(context.Background(), "profile"); err != nil {
|
||||||
|
results <- fmt.Errorf("inspect profile: %w", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if _, err := engine.InspectPrompt(context.Background(), "prompt", "v1"); err != nil {
|
||||||
|
results <- fmt.Errorf("inspect prompt: %w", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
close(start)
|
||||||
|
wait.Wait()
|
||||||
|
close(results)
|
||||||
|
for err := range results {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestListIdentitiesUseCapturedSourcesWithoutExecutionWork(t *testing.T) {
|
||||||
|
files := fstest.MapFS{
|
||||||
|
"config/listing.yaml": &fstest.MapFile{Data: []byte("profiles:\n captured-partial:\n model: partial-model\nprompts:\n captured-prompt:\n v1:\n messages:\n - role: user\n content: Hello.\n output:\n format: text\n validation_mode: basic\n")},
|
||||||
|
}
|
||||||
|
inputs, err := promptkit.FilesystemInputs("application listing", files, "config")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||||
|
Underlays: []configkit.Underlay{{Inputs: inputs}},
|
||||||
|
}}, promptkit.WithLLMClient(rejectingListingLLMClient{}), promptkit.WithArtifactReader(rejectingListingArtifactReader{}))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
files["config/listing.yaml"].Data = []byte("profiles:\n replacement-profile: {}\nprompts:\n replacement-prompt:\n v2:\n messages:\n - role: user\n content: Changed.\n output:\n format: text\n validation_mode: basic\n")
|
||||||
|
|
||||||
|
profiles, err := engine.ListProfiles(context.Background())
|
||||||
|
if err != nil || !containsListingProfile(profiles, "captured-partial") || containsListingProfile(profiles, "replacement-profile") {
|
||||||
|
t.Fatalf("profile listing did not use captured source: %#v, %v", profiles, err)
|
||||||
|
}
|
||||||
|
prompts, err := engine.ListPrompts(context.Background())
|
||||||
|
if err != nil || countListingPrompt(prompts, "captured-prompt", "v1") != 1 || countListingPrompt(prompts, "replacement-prompt", "v2") != 0 {
|
||||||
|
t.Fatalf("prompt listing did not use captured source: %#v, %v", prompts, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func listingContractPrompt() map[string]any {
|
||||||
|
return map[string]any{
|
||||||
|
"messages": []any{map[string]any{"role": "user", "content": "Hello."}},
|
||||||
|
"output": map[string]any{"format": "text", "validation_mode": "basic"},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func containsListingProfile(profiles []string, id string) bool {
|
||||||
|
for _, profile := range profiles {
|
||||||
|
if profile == id {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
func countListingProfile(profiles []string, id string) int {
|
||||||
|
count := 0
|
||||||
|
for _, profile := range profiles {
|
||||||
|
if profile == id {
|
||||||
|
count++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return count
|
||||||
|
}
|
||||||
|
|
||||||
|
func countListingPrompt(prompts []promptkit.PromptIdentity, id, version string) int {
|
||||||
|
count := 0
|
||||||
|
for _, prompt := range prompts {
|
||||||
|
if prompt.ID == id && prompt.Version == version {
|
||||||
|
count++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return count
|
||||||
|
}
|
||||||
|
|
||||||
|
func newListingContractEngine(t *testing.T, values map[string]any, options ...promptkit.Option) (*promptkit.Engine, error) {
|
||||||
|
t.Helper()
|
||||||
|
input := configkit.Input{Source: configkit.Mapping("listing contract", values)}
|
||||||
|
return promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &input}}, options...)
|
||||||
|
}
|
||||||
|
|
||||||
|
type rejectingListingLLMClient struct{}
|
||||||
|
|
||||||
|
func (rejectingListingLLMClient) Generate(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||||
|
return nil, errors.New("unexpected model generation")
|
||||||
|
}
|
||||||
|
|
||||||
|
type rejectingListingArtifactReader struct{}
|
||||||
|
|
||||||
|
func (rejectingListingArtifactReader) Read(context.Context, promptkit.ArtifactRef) (*promptkit.Artifact, error) {
|
||||||
|
return nil, errors.New("unexpected artifact read")
|
||||||
|
}
|
||||||
29
releasekit.json
Normal file
29
releasekit.json
Normal file
@@ -0,0 +1,29 @@
|
|||||||
|
{
|
||||||
|
"product": "Promptkit",
|
||||||
|
"remote": "origin",
|
||||||
|
"validation": {
|
||||||
|
"go": {
|
||||||
|
"race": true,
|
||||||
|
"full_vet": true
|
||||||
|
},
|
||||||
|
"go_format": true,
|
||||||
|
"markdown": {},
|
||||||
|
"module_distribution": {
|
||||||
|
"workspace_files": true,
|
||||||
|
"local_replacements": true
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"checks": [
|
||||||
|
{
|
||||||
|
"args": ["python3", "./scripts/check-examples.py"],
|
||||||
|
"env": {
|
||||||
|
"GOTOOLCHAIN": "local",
|
||||||
|
"GOWORK": "off",
|
||||||
|
"GOFLAGS": "-mod=readonly",
|
||||||
|
"GOPROXY": "off",
|
||||||
|
"GONOPROXY": "none",
|
||||||
|
"GOSUMDB": "off"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
240
requirements.go
Normal file
240
requirements.go
Normal file
@@ -0,0 +1,240 @@
|
|||||||
|
package promptkit
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/usecase"
|
||||||
|
)
|
||||||
|
|
||||||
|
// RequirementInspectionRequest selects a prompt and supplies caller context
|
||||||
|
// for requirement discovery. It requires no artifact bindings or credentials.
|
||||||
|
type RequirementInspectionRequest struct {
|
||||||
|
// PromptID is the exact, case-sensitive configured prompt ID.
|
||||||
|
PromptID string
|
||||||
|
// PromptVersion selects an exact version. An empty value uses the prompt
|
||||||
|
// selection rule that requires exactly one configured version.
|
||||||
|
PromptVersion string
|
||||||
|
// ProfileID selects the exact effective profile identity. If empty, the
|
||||||
|
// prompt's default profile is used; the profile need not be ready for
|
||||||
|
// execution.
|
||||||
|
ProfileID string
|
||||||
|
// Vars supplies string values for caller-variable lookups and simple
|
||||||
|
// conditions. Presence is determined by exact map key, including an empty
|
||||||
|
// value. Inspection reports caller keys and presence, not their values.
|
||||||
|
// A supplied value may nevertheless appear as a resolved input identifier.
|
||||||
|
Vars map[string]string
|
||||||
|
// SessionID, when nonblank after the normal session-ID normalization,
|
||||||
|
// bypasses the prompt's session template during discovery.
|
||||||
|
SessionID string
|
||||||
|
}
|
||||||
|
|
||||||
|
// RequirementKind describes how a selected prompt may depend on an input or
|
||||||
|
// caller variable for the supplied context. It is independent of whether a
|
||||||
|
// caller-variable value is present.
|
||||||
|
type RequirementKind string
|
||||||
|
|
||||||
|
const (
|
||||||
|
// RequirementRequired identifies an input declared required or a dependency
|
||||||
|
// reached by a recognized definite access.
|
||||||
|
RequirementRequired RequirementKind = "required"
|
||||||
|
// RequirementConditional identifies a dependency whose reachability or
|
||||||
|
// identity remains unresolved for the supplied context.
|
||||||
|
RequirementConditional RequirementKind = "conditional"
|
||||||
|
// RequirementOptional identifies a dependency known not to be required,
|
||||||
|
// including a nonrequired declaration, optional lookup, or reference used
|
||||||
|
// only on a known skipped path.
|
||||||
|
RequirementOptional RequirementKind = "optional"
|
||||||
|
)
|
||||||
|
|
||||||
|
// InputRequirement describes a declared or template-referenced prompt input.
|
||||||
|
type InputRequirement struct {
|
||||||
|
// Name is the exact input identifier.
|
||||||
|
Name string
|
||||||
|
// Kind is the strongest classification established for this name.
|
||||||
|
Kind RequirementKind
|
||||||
|
// Declared reports whether the name appears in the effective input
|
||||||
|
// declarations; helper-only references have Declared false.
|
||||||
|
Declared bool
|
||||||
|
// ContentType preserves effective metadata presence: nil means absent or
|
||||||
|
// null, while a non-nil pointer may refer to an empty string.
|
||||||
|
ContentType *string
|
||||||
|
// Description preserves effective metadata presence independently of
|
||||||
|
// Explanation, including an explicitly empty string.
|
||||||
|
Description *string
|
||||||
|
// Explanation is safe human-readable context, such as why a reference is
|
||||||
|
// optional because it occurs only on a known skipped path. It is not a
|
||||||
|
// machine-readable contract.
|
||||||
|
Explanation string
|
||||||
|
}
|
||||||
|
|
||||||
|
// VariableRequirement describes a caller-map key referenced by a prompt.
|
||||||
|
type VariableRequirement struct {
|
||||||
|
// Name is the exact caller-map key, not a template-local name.
|
||||||
|
Name string
|
||||||
|
// Kind is the strongest classification established for this key.
|
||||||
|
Kind RequirementKind
|
||||||
|
// Supplied reports exact-key presence in the request Vars map. An empty
|
||||||
|
// string is supplied; the value itself is not returned.
|
||||||
|
Supplied bool
|
||||||
|
// Explanation is safe human-readable context and is not a machine-readable
|
||||||
|
// contract.
|
||||||
|
Explanation string
|
||||||
|
}
|
||||||
|
|
||||||
|
// RequirementIssueCode identifies a limitation that can affect discovery.
|
||||||
|
type RequirementIssueCode string
|
||||||
|
|
||||||
|
const (
|
||||||
|
// RequirementIssueMissingVariable means an absent caller value prevents
|
||||||
|
// resolving a dependency decision or input name.
|
||||||
|
RequirementIssueMissingVariable RequirementIssueCode = "missing_variable"
|
||||||
|
// RequirementIssueInputContent means unavailable input content affects
|
||||||
|
// dependency discovery.
|
||||||
|
RequirementIssueInputContent RequirementIssueCode = "input_content"
|
||||||
|
// RequirementIssueDynamicInputName means a referenced input name cannot be
|
||||||
|
// determined for the supplied context.
|
||||||
|
RequirementIssueDynamicInputName RequirementIssueCode = "dynamic_input_name"
|
||||||
|
// RequirementIssueUnsupportedConstruct means a dependency-relevant
|
||||||
|
// expression or scope is outside the supported discovery analysis.
|
||||||
|
RequirementIssueUnsupportedConstruct RequirementIssueCode = "unsupported_construct"
|
||||||
|
)
|
||||||
|
|
||||||
|
// RequirementIssue describes one discovery limitation using safe summary
|
||||||
|
// text. Descriptions are not intended as machine-readable details.
|
||||||
|
type RequirementIssue struct {
|
||||||
|
// Code is the machine-readable limitation category.
|
||||||
|
Code RequirementIssueCode
|
||||||
|
// InputName identifies an affected input when one is known; empty means no
|
||||||
|
// particular input was identified.
|
||||||
|
InputName string
|
||||||
|
// VariableName identifies an affected caller key when one is known; empty
|
||||||
|
// means no particular key was identified.
|
||||||
|
VariableName string
|
||||||
|
// Description is a safe human-readable summary, not a string to parse.
|
||||||
|
Description string
|
||||||
|
}
|
||||||
|
|
||||||
|
// RequirementInspection contains discovered dependencies for one selected
|
||||||
|
// prompt context. Input and variable entries are sorted by exact name. Issues
|
||||||
|
// are deduplicated by code and associated names, then sorted by that tuple.
|
||||||
|
// Their descriptions and entry explanations are safe summaries, not
|
||||||
|
// machine-readable details.
|
||||||
|
//
|
||||||
|
// Complete describes the supported analysis's discovery coverage, not whether
|
||||||
|
// requirements are supplied, templates will render, artifacts or schemas are
|
||||||
|
// valid, or execution is ready. A complete result can still contain required
|
||||||
|
// inputs or missing required variables. Valid templates using dependency-
|
||||||
|
// relevant constructs outside the deliberately small recognition scope may
|
||||||
|
// produce useful partial inventories with Complete false; this is not a
|
||||||
|
// template-definition error or an authoring prohibition.
|
||||||
|
type RequirementInspection struct {
|
||||||
|
// PromptID is the selected exact prompt ID.
|
||||||
|
PromptID string
|
||||||
|
// PromptVersion is the selected exact prompt version.
|
||||||
|
PromptVersion string
|
||||||
|
// ProfileID is the selected profile identity; it does not indicate
|
||||||
|
// execution readiness.
|
||||||
|
ProfileID string
|
||||||
|
// Inputs contains declared inputs and recognizable template references,
|
||||||
|
// including entries known to be optional.
|
||||||
|
Inputs []InputRequirement
|
||||||
|
// Variables contains recognizable caller-map keys, including keys used
|
||||||
|
// only on known skipped paths.
|
||||||
|
Variables []VariableRequirement
|
||||||
|
// Complete reports discovery coverage, not requirement satisfaction or
|
||||||
|
// execution readiness.
|
||||||
|
Complete bool
|
||||||
|
// Issues contains deduplicated, structured discovery limitations.
|
||||||
|
Issues []RequirementIssue
|
||||||
|
}
|
||||||
|
|
||||||
|
// InspectRequirements discovers inputs and caller variables for one selected
|
||||||
|
// prompt before its input artifacts exist. It uses exact, case-sensitive
|
||||||
|
// prompt selection and requires a profile ID either in the request or the
|
||||||
|
// selected prompt's default. The selected profile establishes identity only;
|
||||||
|
// it need not be execution-ready. A normalized nonblank SessionID bypasses the
|
||||||
|
// captured session template. Vars are copied before analysis, and returned
|
||||||
|
// slices and metadata pointers belong to the caller.
|
||||||
|
//
|
||||||
|
// Discovery recognizes literal input names, direct caller-map keys, literal-
|
||||||
|
// key root-map lookups, simple caller-derived input names, and if/with
|
||||||
|
// conditions based on direct values, literals, negation, or two-string
|
||||||
|
// equality/inequality. It classifies recognizable references on known skipped
|
||||||
|
// paths as optional. Other dependency-relevant expressions or scopes may leave
|
||||||
|
// a useful partial result with Complete false. The returned inventories do
|
||||||
|
// not reject otherwise valid templates or prohibit their use.
|
||||||
|
//
|
||||||
|
// Results exclude artifact payloads, prompt contents, and unrelated supplied
|
||||||
|
// values. Resolved input identifiers may equal the caller values that supply
|
||||||
|
// them; those identifiers are intentionally included in the inventory.
|
||||||
|
//
|
||||||
|
// This method does not perform artifact, schema, credential, rendering,
|
||||||
|
// validation, capacity, observation, provider, or execution work. Complete
|
||||||
|
// describes dependency discovery, not whether every dependency is supplied or
|
||||||
|
// the prompt is ready to execute. Discovery limitations return a partial
|
||||||
|
// inspection and no error; selection and operational failures return no
|
||||||
|
// inspection. Blank prompt IDs, invalid session IDs, and absent profile
|
||||||
|
// selection match ErrInvalidRequest; an absent profile selection also matches
|
||||||
|
// ErrProfileRequired. Missing prompt/profile IDs match ErrPromptNotFound or
|
||||||
|
// ErrProfileNotFound. Other selection/snapshot failures match ErrPromptLoad or
|
||||||
|
// ErrProfileLoad. When cancellation prevents lookup or analysis, no inspection
|
||||||
|
// is returned and the context error remains discoverable with errors.Is. A nil
|
||||||
|
// Engine matches ErrInvalidConfig.
|
||||||
|
func (e *Engine) InspectRequirements(
|
||||||
|
ctx context.Context,
|
||||||
|
req RequirementInspectionRequest,
|
||||||
|
) (*RequirementInspection, error) {
|
||||||
|
if e == nil || e.internal == nil {
|
||||||
|
return nil, ErrInvalidConfig
|
||||||
|
}
|
||||||
|
inspection, err := e.internal.InspectRequirements(ctx, usecase.RequirementInspectionRequest{
|
||||||
|
PromptID: req.PromptID, PromptVersion: req.PromptVersion, ProfileID: req.ProfileID,
|
||||||
|
Vars: copyStringMap(req.Vars), SessionID: req.SessionID,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, mapPublicError(err)
|
||||||
|
}
|
||||||
|
return fromDomainRequirementInspection(inspection), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func fromDomainRequirementInspection(inspection *domain.RequirementInspection) *RequirementInspection {
|
||||||
|
if inspection == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
result := &RequirementInspection{
|
||||||
|
PromptID: inspection.PromptID, PromptVersion: inspection.PromptVersion,
|
||||||
|
ProfileID: inspection.ProfileID, Complete: inspection.Complete,
|
||||||
|
Inputs: make([]InputRequirement, len(inspection.Inputs)),
|
||||||
|
Variables: make([]VariableRequirement, len(inspection.Variables)),
|
||||||
|
Issues: make([]RequirementIssue, len(inspection.Issues)),
|
||||||
|
}
|
||||||
|
for index, input := range inspection.Inputs {
|
||||||
|
result.Inputs[index] = InputRequirement{
|
||||||
|
Name: input.Name, Kind: RequirementKind(input.Kind), Declared: input.Declared,
|
||||||
|
ContentType: copyRequirementString(input.ContentType), Description: copyRequirementString(input.Description),
|
||||||
|
Explanation: input.Explanation,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for index, variable := range inspection.Variables {
|
||||||
|
result.Variables[index] = VariableRequirement{
|
||||||
|
Name: variable.Name, Kind: RequirementKind(variable.Kind), Supplied: variable.Supplied,
|
||||||
|
Explanation: variable.Explanation,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for index, issue := range inspection.Issues {
|
||||||
|
result.Issues[index] = RequirementIssue{
|
||||||
|
Code: RequirementIssueCode(issue.Code), InputName: issue.InputName,
|
||||||
|
VariableName: issue.VariableName, Description: issue.Description,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
|
func copyRequirementString(value *string) *string {
|
||||||
|
if value == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
copy := *value
|
||||||
|
return ©
|
||||||
|
}
|
||||||
72
requirements_inspection_benchmark_test.go
Normal file
72
requirements_inspection_benchmark_test.go
Normal file
@@ -0,0 +1,72 @@
|
|||||||
|
package promptkit_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/ConfigKit"
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
var inspectedRequirements *promptkit.RequirementInspection
|
||||||
|
|
||||||
|
func requirementInspectionExtras(entries int) map[string]any {
|
||||||
|
providers := make(map[string]any, entries)
|
||||||
|
for index := range entries {
|
||||||
|
providers[fmt.Sprintf("provider-%03d", index)] = map[string]any{
|
||||||
|
"enabled": true,
|
||||||
|
"labels": []any{fmt.Sprintf("tier-%d", index%4), fmt.Sprintf("region-%d", index%3)},
|
||||||
|
"limits": map[string]any{"burst": index + 1, "period": "minute"},
|
||||||
|
"routes": []any{map[string]any{"weight": index + 1, "fallback": true}},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return map[string]any{"provider_options": providers}
|
||||||
|
}
|
||||||
|
|
||||||
|
func BenchmarkRequirementInspection(b *testing.B) {
|
||||||
|
profiles := map[string]any{
|
||||||
|
"small": map[string]any{
|
||||||
|
"model": "inspection-model", "extra_params": requirementInspectionExtras(2),
|
||||||
|
},
|
||||||
|
"large": map[string]any{
|
||||||
|
"model": "inspection-model", "extra_params": requirementInspectionExtras(64),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
definition := contractPrompt("small", "Hello {{.name}} {{input \"transcript\"}}", []any{
|
||||||
|
map[string]any{"name": "transcript", "required": true},
|
||||||
|
}, "", noValidationOutput)
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
|
||||||
|
Source: configkit.Mapping("requirement inspection benchmark", map[string]any{
|
||||||
|
"prompts": oneVersionPrompt("inspection", "1", definition),
|
||||||
|
"profiles": profiles,
|
||||||
|
}),
|
||||||
|
}}})
|
||||||
|
if err != nil {
|
||||||
|
b.Fatal(err)
|
||||||
|
}
|
||||||
|
ctx := context.Background()
|
||||||
|
for _, profileID := range []string{"small", "large"} {
|
||||||
|
request := promptkit.RequirementInspectionRequest{
|
||||||
|
PromptID: "inspection", ProfileID: profileID, Vars: map[string]string{"name": "Ada"},
|
||||||
|
}
|
||||||
|
inspection, err := engine.InspectRequirements(ctx, request)
|
||||||
|
if err != nil || inspection.PromptID != "inspection" || inspection.PromptVersion != "1" ||
|
||||||
|
inspection.ProfileID != profileID || !inspection.Complete || len(inspection.Inputs) != 1 ||
|
||||||
|
inspection.Inputs[0].Name != "transcript" || inspection.Inputs[0].Kind != promptkit.RequirementRequired ||
|
||||||
|
len(inspection.Variables) != 1 || inspection.Variables[0].Name != "name" || !inspection.Variables[0].Supplied {
|
||||||
|
b.Fatalf("requirement inspection fixture for %q is invalid: inspection=%+v err=%v", profileID, inspection, err)
|
||||||
|
}
|
||||||
|
selectedRequest := request
|
||||||
|
b.Run(profileID+"_extra", func(b *testing.B) {
|
||||||
|
b.ReportAllocs()
|
||||||
|
for b.Loop() {
|
||||||
|
inspection, err := engine.InspectRequirements(ctx, selectedRequest)
|
||||||
|
if err != nil {
|
||||||
|
b.Fatal(err)
|
||||||
|
}
|
||||||
|
inspectedRequirements = inspection
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
370
requirements_inspection_contract_test.go
Normal file
370
requirements_inspection_contract_test.go
Normal file
@@ -0,0 +1,370 @@
|
|||||||
|
package promptkit_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
|
"testing"
|
||||||
|
"testing/fstest"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/ConfigKit"
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
func requirementsEngine(t *testing.T, prompts, profiles map[string]any, options ...promptkit.Option) *promptkit.Engine {
|
||||||
|
t.Helper()
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
|
||||||
|
Source: configkit.Mapping("requirements contract", map[string]any{
|
||||||
|
"prompts": prompts, "profiles": profiles,
|
||||||
|
}),
|
||||||
|
}}}, options...)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("construct requirements engine: %v", err)
|
||||||
|
}
|
||||||
|
return engine
|
||||||
|
}
|
||||||
|
|
||||||
|
type requirementCountingArtifactReader struct{ calls atomic.Int64 }
|
||||||
|
|
||||||
|
func (reader *requirementCountingArtifactReader) Read(context.Context, promptkit.ArtifactRef) (*promptkit.Artifact, error) {
|
||||||
|
reader.calls.Add(1)
|
||||||
|
return nil, errors.New("artifact reader must not be called")
|
||||||
|
}
|
||||||
|
|
||||||
|
func contractPrompt(defaultProfile string, content string, inputs []any, session string, output map[string]any) map[string]any {
|
||||||
|
definition := map[string]any{
|
||||||
|
"messages": []any{map[string]any{"role": "user", "content": content}},
|
||||||
|
"output": output,
|
||||||
|
}
|
||||||
|
if defaultProfile != "" {
|
||||||
|
definition["default_profile"] = defaultProfile
|
||||||
|
}
|
||||||
|
if inputs != nil {
|
||||||
|
definition["inputs"] = inputs
|
||||||
|
}
|
||||||
|
if session != "" {
|
||||||
|
definition["session_id"] = session
|
||||||
|
}
|
||||||
|
return definition
|
||||||
|
}
|
||||||
|
|
||||||
|
func oneVersionPrompt(id, version string, definition map[string]any) map[string]any {
|
||||||
|
return map[string]any{id: map[string]any{version: definition}}
|
||||||
|
}
|
||||||
|
|
||||||
|
var noValidationOutput = map[string]any{"format": "text", "validation_mode": "none"}
|
||||||
|
|
||||||
|
func TestInspectRequirementsEnumeratesCapturedDependenciesAndInactiveNames(t *testing.T) {
|
||||||
|
var modelCalls atomic.Int64
|
||||||
|
reader := &requirementCountingArtifactReader{}
|
||||||
|
definition := contractPrompt("partial", `TEMPLATE_PRIVATE_MARKER {{if eq .mode "brief"}}{{input "brief"}}{{else}}{{input "long"}}{{end}} {{input "undeclared"}} {{.required}} {{if false}}{{input "inactive"}}{{.inactive_var}}{{end}}`, []any{
|
||||||
|
map[string]any{"name": "transcript", "required": true, "content_type": "audio/wav", "description": "Transcript source."},
|
||||||
|
map[string]any{"name": "optional", "content_type": "text/plain", "description": ""},
|
||||||
|
}, `{{input "session-only"}} {{.session_variable}}`, map[string]any{
|
||||||
|
"format": "json", "validation_mode": "json_schema", "schema_path": "not-loaded.json",
|
||||||
|
})
|
||||||
|
engine := requirementsEngine(t, oneVersionPrompt("requirements", "v1", definition), map[string]any{
|
||||||
|
"partial": map[string]any{"model": "partial-model", "api_key_required": true},
|
||||||
|
}, promptkit.WithArtifactReader(reader), promptkit.WithLLMClient(countingLLMClient{calls: &modelCalls}))
|
||||||
|
inspection, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{
|
||||||
|
PromptID: "requirements", Vars: map[string]string{"mode": "brief", "unrelated": "VARIABLE_PRIVATE_MARKER"},
|
||||||
|
SessionID: "SESSION_PRIVATE_MARKER",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("inspect before artifacts, credentials, and schemas exist: %v", err)
|
||||||
|
}
|
||||||
|
if inspection.PromptID != "requirements" || inspection.PromptVersion != "v1" || inspection.ProfileID != "partial" || !inspection.Complete {
|
||||||
|
t.Fatalf("selected context or completeness is wrong: %+v", inspection)
|
||||||
|
}
|
||||||
|
inputs := make(map[string]promptkit.InputRequirement, len(inspection.Inputs))
|
||||||
|
for _, input := range inspection.Inputs {
|
||||||
|
inputs[input.Name] = input
|
||||||
|
}
|
||||||
|
if len(inputs) != 6 || inputs["transcript"].Kind != promptkit.RequirementRequired || !inputs["transcript"].Declared ||
|
||||||
|
inputs["transcript"].ContentType == nil || *inputs["transcript"].ContentType != "audio/wav" ||
|
||||||
|
inputs["transcript"].Description == nil || *inputs["transcript"].Description != "Transcript source." {
|
||||||
|
t.Fatalf("required declaration metadata or discovered inputs are wrong: %+v", inspection.Inputs)
|
||||||
|
}
|
||||||
|
if inputs["brief"].Kind != promptkit.RequirementRequired || inputs["long"].Kind != promptkit.RequirementOptional || inputs["long"].Explanation == "" ||
|
||||||
|
inputs["inactive"].Kind != promptkit.RequirementOptional || inputs["inactive"].Explanation == "" ||
|
||||||
|
inputs["undeclared"].Kind != promptkit.RequirementRequired || inputs["undeclared"].Declared ||
|
||||||
|
inputs["optional"].Kind != promptkit.RequirementOptional || !inputs["optional"].Declared ||
|
||||||
|
inputs["optional"].Description == nil || *inputs["optional"].Description != "" {
|
||||||
|
t.Fatalf("selected, inactive, or helper-only input classifications are wrong: %+v", inspection.Inputs)
|
||||||
|
}
|
||||||
|
if _, found := inputs["session-only"]; found {
|
||||||
|
t.Fatalf("bypassed session input appeared in inventory: %+v", inspection.Inputs)
|
||||||
|
}
|
||||||
|
variables := make(map[string]promptkit.VariableRequirement, len(inspection.Variables))
|
||||||
|
for _, variable := range inspection.Variables {
|
||||||
|
variables[variable.Name] = variable
|
||||||
|
}
|
||||||
|
if len(variables) != 3 || variables["mode"].Kind != promptkit.RequirementRequired || !variables["mode"].Supplied ||
|
||||||
|
variables["required"].Kind != promptkit.RequirementRequired || variables["required"].Supplied ||
|
||||||
|
variables["inactive_var"].Kind != promptkit.RequirementOptional || variables["inactive_var"].Explanation == "" {
|
||||||
|
t.Fatalf("caller-variable inventory conflated need and presence: %+v", inspection.Variables)
|
||||||
|
}
|
||||||
|
if _, found := variables["unrelated"]; found {
|
||||||
|
t.Fatalf("unreferenced caller key entered the inventory: %+v", inspection.Variables)
|
||||||
|
}
|
||||||
|
formatted := fmt.Sprintf("%+v", inspection)
|
||||||
|
for _, private := range []string{
|
||||||
|
"TEMPLATE_PRIVATE_MARKER", "VARIABLE_PRIVATE_MARKER", "SESSION_PRIVATE_MARKER",
|
||||||
|
} {
|
||||||
|
if strings.Contains(formatted, private) {
|
||||||
|
t.Fatalf("inspection disclosed %q: %s", private, formatted)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if reader.calls.Load() != 0 || modelCalls.Load() != 0 {
|
||||||
|
t.Fatalf("inspection invoked execution collaborators: artifact reads=%d provider calls=%d", reader.calls.Load(), modelCalls.Load())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestInspectRequirementsSelectionUsesExactVersionsAndProfileIdentity(t *testing.T) {
|
||||||
|
prompts := map[string]any{
|
||||||
|
"multi": map[string]any{
|
||||||
|
"v1": contractPrompt("first", `{{input "one"}}`, nil, "", noValidationOutput),
|
||||||
|
"v2": contractPrompt("second", `{{input "two"}}`, nil, "", noValidationOutput),
|
||||||
|
},
|
||||||
|
"without-default": map[string]any{
|
||||||
|
"v1": contractPrompt("", "plain", nil, "", noValidationOutput),
|
||||||
|
},
|
||||||
|
"direct-required": map[string]any{"v1": contractPrompt("direct", "plain", nil, "", noValidationOutput)},
|
||||||
|
}
|
||||||
|
profiles := map[string]any{
|
||||||
|
"first": map[string]any{"model": "first-model"},
|
||||||
|
"second": map[string]any{"model": "second-model"},
|
||||||
|
"direct": map[string]any{"endpoint": "https://example.test/v1", "model": "direct-model", "api_key_required": true},
|
||||||
|
}
|
||||||
|
engine := requirementsEngine(t, prompts, profiles)
|
||||||
|
if _, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{PromptID: "multi"}); !errors.Is(err, promptkit.ErrPromptLoad) {
|
||||||
|
t.Fatalf("ambiguous omitted version error=%v", err)
|
||||||
|
}
|
||||||
|
first, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{PromptID: "multi", PromptVersion: "v1"})
|
||||||
|
if err != nil || first.PromptVersion != "v1" || first.ProfileID != "first" || len(first.Inputs) != 1 || first.Inputs[0].Name != "one" {
|
||||||
|
t.Fatalf("exact first version/default profile=(%+v, %v)", first, err)
|
||||||
|
}
|
||||||
|
explicit, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{
|
||||||
|
PromptID: "multi", PromptVersion: "v2", ProfileID: "first",
|
||||||
|
})
|
||||||
|
if err != nil || explicit.PromptVersion != "v2" || explicit.ProfileID != "first" || explicit.Inputs[0].Name != "two" {
|
||||||
|
t.Fatalf("exact version/request profile=(%+v, %v)", explicit, err)
|
||||||
|
}
|
||||||
|
if _, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{PromptID: "Multi", PromptVersion: "v1"}); !errors.Is(err, promptkit.ErrPromptNotFound) {
|
||||||
|
t.Fatalf("case-folded prompt ID unexpectedly selected: %v", err)
|
||||||
|
}
|
||||||
|
if _, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{PromptID: "multi", PromptVersion: "V1"}); !errors.Is(err, promptkit.ErrPromptNotFound) {
|
||||||
|
t.Fatalf("case-folded version unexpectedly selected: %v", err)
|
||||||
|
}
|
||||||
|
if _, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{PromptID: "multi", PromptVersion: "v1", ProfileID: "missing"}); !errors.Is(err, promptkit.ErrProfileNotFound) {
|
||||||
|
t.Fatalf("missing profile error=%v", err)
|
||||||
|
}
|
||||||
|
if _, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{PromptID: "without-default"}); !errors.Is(err, promptkit.ErrProfileRequired) || !errors.Is(err, promptkit.ErrInvalidRequest) {
|
||||||
|
t.Fatalf("missing profile selection error=%v", err)
|
||||||
|
}
|
||||||
|
if direct, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{PromptID: "direct-required"}); err != nil || direct.ProfileID != "direct" {
|
||||||
|
t.Fatalf("direct-key-required profile should be inspectable without a key: (%+v, %v)", direct, err)
|
||||||
|
}
|
||||||
|
if _, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{}); !errors.Is(err, promptkit.ErrInvalidRequest) {
|
||||||
|
t.Fatalf("blank prompt error=%v", err)
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
cancel()
|
||||||
|
if _, err := engine.InspectRequirements(ctx, promptkit.RequirementInspectionRequest{PromptID: "multi", PromptVersion: "v1"}); !errors.Is(err, context.Canceled) {
|
||||||
|
t.Fatalf("canceled inspection error=%v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestInspectPromptHashMatchesPreparationForCapturedConfiguration(t *testing.T) {
|
||||||
|
prompts := oneVersionPrompt("hash-equality", "v1", contractPrompt(
|
||||||
|
"profile", "Hello {{.name}}", nil, "", noValidationOutput,
|
||||||
|
))
|
||||||
|
engine := requirementsEngine(t, prompts, map[string]any{
|
||||||
|
"profile": map[string]any{"endpoint": "https://example.test/v1", "model": "inspection-model"},
|
||||||
|
})
|
||||||
|
inspection, err := engine.InspectPrompt(context.Background(), "hash-equality", "v1")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("inspect captured prompt: %v", err)
|
||||||
|
}
|
||||||
|
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||||
|
PromptID: "hash-equality", PromptVersion: "v1", Vars: map[string]string{"name": "Ada"},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("prepare captured prompt: %v", err)
|
||||||
|
}
|
||||||
|
if inspection.PromptHash != prepared.PromptHash {
|
||||||
|
t.Fatalf("inspection hash=%q, preparation hash=%q", inspection.PromptHash, prepared.PromptHash)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestInspectRequirementsReportsConditionalAndCompleteOptionalInventories(t *testing.T) {
|
||||||
|
prompts := map[string]any{
|
||||||
|
"conditional": map[string]any{"v1": contractPrompt("profile",
|
||||||
|
`{{if .mode}}{{input "left"}}{{else}}{{input "right"}}{{end}}`, nil, "", noValidationOutput)},
|
||||||
|
"optional": map[string]any{"v1": contractPrompt("profile", `{{index . "lookup"}}`,
|
||||||
|
[]any{map[string]any{"name": "declared-optional"}}, "", noValidationOutput)},
|
||||||
|
"empty": map[string]any{"v1": contractPrompt("profile", "plain", nil, "", noValidationOutput)},
|
||||||
|
}
|
||||||
|
engine := requirementsEngine(t, prompts, map[string]any{"profile": map[string]any{"model": "partial"}})
|
||||||
|
conditional, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{PromptID: "conditional"})
|
||||||
|
if err != nil || conditional.Complete || len(conditional.Issues) != 1 ||
|
||||||
|
conditional.Issues[0].Code != promptkit.RequirementIssueMissingVariable || len(conditional.Inputs) != 2 ||
|
||||||
|
conditional.Inputs[0].Kind != promptkit.RequirementConditional || conditional.Inputs[1].Kind != promptkit.RequirementConditional {
|
||||||
|
t.Fatalf("conditional discovery=(%+v, %v)", conditional, err)
|
||||||
|
}
|
||||||
|
optional, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{PromptID: "optional"})
|
||||||
|
if err != nil || !optional.Complete || len(optional.Issues) != 0 || len(optional.Inputs) != 1 ||
|
||||||
|
optional.Inputs[0].Kind != promptkit.RequirementOptional || len(optional.Variables) != 1 ||
|
||||||
|
optional.Variables[0].Name != "lookup" || optional.Variables[0].Kind != promptkit.RequirementOptional {
|
||||||
|
t.Fatalf("all-optional result should be complete without artifact bindings: (%+v, %v)", optional, err)
|
||||||
|
}
|
||||||
|
empty, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{PromptID: "empty"})
|
||||||
|
if err != nil || !empty.Complete || len(empty.Inputs) != 0 || len(empty.Variables) != 0 {
|
||||||
|
t.Fatalf("empty complete result=(%+v, %v)", empty, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestInspectRequirementsBeforePreparationAndWithDirectSessionBypass(t *testing.T) {
|
||||||
|
definition := contractPrompt("profile", `Name={{.name}} Transcript={{input "transcript"}}{{if false}}ignored {{input "skipped"}} {{.inactive}}{{end}}`, []any{
|
||||||
|
map[string]any{"name": "transcript", "required": true},
|
||||||
|
}, `{{.session_var}} {{input "session-only"}}`, noValidationOutput)
|
||||||
|
engine := requirementsEngine(t, oneVersionPrompt("ready", "v1", definition), map[string]any{
|
||||||
|
"profile": map[string]any{"endpoint": "https://example.test/v1", "model": "offline"},
|
||||||
|
})
|
||||||
|
inspection, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{
|
||||||
|
PromptID: "ready", Vars: map[string]string{"name": "Ada"}, SessionID: "direct-session",
|
||||||
|
})
|
||||||
|
if err != nil || inspection == nil || !inspection.Complete || len(inspection.Inputs) != 2 ||
|
||||||
|
inspection.Inputs[0].Name != "skipped" || inspection.Inputs[0].Kind != promptkit.RequirementOptional ||
|
||||||
|
inspection.Inputs[1].Name != "transcript" || inspection.Inputs[1].Kind != promptkit.RequirementRequired ||
|
||||||
|
len(inspection.Variables) != 2 || inspection.Variables[0].Name != "inactive" || inspection.Variables[0].Kind != promptkit.RequirementOptional ||
|
||||||
|
inspection.Variables[1].Name != "name" || inspection.Variables[1].Kind != promptkit.RequirementRequired {
|
||||||
|
t.Fatalf("artifact-free inspection=(%+v, %v)", inspection, err)
|
||||||
|
}
|
||||||
|
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||||
|
PromptID: "ready", Vars: map[string]string{"name": "Ada"}, SessionID: "direct-session",
|
||||||
|
Inputs: map[string]promptkit.ArtifactRef{"transcript": promptkit.Inline("source transcript")},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("prepare with matching synthetic artifact: %v", err)
|
||||||
|
}
|
||||||
|
if prepared.Messages[0].Content != "Name=Ada Transcript=source transcript" || prepared.SessionID != "direct-session" {
|
||||||
|
t.Fatalf("prepared run did not use the inspected dependencies: %+v", prepared)
|
||||||
|
}
|
||||||
|
if _, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{
|
||||||
|
PromptID: "ready", Vars: map[string]string{"name": "Ada"},
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatalf("inspect without session bypass: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestInspectRequirementsUsesCapturedFileBackedTemplates(t *testing.T) {
|
||||||
|
files := fstest.MapFS{
|
||||||
|
"config/prompt.yaml": &fstest.MapFile{Data: []byte(capturedPromptDocument)},
|
||||||
|
"config/body.tmpl": &fstest.MapFile{Data: []byte(`{{input "captured-input"}}`)},
|
||||||
|
}
|
||||||
|
source := &unreadableAfterConstructionFS{FS: files}
|
||||||
|
engine := capturedPromptEngine(t, source)
|
||||||
|
source.closed.Store(true)
|
||||||
|
delete(files, "config/body.tmpl")
|
||||||
|
inspection, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{PromptID: "captured"})
|
||||||
|
if err != nil || inspection == nil || !inspection.Complete || len(inspection.Inputs) != 1 ||
|
||||||
|
inspection.Inputs[0].Name != "captured-input" || source.reads.Load() != 0 {
|
||||||
|
t.Fatalf("captured template inspection=(%+v, %v), post-construction reads=%d", inspection, err, source.reads.Load())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestInspectRequirementsReturnsIndependentValuesAndCanOverlapPreparation(t *testing.T) {
|
||||||
|
definition := contractPrompt("profile", `{{if .decision}}{{input "conditional"}}{{end}} {{input "declared"}}`, []any{
|
||||||
|
map[string]any{"name": "declared", "required": true, "description": "captured description"},
|
||||||
|
}, `{{.session}}`, noValidationOutput)
|
||||||
|
engine := requirementsEngine(t, oneVersionPrompt("owned", "v1", definition), map[string]any{
|
||||||
|
"profile": map[string]any{"endpoint": "https://example.test/v1", "model": "offline"},
|
||||||
|
})
|
||||||
|
request := promptkit.RequirementInspectionRequest{PromptID: "owned"}
|
||||||
|
first, err := engine.InspectRequirements(context.Background(), request)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for index := range first.Inputs {
|
||||||
|
if first.Inputs[index].Name == "declared" {
|
||||||
|
first.Inputs[index].Name = "mutated"
|
||||||
|
if first.Inputs[index].Description != nil {
|
||||||
|
*first.Inputs[index].Description = "mutated description"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
first.Variables[0].Name = "mutated-variable"
|
||||||
|
first.Issues[0].Description = "mutated issue"
|
||||||
|
second, err := engine.InspectRequirements(context.Background(), request)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("inspect after mutating returned values: %v", err)
|
||||||
|
}
|
||||||
|
var description *string
|
||||||
|
for _, input := range second.Inputs {
|
||||||
|
if input.Name == "declared" {
|
||||||
|
description = input.Description
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if second.Inputs[0].Name == "mutated" || second.Variables[0].Name != "decision" ||
|
||||||
|
description == nil || *description != "captured description" ||
|
||||||
|
len(second.Issues) != 1 || second.Issues[0].Description == "mutated issue" {
|
||||||
|
t.Fatalf("result mutation reached captured state or later result: (%+v, %v)", second, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
const count = 12
|
||||||
|
var wait sync.WaitGroup
|
||||||
|
errs := make(chan error, count*2)
|
||||||
|
for index := 0; index < count; index++ {
|
||||||
|
wait.Add(2)
|
||||||
|
go func() {
|
||||||
|
defer wait.Done()
|
||||||
|
_, err := engine.InspectRequirements(context.Background(), request)
|
||||||
|
errs <- err
|
||||||
|
}()
|
||||||
|
go func(index int) {
|
||||||
|
defer wait.Done()
|
||||||
|
_, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||||
|
PromptID: "owned", SessionID: fmt.Sprintf("session-%d", index), Vars: map[string]string{"decision": "yes"},
|
||||||
|
Inputs: map[string]promptkit.ArtifactRef{
|
||||||
|
"conditional": promptkit.Inline("condition"), "declared": promptkit.Inline("body"),
|
||||||
|
},
|
||||||
|
})
|
||||||
|
errs <- err
|
||||||
|
}(index)
|
||||||
|
}
|
||||||
|
wait.Wait()
|
||||||
|
close(errs)
|
||||||
|
for err := range errs {
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestInspectRequirementsDoesNotEchoUnrelatedPrivateValues(t *testing.T) {
|
||||||
|
t.Setenv("PROMPTKIT_REQUIREMENTS_SECRET", "credential-value-marker")
|
||||||
|
definition := contractPrompt("profile", `template-body-marker {{input "required-input"}} {{.known}}`, nil, `{{.session}}`, noValidationOutput)
|
||||||
|
engine := requirementsEngine(t, oneVersionPrompt("privacy", "v1", definition), map[string]any{
|
||||||
|
"profile": map[string]any{"endpoint": "https://example.test/v1", "model": "offline", "api_key_env": "PROMPTKIT_REQUIREMENTS_SECRET"},
|
||||||
|
})
|
||||||
|
inspection, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{
|
||||||
|
PromptID: "privacy", Vars: map[string]string{"known": "known-value-marker", "unrelated": "unrelated-value-marker"},
|
||||||
|
SessionID: "direct-session-marker",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
formatted := fmt.Sprintf("%+v", inspection)
|
||||||
|
for _, marker := range []string{
|
||||||
|
"template-body-marker", "known-value-marker", "unrelated-value-marker",
|
||||||
|
"direct-session-marker", "credential-value-marker", "PROMPTKIT_REQUIREMENTS_SECRET",
|
||||||
|
} {
|
||||||
|
if strings.Contains(formatted, marker) {
|
||||||
|
t.Fatalf("inspection disclosed %q: %s", marker, formatted)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
15
requirements_test.go
Normal file
15
requirements_test.go
Normal file
@@ -0,0 +1,15 @@
|
|||||||
|
package promptkit
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestInspectRequirementsRejectsNilEngine(t *testing.T) {
|
||||||
|
var engine *Engine
|
||||||
|
inspection, err := engine.InspectRequirements(context.Background(), RequirementInspectionRequest{PromptID: "prompt"})
|
||||||
|
if inspection != nil || !errors.Is(err, ErrInvalidConfig) {
|
||||||
|
t.Fatalf("nil engine returned (%+v, %v), want ErrInvalidConfig", inspection, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
161
scripts/check-examples.py
Normal file
161
scripts/check-examples.py
Normal file
@@ -0,0 +1,161 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Run the maintained consumer examples and check their semantic output."""
|
||||||
|
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
|
||||||
|
REPOSITORY_ROOT = Path(__file__).resolve().parent.parent
|
||||||
|
|
||||||
|
|
||||||
|
def example_environment():
|
||||||
|
environment = os.environ.copy()
|
||||||
|
environment["GOTOOLCHAIN"] = "local"
|
||||||
|
try:
|
||||||
|
host = subprocess.run(
|
||||||
|
["go", "env", "GOHOSTOS", "GOHOSTARCH"],
|
||||||
|
cwd=REPOSITORY_ROOT,
|
||||||
|
env=environment,
|
||||||
|
text=True,
|
||||||
|
capture_output=True,
|
||||||
|
check=False,
|
||||||
|
)
|
||||||
|
except OSError as error:
|
||||||
|
raise RuntimeError(f"could not run go env: {error}") from error
|
||||||
|
|
||||||
|
if host.returncode != 0:
|
||||||
|
raise RuntimeError(
|
||||||
|
"go env GOHOSTOS GOHOSTARCH failed "
|
||||||
|
f"with exit status {host.returncode}: {host.stderr.strip()}"
|
||||||
|
)
|
||||||
|
|
||||||
|
values = host.stdout.splitlines()
|
||||||
|
if len(values) != 2 or not all(values):
|
||||||
|
raise RuntimeError(
|
||||||
|
"go env GOHOSTOS GOHOSTARCH returned unexpected output: "
|
||||||
|
f"{host.stdout!r}"
|
||||||
|
)
|
||||||
|
|
||||||
|
environment["GOOS"], environment["GOARCH"] = values
|
||||||
|
return environment
|
||||||
|
|
||||||
|
|
||||||
|
def run_example(name, environment, failures):
|
||||||
|
command = ["go", "run", f"./examples/go-library/{name}"]
|
||||||
|
try:
|
||||||
|
result = subprocess.run(
|
||||||
|
command,
|
||||||
|
cwd=REPOSITORY_ROOT,
|
||||||
|
env=environment,
|
||||||
|
text=True,
|
||||||
|
capture_output=True,
|
||||||
|
check=False,
|
||||||
|
)
|
||||||
|
except OSError as error:
|
||||||
|
failures.append(f"{name}: could not start {command[0]}: {error}")
|
||||||
|
return None
|
||||||
|
|
||||||
|
if result.returncode != 0:
|
||||||
|
failures.append(f"{name}: command exited with status {result.returncode}")
|
||||||
|
if result.stderr:
|
||||||
|
failures.append(f"{name}: stderr:\n{result.stderr.rstrip()}")
|
||||||
|
if result.stdout:
|
||||||
|
failures.append(f"{name}: stdout:\n{result.stdout.rstrip()}")
|
||||||
|
return None
|
||||||
|
|
||||||
|
try:
|
||||||
|
value = json.loads(result.stdout)
|
||||||
|
except json.JSONDecodeError as error:
|
||||||
|
failures.append(f"{name}: output is not valid JSON: {error}")
|
||||||
|
if result.stdout:
|
||||||
|
failures.append(f"{name}: stdout:\n{result.stdout.rstrip()}")
|
||||||
|
return None
|
||||||
|
|
||||||
|
if not isinstance(value, dict):
|
||||||
|
failures.append(
|
||||||
|
f"{name}: expected a JSON object, got {type(value).__name__}"
|
||||||
|
)
|
||||||
|
return None
|
||||||
|
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def expect_equal(example_name, value, key, expected, failures):
|
||||||
|
actual = value.get(key)
|
||||||
|
matches = actual is expected if isinstance(expected, bool) else actual == expected
|
||||||
|
if not matches:
|
||||||
|
failures.append(
|
||||||
|
f"{example_name}: expected {key}={expected!r}, got {actual!r}"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def expect_positive(example_name, value, key, failures):
|
||||||
|
actual = value.get(key)
|
||||||
|
try:
|
||||||
|
matches = actual > 0
|
||||||
|
except TypeError:
|
||||||
|
matches = False
|
||||||
|
if not matches:
|
||||||
|
failures.append(f"{example_name}: expected {key} to be positive, got {actual!r}")
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
try:
|
||||||
|
environment = example_environment()
|
||||||
|
except RuntimeError as error:
|
||||||
|
print(error, file=sys.stderr)
|
||||||
|
return 1
|
||||||
|
|
||||||
|
failures = []
|
||||||
|
prepared = run_example("prepare", environment, failures)
|
||||||
|
result = run_example("run", environment, failures)
|
||||||
|
observed = run_example("observe", environment, failures)
|
||||||
|
|
||||||
|
if prepared is not None:
|
||||||
|
expect_equal("prepare", prepared, "prompt_id", "example.prepare", failures)
|
||||||
|
expect_equal(
|
||||||
|
"prepare", prepared, "selected_profile", "offline-example", failures
|
||||||
|
)
|
||||||
|
expect_equal("prepare", prepared, "model", "offline-model", failures)
|
||||||
|
expect_positive("prepare", prepared, "message_count", failures)
|
||||||
|
|
||||||
|
if result is not None:
|
||||||
|
expect_equal(
|
||||||
|
"run", result, "output", "Ada finished the migration review.", failures
|
||||||
|
)
|
||||||
|
expect_equal("run", result, "is_valid", True, failures)
|
||||||
|
expect_equal("run", result, "validation_status", "passed", failures)
|
||||||
|
expect_equal("run", result, "model", "offline-model", failures)
|
||||||
|
expect_equal("run", result, "total_tokens", 18, failures)
|
||||||
|
|
||||||
|
if observed is not None:
|
||||||
|
expect_equal("observe", observed, "proposal_attempts", 2, failures)
|
||||||
|
expect_equal(
|
||||||
|
"observe", observed, "proposal_transient_retries", 1, failures
|
||||||
|
)
|
||||||
|
expect_equal(
|
||||||
|
"observe", observed, "proposal_usage_complete", False, failures
|
||||||
|
)
|
||||||
|
expect_equal("observe", observed, "assessment_attempts", 1, failures)
|
||||||
|
expect_equal(
|
||||||
|
"observe", observed, "assessment_transient_retries", 0, failures
|
||||||
|
)
|
||||||
|
expect_equal(
|
||||||
|
"observe", observed, "assessment_outcome", "accepted", failures
|
||||||
|
)
|
||||||
|
|
||||||
|
if failures:
|
||||||
|
print("Consumer example checks failed:", file=sys.stderr)
|
||||||
|
for failure in failures:
|
||||||
|
print(f"- {failure}", file=sys.stderr)
|
||||||
|
return 1
|
||||||
|
|
||||||
|
print("Consumer example checks passed: prepare, run, observe")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
@@ -1,152 +0,0 @@
|
|||||||
#!/bin/sh
|
|
||||||
# Run local checks without publishing or changing files.
|
|
||||||
set -eu
|
|
||||||
cd "$(dirname "$0")/.."
|
|
||||||
|
|
||||||
go test -race -vet=off ./...
|
|
||||||
go vet ./...
|
|
||||||
python3 - <<'PY'
|
|
||||||
import json
|
|
||||||
import subprocess
|
|
||||||
|
|
||||||
def example(name):
|
|
||||||
return json.loads(subprocess.check_output(
|
|
||||||
["go", "run", "./examples/go-library/" + name], text=True))
|
|
||||||
|
|
||||||
prepared = example("prepare")
|
|
||||||
assert prepared["prompt_id"] == "example.prepare", prepared
|
|
||||||
assert prepared["selected_profile"] == "offline-example", prepared
|
|
||||||
assert prepared["model"] == "offline-model", prepared
|
|
||||||
assert prepared["message_count"] > 0, prepared
|
|
||||||
result = example("run")
|
|
||||||
assert result["output"] == "Ada finished the migration review.", result
|
|
||||||
assert result["is_valid"] is True, result
|
|
||||||
assert result["validation_status"] == "passed", result
|
|
||||||
assert result["model"] == "offline-model", result
|
|
||||||
assert result["total_tokens"] == 18, result
|
|
||||||
observed = example("observe")
|
|
||||||
assert observed["proposal_attempts"] == 2, observed
|
|
||||||
assert observed["proposal_transient_retries"] == 1, observed
|
|
||||||
assert observed["proposal_usage_complete"] is False, observed
|
|
||||||
assert observed["assessment_attempts"] == 1, observed
|
|
||||||
assert observed["assessment_transient_retries"] == 0, observed
|
|
||||||
assert observed["assessment_outcome"] == "accepted", observed
|
|
||||||
PY
|
|
||||||
|
|
||||||
unformatted=$(
|
|
||||||
git ls-files '*.go' |
|
|
||||||
while IFS= read -r go_file
|
|
||||||
do
|
|
||||||
gofmt -l "$go_file"
|
|
||||||
done
|
|
||||||
)
|
|
||||||
test -z "$unformatted"
|
|
||||||
|
|
||||||
python3 - <<'PY'
|
|
||||||
from pathlib import Path
|
|
||||||
import re
|
|
||||||
import subprocess
|
|
||||||
import sys
|
|
||||||
from urllib.parse import unquote
|
|
||||||
|
|
||||||
root = Path.cwd().resolve()
|
|
||||||
markdown_files = [
|
|
||||||
root / name
|
|
||||||
for name in subprocess.check_output(
|
|
||||||
["git", "ls-files", "*.md"], text=True
|
|
||||||
).splitlines()
|
|
||||||
]
|
|
||||||
link_pattern = re.compile(r"!?\[[^]]*\]\(([^)]+)\)")
|
|
||||||
heading_pattern = re.compile(r"^#{1,6}\s+(.+?)\s*#*\s*$")
|
|
||||||
scheme_pattern = re.compile(r"^[a-z][a-z0-9+.-]*:", re.IGNORECASE)
|
|
||||||
|
|
||||||
|
|
||||||
def markdown_lines(path):
|
|
||||||
in_fence = False
|
|
||||||
fence = ""
|
|
||||||
for line in path.read_text(encoding="utf-8").splitlines():
|
|
||||||
stripped = line.lstrip()
|
|
||||||
marker = stripped[:3]
|
|
||||||
if marker in {"```", "~~~"}:
|
|
||||||
if not in_fence:
|
|
||||||
in_fence = True
|
|
||||||
fence = marker
|
|
||||||
elif marker == fence:
|
|
||||||
in_fence = False
|
|
||||||
fence = ""
|
|
||||||
continue
|
|
||||||
if not in_fence:
|
|
||||||
yield line
|
|
||||||
|
|
||||||
|
|
||||||
anchor_cache = {}
|
|
||||||
|
|
||||||
|
|
||||||
def anchors(path):
|
|
||||||
if path in anchor_cache:
|
|
||||||
return anchor_cache[path]
|
|
||||||
found = set()
|
|
||||||
counts = {}
|
|
||||||
for line in markdown_lines(path):
|
|
||||||
match = heading_pattern.match(line)
|
|
||||||
if not match:
|
|
||||||
continue
|
|
||||||
heading = re.sub(r"<[^>]+>", "", match.group(1)).replace("`", "")
|
|
||||||
base = re.sub(r"[^\w\- ]", "", heading.lower()).replace(" ", "-")
|
|
||||||
count = counts.get(base, 0)
|
|
||||||
counts[base] = count + 1
|
|
||||||
found.add(base if count == 0 else f"{base}-{count}")
|
|
||||||
anchor_cache[path] = found
|
|
||||||
return found
|
|
||||||
|
|
||||||
|
|
||||||
failures = []
|
|
||||||
for source in markdown_files:
|
|
||||||
text = "\n".join(markdown_lines(source))
|
|
||||||
for match in link_pattern.finditer(text):
|
|
||||||
target = match.group(1).strip()
|
|
||||||
if target.startswith("<") and target.endswith(">"):
|
|
||||||
target = target[1:-1]
|
|
||||||
if scheme_pattern.match(target) or target.startswith("//"):
|
|
||||||
continue
|
|
||||||
path_text, separator, fragment = target.partition("#")
|
|
||||||
destination = source if not path_text else source.parent / unquote(path_text)
|
|
||||||
try:
|
|
||||||
destination = destination.resolve()
|
|
||||||
destination.relative_to(root)
|
|
||||||
except ValueError:
|
|
||||||
failures.append(f"{source.relative_to(root)}: escapes repository: {target}")
|
|
||||||
continue
|
|
||||||
if not destination.exists():
|
|
||||||
failures.append(f"{source.relative_to(root)}: missing target: {target}")
|
|
||||||
continue
|
|
||||||
if separator and destination.suffix.lower() == ".md":
|
|
||||||
fragment = unquote(fragment).lower()
|
|
||||||
if fragment not in anchors(destination):
|
|
||||||
failures.append(f"{source.relative_to(root)}: missing anchor: {target}")
|
|
||||||
|
|
||||||
if failures:
|
|
||||||
print("\n".join(failures), file=sys.stderr)
|
|
||||||
raise SystemExit(1)
|
|
||||||
PY
|
|
||||||
|
|
||||||
case "$(go env GOWORK)" in
|
|
||||||
''|off) ;;
|
|
||||||
*) printf '%s\n' 'an active Go workspace is not allowed' >&2; exit 1 ;;
|
|
||||||
esac
|
|
||||||
test -z "$(git ls-files go.work go.work.sum)"
|
|
||||||
test ! -e vendor
|
|
||||||
if grep -Eq '^[[:space:]]*replace([[:space:]]|\()' go.mod
|
|
||||||
then
|
|
||||||
printf '%s\n' 'go.mod contains a replacement' >&2
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
git diff --check
|
|
||||||
git diff --cached --check
|
|
||||||
credential_pattern='-----BEGIN ([A-Z0-9]+ )?PRIV''ATE KEY-----|AKI''A[0-9A-Z]{16}|gh[pousr]_[A-Za-z0-9]{36,}|sk-[A-Za-z0-9]{32,}'
|
|
||||||
if git grep -nEI -e "$credential_pattern" -- .
|
|
||||||
then
|
|
||||||
printf '%s\n' 'possible credential found' >&2
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
@@ -1,57 +1,5 @@
|
|||||||
#!/bin/sh
|
#!/bin/sh
|
||||||
# Validate one committed candidate and publish only its selected annotated tag.
|
|
||||||
set -eu
|
set -eu
|
||||||
|
|
||||||
cd "$(dirname "$0")/.."
|
cd "$(dirname "$0")/.."
|
||||||
|
exec ./scripts/releasekit.sh publish "$@"
|
||||||
release_version=${1:?usage: scripts/release.sh vMAJOR.MINOR.PATCH}
|
|
||||||
printf '%s\n' "$release_version" |
|
|
||||||
grep -Eq '^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$'
|
|
||||||
release_commit=$(git rev-parse --verify 'HEAD^{commit}')
|
|
||||||
|
|
||||||
check_candidate() {
|
|
||||||
test "$(git branch --show-current)" = main
|
|
||||||
test -z "$(git status --porcelain)"
|
|
||||||
test "$(git rev-parse HEAD)" = "$release_commit"
|
|
||||||
git fetch origin main
|
|
||||||
test "$(git rev-parse refs/remotes/origin/main)" = "$release_commit"
|
|
||||||
if git show-ref --verify --quiet "refs/tags/$release_version"; then
|
|
||||||
printf '%s\n' "local tag already exists: $release_version" >&2
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
remote_refs=$(git ls-remote --tags origin "refs/tags/$release_version" "refs/tags/$release_version^{}")
|
|
||||||
test -z "$remote_refs"
|
|
||||||
}
|
|
||||||
|
|
||||||
check_candidate
|
|
||||||
python3 - "$release_version" <<'PY'
|
|
||||||
from pathlib import Path
|
|
||||||
import sys
|
|
||||||
|
|
||||||
version = sys.argv[1]
|
|
||||||
note = Path("docs/releases") / (version + ".md")
|
|
||||||
lines = note.read_text(encoding="utf-8").splitlines()
|
|
||||||
assert lines and lines[0] == "# Promptkit " + version, f"wrong release title: {note}"
|
|
||||||
headings = set()
|
|
||||||
fence = None
|
|
||||||
for line in lines:
|
|
||||||
marker = line.lstrip()[:3]
|
|
||||||
if marker in {"```", "~~~"}:
|
|
||||||
if fence is None:
|
|
||||||
fence = marker
|
|
||||||
elif fence == marker:
|
|
||||||
fence = None
|
|
||||||
elif fence is None:
|
|
||||||
headings.add(line)
|
|
||||||
for section in ("Summary", "Compatibility", "Upgrade", "Changes"):
|
|
||||||
assert "## " + section in headings, f"missing {section}: {note}"
|
|
||||||
PY
|
|
||||||
|
|
||||||
./scripts/release-check.sh
|
|
||||||
check_candidate
|
|
||||||
release_note=$(git show "$release_commit:docs/releases/$release_version.md")
|
|
||||||
printf '%s\n\nValidated commit: %s\n' "$release_note" "$release_commit" |
|
|
||||||
git tag --annotate --cleanup=verbatim "$release_version" --file - "$release_commit"
|
|
||||||
test "$(git cat-file -t "refs/tags/$release_version")" = tag
|
|
||||||
test "$(git rev-parse "refs/tags/$release_version^{commit}")" = "$release_commit"
|
|
||||||
git push origin "refs/tags/$release_version:refs/tags/$release_version"
|
|
||||||
printf '%s\n' "Released $release_version. No CI or module-availability wait is required."
|
|
||||||
|
|||||||
18
scripts/releasekit.sh
Executable file
18
scripts/releasekit.sh
Executable file
@@ -0,0 +1,18 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
set -eu
|
||||||
|
|
||||||
|
cd "$(dirname "$0")/.."
|
||||||
|
|
||||||
|
export GOTOOLCHAIN=local
|
||||||
|
export GOWORK=off
|
||||||
|
export GOFLAGS=-mod=readonly
|
||||||
|
|
||||||
|
GOOS=$(go env GOHOSTOS)
|
||||||
|
GOARCH=$(go env GOHOSTARCH)
|
||||||
|
export GOOS GOARCH
|
||||||
|
|
||||||
|
mkdir -p .releasekit/bin
|
||||||
|
go build -mod=readonly -o .releasekit/bin/releasekit \
|
||||||
|
"gitea.maximumdirect.net/eric/ReleaseKit/cmd/releasekit"
|
||||||
|
|
||||||
|
exec ./.releasekit/bin/releasekit "$@"
|
||||||
476
scripts/test-release.py
Normal file
476
scripts/test-release.py
Normal file
@@ -0,0 +1,476 @@
|
|||||||
|
"""Exercise ReleaseKit publication wiring in disposable repositories."""
|
||||||
|
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import shutil
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
import textwrap
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
|
||||||
|
REPOSITORY_ROOT = Path(__file__).resolve().parent.parent
|
||||||
|
RELEASE_VERSION = "v0.0.1"
|
||||||
|
FAILED_RELEASE_VERSION = "v0.0.2"
|
||||||
|
RELEASEKIT_MODULE = "gitea.maximumdirect.net/eric/ReleaseKit"
|
||||||
|
|
||||||
|
|
||||||
|
def execute(command, cwd, environment):
|
||||||
|
try:
|
||||||
|
return subprocess.run(
|
||||||
|
command,
|
||||||
|
cwd=cwd,
|
||||||
|
env=environment,
|
||||||
|
text=True,
|
||||||
|
capture_output=True,
|
||||||
|
check=False,
|
||||||
|
)
|
||||||
|
except OSError as error:
|
||||||
|
raise RuntimeError(f"could not run {command[0]}: {error}") from error
|
||||||
|
|
||||||
|
|
||||||
|
def run(command, cwd, environment):
|
||||||
|
result = execute(command, cwd, environment)
|
||||||
|
if result.returncode != 0:
|
||||||
|
details = [
|
||||||
|
f"command failed with status {result.returncode}: {' '.join(command)}"
|
||||||
|
]
|
||||||
|
if result.stdout:
|
||||||
|
details.append(f"stdout:\n{result.stdout.rstrip()}")
|
||||||
|
if result.stderr:
|
||||||
|
details.append(f"stderr:\n{result.stderr.rstrip()}")
|
||||||
|
raise RuntimeError("\n".join(details))
|
||||||
|
return result.stdout
|
||||||
|
|
||||||
|
|
||||||
|
def write_file(path, contents):
|
||||||
|
path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
path.write_text(textwrap.dedent(contents).lstrip("\n"), encoding="utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
def repo_git(repository, environment, *arguments):
|
||||||
|
return run(
|
||||||
|
["git", "-C", str(repository), *arguments],
|
||||||
|
cwd=repository.parent,
|
||||||
|
environment=environment,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def remote_git(remote, temporary_root, environment, *arguments):
|
||||||
|
return run(
|
||||||
|
["git", "--git-dir", str(remote), *arguments],
|
||||||
|
cwd=temporary_root,
|
||||||
|
environment=environment,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def fixture_sources(repository, environment):
|
||||||
|
write_file(
|
||||||
|
repository / "fixture.go",
|
||||||
|
"""
|
||||||
|
package fixture
|
||||||
|
|
||||||
|
func candidateValue() string { return "ready" }
|
||||||
|
""",
|
||||||
|
)
|
||||||
|
write_file(
|
||||||
|
repository / "fixture_test.go",
|
||||||
|
f"""
|
||||||
|
package fixture
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"os"
|
||||||
|
"runtime"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestCandidateSuite(t *testing.T) {{
|
||||||
|
if candidateValue() != "ready" {{
|
||||||
|
t.Fatal("candidate package is not ready")
|
||||||
|
}}
|
||||||
|
marker := os.Getenv("FIXTURE_SUITE_MARKER")
|
||||||
|
if marker == "" {{
|
||||||
|
t.Fatal("FIXTURE_SUITE_MARKER is not set")
|
||||||
|
}}
|
||||||
|
record := struct {{
|
||||||
|
GoVersion string `json:"go_version"`
|
||||||
|
Race bool `json:"race"`
|
||||||
|
}}{{GoVersion: runtime.Version(), Race: raceDetectorEnabled}}
|
||||||
|
data, err := json.Marshal(record)
|
||||||
|
if err != nil {{
|
||||||
|
t.Fatal(err)
|
||||||
|
}}
|
||||||
|
file, err := os.OpenFile(marker, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600)
|
||||||
|
if err != nil {{
|
||||||
|
t.Fatal(err)
|
||||||
|
}}
|
||||||
|
if _, err := file.Write(append(data, '\\n')); err != nil {{
|
||||||
|
file.Close()
|
||||||
|
t.Fatal(err)
|
||||||
|
}}
|
||||||
|
if err := file.Close(); err != nil {{
|
||||||
|
t.Fatal(err)
|
||||||
|
}}
|
||||||
|
}}
|
||||||
|
""",
|
||||||
|
)
|
||||||
|
write_file(
|
||||||
|
repository / "race_enabled_test.go",
|
||||||
|
"""
|
||||||
|
//go:build race
|
||||||
|
|
||||||
|
package fixture
|
||||||
|
|
||||||
|
const raceDetectorEnabled = true
|
||||||
|
""",
|
||||||
|
)
|
||||||
|
write_file(
|
||||||
|
repository / "race_disabled_test.go",
|
||||||
|
"""
|
||||||
|
//go:build !race
|
||||||
|
|
||||||
|
package fixture
|
||||||
|
|
||||||
|
const raceDetectorEnabled = false
|
||||||
|
""",
|
||||||
|
)
|
||||||
|
go_files = [
|
||||||
|
str(repository / filename)
|
||||||
|
for filename in (
|
||||||
|
"fixture.go",
|
||||||
|
"fixture_test.go",
|
||||||
|
"race_enabled_test.go",
|
||||||
|
"race_disabled_test.go",
|
||||||
|
)
|
||||||
|
]
|
||||||
|
run(["gofmt", "-w", *go_files], cwd=repository, environment=environment)
|
||||||
|
|
||||||
|
|
||||||
|
def prepare_repository(repository, remote, environment, module_bytes, version):
|
||||||
|
run(
|
||||||
|
["git", "init", "--bare", "--initial-branch=main", str(remote)],
|
||||||
|
cwd=repository.parent,
|
||||||
|
environment=environment,
|
||||||
|
)
|
||||||
|
run(
|
||||||
|
["git", "init", "--initial-branch=main", str(repository)],
|
||||||
|
cwd=repository.parent,
|
||||||
|
environment=environment,
|
||||||
|
)
|
||||||
|
repo_git(repository, environment, "config", "user.name", "ReleaseKit Fixture")
|
||||||
|
repo_git(repository, environment, "config", "user.email", "fixture@example.invalid")
|
||||||
|
repo_git(repository, environment, "remote", "add", "origin", str(remote))
|
||||||
|
|
||||||
|
for name, contents in module_bytes.items():
|
||||||
|
(repository / name).write_bytes(contents)
|
||||||
|
|
||||||
|
write_file(
|
||||||
|
repository / ".gitignore",
|
||||||
|
"""
|
||||||
|
.releasekit/
|
||||||
|
""",
|
||||||
|
)
|
||||||
|
write_file(
|
||||||
|
repository / "releasekit.json",
|
||||||
|
(REPOSITORY_ROOT / "releasekit.json").read_text(encoding="utf-8"),
|
||||||
|
)
|
||||||
|
|
||||||
|
scripts = repository / "scripts"
|
||||||
|
scripts.mkdir(parents=True, exist_ok=True)
|
||||||
|
shutil.copy2(REPOSITORY_ROOT / "scripts/release.sh", scripts / "release.sh")
|
||||||
|
shutil.copy2(REPOSITORY_ROOT / "scripts/releasekit.sh", scripts / "releasekit.sh")
|
||||||
|
write_file(
|
||||||
|
scripts / "check-examples.py",
|
||||||
|
"""
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
import sys
|
||||||
|
|
||||||
|
marker = Path(os.environ["FIXTURE_EXAMPLE_MARKER"])
|
||||||
|
with marker.open("a", encoding="utf-8") as stream:
|
||||||
|
stream.write("called\\n")
|
||||||
|
if os.environ.get("FIXTURE_EXAMPLE_FAILURE") == "1":
|
||||||
|
print("fixture example check rejected candidate", file=sys.stderr)
|
||||||
|
raise SystemExit(1)
|
||||||
|
""",
|
||||||
|
)
|
||||||
|
write_file(
|
||||||
|
repository / f"docs/releases/{version}.md",
|
||||||
|
f"""
|
||||||
|
# Promptkit {version}
|
||||||
|
|
||||||
|
Disposable publication fixture.
|
||||||
|
""",
|
||||||
|
)
|
||||||
|
fixture_sources(repository, environment)
|
||||||
|
|
||||||
|
repo_git(repository, environment, "add", "-A")
|
||||||
|
run(
|
||||||
|
["git", "-C", str(repository), "commit", "-m", "Add disposable release fixture"],
|
||||||
|
cwd=repository.parent,
|
||||||
|
environment=environment,
|
||||||
|
)
|
||||||
|
repo_git(repository, environment, "push", "origin", "main")
|
||||||
|
|
||||||
|
|
||||||
|
def verify_publication(repository, remote, temporary_root, environment, module_bytes, selected_go):
|
||||||
|
selected_releasekit = run(
|
||||||
|
["go", "list", "-mod=readonly", "-m", "-f", "{{.Version}}", RELEASEKIT_MODULE],
|
||||||
|
cwd=repository,
|
||||||
|
environment=environment,
|
||||||
|
).strip()
|
||||||
|
publication = repository / "scripts/release.sh"
|
||||||
|
run(
|
||||||
|
[str(publication), RELEASE_VERSION],
|
||||||
|
cwd=temporary_root,
|
||||||
|
environment=environment,
|
||||||
|
)
|
||||||
|
|
||||||
|
suite_lines = Path(environment["FIXTURE_SUITE_MARKER"]).read_text(
|
||||||
|
encoding="utf-8"
|
||||||
|
).splitlines()
|
||||||
|
if len(suite_lines) != 1:
|
||||||
|
raise RuntimeError(f"expected one suite marker, found {len(suite_lines)}")
|
||||||
|
suite_record = json.loads(suite_lines[0])
|
||||||
|
if suite_record != {"go_version": selected_go, "race": True}:
|
||||||
|
raise RuntimeError(f"unexpected suite marker: {suite_record!r}")
|
||||||
|
|
||||||
|
example_lines = Path(environment["FIXTURE_EXAMPLE_MARKER"]).read_text(
|
||||||
|
encoding="utf-8"
|
||||||
|
).splitlines()
|
||||||
|
if example_lines != ["called"]:
|
||||||
|
raise RuntimeError(f"expected one additional-check marker, got {example_lines!r}")
|
||||||
|
|
||||||
|
binary = repository / ".releasekit/bin/releasekit"
|
||||||
|
if not binary.is_file():
|
||||||
|
raise RuntimeError("ReleaseKit bootstrap did not leave its ignored executable")
|
||||||
|
build_info = run(
|
||||||
|
["go", "version", "-m", str(binary)],
|
||||||
|
cwd=temporary_root,
|
||||||
|
environment=environment,
|
||||||
|
)
|
||||||
|
first_line = build_info.splitlines()[0]
|
||||||
|
if selected_go not in first_line:
|
||||||
|
raise RuntimeError(f"tool was built with {first_line!r}, expected {selected_go}")
|
||||||
|
expected_module = f"\tmod\t{RELEASEKIT_MODULE}\t{selected_releasekit}\t"
|
||||||
|
if expected_module not in build_info:
|
||||||
|
raise RuntimeError(
|
||||||
|
f"bootstrap executable does not record selected ReleaseKit {selected_releasekit}"
|
||||||
|
)
|
||||||
|
|
||||||
|
candidate = repo_git(repository, environment, "rev-parse", "HEAD").strip()
|
||||||
|
if repo_git(repository, environment, "rev-parse", "main").strip() != candidate:
|
||||||
|
raise RuntimeError("fixture main moved away from the published candidate")
|
||||||
|
if remote_git(remote, temporary_root, environment, "rev-parse", "refs/heads/main").strip() != candidate:
|
||||||
|
raise RuntimeError("local origin main does not identify the fixture candidate")
|
||||||
|
|
||||||
|
tag_ref = f"refs/tags/{RELEASE_VERSION}"
|
||||||
|
if repo_git(repository, environment, "cat-file", "-t", tag_ref).strip() != "tag":
|
||||||
|
raise RuntimeError("local release ref is not an annotated tag")
|
||||||
|
local_tag = repo_git(repository, environment, "rev-parse", tag_ref).strip()
|
||||||
|
local_tag_commit = repo_git(
|
||||||
|
repository, environment, "rev-parse", f"{tag_ref}^{{}}"
|
||||||
|
).strip()
|
||||||
|
if local_tag_commit != candidate:
|
||||||
|
raise RuntimeError("local annotated tag does not identify the fixture candidate")
|
||||||
|
|
||||||
|
remote_tag = remote_git(remote, temporary_root, environment, "rev-parse", tag_ref).strip()
|
||||||
|
remote_tag_commit = remote_git(
|
||||||
|
remote, temporary_root, environment, "rev-parse", f"{tag_ref}^{{}}"
|
||||||
|
).strip()
|
||||||
|
if remote_tag != local_tag or remote_tag_commit != candidate:
|
||||||
|
raise RuntimeError("remote annotated tag does not identify the fixture candidate")
|
||||||
|
if remote_git(
|
||||||
|
remote, temporary_root, environment, "cat-file", "-t", tag_ref
|
||||||
|
).strip() != "tag":
|
||||||
|
raise RuntimeError("remote release ref is not an annotated tag")
|
||||||
|
|
||||||
|
tag_object = repo_git(repository, environment, "cat-file", "tag", tag_ref)
|
||||||
|
separator = tag_object.find("\n\n")
|
||||||
|
if separator < 0:
|
||||||
|
raise RuntimeError("annotated tag has no message")
|
||||||
|
tag_message = tag_object[separator + 2 :]
|
||||||
|
note = (repository / f"docs/releases/{RELEASE_VERSION}.md").read_text(
|
||||||
|
encoding="utf-8"
|
||||||
|
)
|
||||||
|
if tag_message != note:
|
||||||
|
raise RuntimeError("annotated tag message does not match the committed release note")
|
||||||
|
|
||||||
|
for name, contents in module_bytes.items():
|
||||||
|
if (repository / name).read_bytes() != contents:
|
||||||
|
raise RuntimeError(f"publication changed {name}")
|
||||||
|
|
||||||
|
status = repo_git(
|
||||||
|
repository, environment, "status", "--porcelain", "--untracked-files=all"
|
||||||
|
)
|
||||||
|
if status:
|
||||||
|
raise RuntimeError(f"fixture checkout is not clean after publication:\n{status}")
|
||||||
|
repo_git(repository, environment, "check-ignore", "-q", ".releasekit/bin/releasekit")
|
||||||
|
|
||||||
|
|
||||||
|
def verify_validation_failure(
|
||||||
|
repository, remote, temporary_root, environment, module_bytes, selected_go
|
||||||
|
):
|
||||||
|
scenario_environment = environment.copy()
|
||||||
|
suite_marker = temporary_root / "failure-suite.jsonl"
|
||||||
|
example_marker = temporary_root / "failure-example.log"
|
||||||
|
scenario_environment.update(
|
||||||
|
{
|
||||||
|
"FIXTURE_SUITE_MARKER": str(suite_marker),
|
||||||
|
"FIXTURE_EXAMPLE_MARKER": str(example_marker),
|
||||||
|
"FIXTURE_EXAMPLE_FAILURE": "1",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
prepare_repository(
|
||||||
|
repository,
|
||||||
|
remote,
|
||||||
|
scenario_environment,
|
||||||
|
module_bytes,
|
||||||
|
FAILED_RELEASE_VERSION,
|
||||||
|
)
|
||||||
|
|
||||||
|
result = execute(
|
||||||
|
[str(repository / "scripts/release.sh"), FAILED_RELEASE_VERSION],
|
||||||
|
cwd=temporary_root,
|
||||||
|
environment=scenario_environment,
|
||||||
|
)
|
||||||
|
if result.returncode != 1:
|
||||||
|
raise RuntimeError(
|
||||||
|
"validation failure returned status "
|
||||||
|
f"{result.returncode}, want 1; stdout:\n{result.stdout}\nstderr:\n{result.stderr}"
|
||||||
|
)
|
||||||
|
if "fixture example check rejected candidate" not in result.stderr:
|
||||||
|
raise RuntimeError(f"failure diagnostic omitted useful context:\n{result.stderr}")
|
||||||
|
|
||||||
|
suite_lines = suite_marker.read_text(encoding="utf-8").splitlines()
|
||||||
|
if len(suite_lines) != 1:
|
||||||
|
raise RuntimeError(f"expected one failure-case suite marker, found {len(suite_lines)}")
|
||||||
|
suite_record = json.loads(suite_lines[0])
|
||||||
|
if suite_record != {"go_version": selected_go, "race": True}:
|
||||||
|
raise RuntimeError(f"unexpected failure-case suite marker: {suite_record!r}")
|
||||||
|
if example_marker.read_text(encoding="utf-8").splitlines() != ["called"]:
|
||||||
|
raise RuntimeError("expected one additional-check invocation before failure")
|
||||||
|
|
||||||
|
tag_ref = f"refs/tags/{FAILED_RELEASE_VERSION}"
|
||||||
|
for description, command in (
|
||||||
|
(
|
||||||
|
"local",
|
||||||
|
["git", "-C", str(repository), "show-ref", "--verify", "--quiet", tag_ref],
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"remote",
|
||||||
|
["git", "--git-dir", str(remote), "show-ref", "--verify", "--quiet", tag_ref],
|
||||||
|
),
|
||||||
|
):
|
||||||
|
result = execute(command, cwd=temporary_root, environment=scenario_environment)
|
||||||
|
if result.returncode == 0:
|
||||||
|
raise RuntimeError(f"validation failure created the {description} tag {tag_ref}")
|
||||||
|
if result.returncode != 1:
|
||||||
|
raise RuntimeError(
|
||||||
|
f"could not verify {description} tag absence: "
|
||||||
|
f"status {result.returncode}: {result.stderr.strip()}"
|
||||||
|
)
|
||||||
|
|
||||||
|
for name, contents in module_bytes.items():
|
||||||
|
if (repository / name).read_bytes() != contents:
|
||||||
|
raise RuntimeError(f"failed publication changed {name}")
|
||||||
|
status = repo_git(
|
||||||
|
repository,
|
||||||
|
scenario_environment,
|
||||||
|
"status",
|
||||||
|
"--porcelain",
|
||||||
|
"--untracked-files=all",
|
||||||
|
)
|
||||||
|
if status:
|
||||||
|
raise RuntimeError(f"failure fixture checkout is not clean:\n{status}")
|
||||||
|
|
||||||
|
|
||||||
|
def verify_usage_classification(binary, temporary_root, environment):
|
||||||
|
result = execute(
|
||||||
|
[str(binary), "unknown"], cwd=temporary_root, environment=environment
|
||||||
|
)
|
||||||
|
if result.returncode != 2:
|
||||||
|
raise RuntimeError(
|
||||||
|
f"unsupported command returned status {result.returncode}, want 2; "
|
||||||
|
f"stdout:\n{result.stdout}\nstderr:\n{result.stderr}"
|
||||||
|
)
|
||||||
|
if result.stdout:
|
||||||
|
raise RuntimeError(f"unsupported command wrote to stdout: {result.stdout!r}")
|
||||||
|
if 'unknown command "unknown"' not in result.stderr or "usage: releasekit" not in result.stderr:
|
||||||
|
raise RuntimeError(f"unsupported command diagnostic was not useful:\n{result.stderr}")
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
environment = os.environ.copy()
|
||||||
|
environment.update(
|
||||||
|
{
|
||||||
|
"GOTOOLCHAIN": "local",
|
||||||
|
"GOWORK": "off",
|
||||||
|
"GOPROXY": "off",
|
||||||
|
"GONOPROXY": "none",
|
||||||
|
"GOSUMDB": "off",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
selected_go = run(
|
||||||
|
["go", "env", "GOVERSION"],
|
||||||
|
cwd=REPOSITORY_ROOT,
|
||||||
|
environment=environment,
|
||||||
|
).strip()
|
||||||
|
if not selected_go:
|
||||||
|
print("go env GOVERSION returned no version", file=sys.stderr)
|
||||||
|
return 1
|
||||||
|
|
||||||
|
ignored_rule = ".releasekit/"
|
||||||
|
ignore_lines = (REPOSITORY_ROOT / ".gitignore").read_text(
|
||||||
|
encoding="utf-8"
|
||||||
|
).splitlines()
|
||||||
|
if ignored_rule not in ignore_lines:
|
||||||
|
print(".gitignore does not contain the ReleaseKit output rule", file=sys.stderr)
|
||||||
|
return 1
|
||||||
|
|
||||||
|
module_bytes = {
|
||||||
|
name: (REPOSITORY_ROOT / name).read_bytes() for name in ("go.mod", "go.sum")
|
||||||
|
}
|
||||||
|
with tempfile.TemporaryDirectory(prefix="promptkit-release-fixture-") as directory:
|
||||||
|
temporary_root = Path(directory)
|
||||||
|
repository = temporary_root / "checkout"
|
||||||
|
remote = temporary_root / "origin.git"
|
||||||
|
suite_marker = temporary_root / "suite.jsonl"
|
||||||
|
example_marker = temporary_root / "example.log"
|
||||||
|
environment["FIXTURE_SUITE_MARKER"] = str(suite_marker)
|
||||||
|
environment["FIXTURE_EXAMPLE_MARKER"] = str(example_marker)
|
||||||
|
try:
|
||||||
|
prepare_repository(
|
||||||
|
repository, remote, environment, module_bytes, RELEASE_VERSION
|
||||||
|
)
|
||||||
|
verify_publication(
|
||||||
|
repository,
|
||||||
|
remote,
|
||||||
|
temporary_root,
|
||||||
|
environment,
|
||||||
|
module_bytes,
|
||||||
|
selected_go,
|
||||||
|
)
|
||||||
|
verify_usage_classification(
|
||||||
|
repository / ".releasekit/bin/releasekit", temporary_root, environment
|
||||||
|
)
|
||||||
|
verify_validation_failure(
|
||||||
|
temporary_root / "failure-checkout",
|
||||||
|
temporary_root / "failure-origin.git",
|
||||||
|
temporary_root,
|
||||||
|
environment,
|
||||||
|
module_bytes,
|
||||||
|
selected_go,
|
||||||
|
)
|
||||||
|
except (RuntimeError, OSError, json.JSONDecodeError) as error:
|
||||||
|
print(error, file=sys.stderr)
|
||||||
|
return 1
|
||||||
|
|
||||||
|
print("ReleaseKit publication success and failure fixtures passed")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
Reference in New Issue
Block a user