165 Commits

Author SHA1 Message Date
9178f3eb91 Repair stale documentation anchors for release 2026-10-17 14:35:52 +00:00
a70fd7bb65 Document the v0.16.0 release and migration 2026-10-17 14:33:22 +00:00
7091b51c87 Retire completed implementation and review roadmaps 2026-10-17 14:31:29 +00:00
8627c751a4 Clarify shared ownership of namespace views 2026-10-17 13:52:50 +00:00
6fff3d822c Detach version labels in prepared execution snapshots 2026-10-17 13:51:30 +00:00
a1f12f3185 Reuse origin metadata across configuration collections 2026-10-17 13:46:58 +00:00
165cef7aec Record configuration collections review findings 2026-10-17 13:37:52 +00:00
0221cd4610 Verify scoped collection behavior and complete roadmap 2026-10-17 06:39:24 +00:00
dfd8da843e Clarify collection contracts and migration docs 2026-10-17 06:35:56 +00:00
29f104fbc0 Add collection migration example and guide 2026-10-17 06:29:57 +00:00
8669294cf6 Add namespace and collection inventories 2026-10-17 06:25:14 +00:00
c9b623b374 Add scoped identity listings and backend inventory 2026-10-17 06:22:22 +00:00
15f79ce7b1 Apply namespace scopes to inspections and diagnostics 2026-10-17 06:17:16 +00:00
aa54b8d1c4 Add immutable namespace views 2026-10-17 06:14:09 +00:00
60c58260ec Use caller profile in inspection test 2026-10-17 06:13:07 +00:00
8dfe35a1f0 Add collection input helper 2026-10-17 06:09:52 +00:00
13b60a72ac Remove bundled catalog integration 2026-10-17 06:08:35 +00:00
ee40a1bdd2 Decouple application tests from bundled profiles 2026-10-17 06:05:27 +00:00
473fc1f6d7 Assemble collection resources by qualified ID 2026-10-17 06:03:28 +00:00
a10a9053b7 Enable shared configuration version checks 2026-10-17 05:56:54 +00:00
ec3245c97e Carry optional prompt labels through results 2026-10-17 05:54:32 +00:00
8cbb7c0daf Remove prompt version selectors 2026-10-17 05:47:14 +00:00
d40a010538 Store one prompt definition per ID 2026-10-17 05:41:21 +00:00
0cd833513c Pass effective paths through configuration loading 2026-10-17 05:33:03 +00:00
5e58b6944b Add shared resource naming primitives 2026-10-17 05:30:26 +00:00
0a85ac8cd1 Adopt ConfigKit v0.8.0 2026-10-17 05:29:24 +00:00
89af2a1281 Plan configuration collections implementation 2026-10-17 05:26:42 +00:00
683689a4f8 Consolidate collection and namespace roadmaps 2026-10-17 05:13:16 +00:00
ddda2f8767 Settle collection configuration decisions around simplicity 2026-10-16 13:50:40 +00:00
190ec69345 Retire unselected collection configuration proposals 2026-10-16 11:59:34 +00:00
b5866dd749 Align collection proposal with implemented ConfigKit capabilities 2026-10-16 11:51:18 +00:00
c7490b783b Propose collections built from mounted configuration fragments 2026-10-14 02:24:50 +00:00
c7d1b0dca7 Propose identity-based collection records for ConfigKit 2026-10-14 01:32:10 +00:00
9e5ed08635 Propose collection configuration using current ConfigKit 2026-10-14 01:15:13 +00:00
4bddbbe8f4 Record collection ownership and ConfigKit boundary decisions 2026-10-14 00:19:05 +00:00
1516631a30 Record accepted namespace scope and filtering decisions 2026-10-13 22:47:21 +00:00
cf997c2034 Explain namespace design recommendations and alternatives 2026-10-13 21:33:21 +00:00
a0ec67c256 Reserve slash for namespace-qualified identifiers 2026-10-13 21:29:36 +00:00
75891b6561 Document namespace and scoped lookup roadmap 2026-10-13 21:25:57 +00:00
5dc6370a97 Avoid redundant smoke report copying 2026-10-13 21:12:17 +00:00
4cda3674c4 Record redundant smoke report copying 2026-10-13 20:50:54 +00:00
5ea6f06a6d Report known zero usage when smoke tests do not dispatch 2026-10-13 20:32:24 +00:00
116bd0e39b Report actual inference smoke test destinations 2026-10-13 20:27:46 +00:00
208ff42915 Preserve caller cancellation in HTTP smoke probes 2026-10-13 20:22:06 +00:00
972a103a97 Record smoke test review findings 2026-10-13 19:59:18 +00:00
2f735aedf9 Verify smoke-test release candidate 2026-10-13 18:38:45 +00:00
f4c721d934 Document smoke-test diagnostics 2026-10-13 18:32:41 +00:00
b7a951685b Verify smoke-test integration boundaries 2026-10-13 18:24:57 +00:00
abb2461244 Expose the public smoke-test API 2026-10-13 18:16:34 +00:00
b268c55783 Integrate inference smoke diagnostics 2026-10-13 18:09:20 +00:00
f0182fb2c9 Assemble HTTP smoke-test reports 2026-10-13 17:48:21 +00:00
8d05b594e2 Add private inference smoke cases 2026-10-13 17:30:45 +00:00
edaaecc982 Add OpenRouter authentication diagnostics 2026-10-13 17:23:47 +00:00
2e81c9e61b Add the unauthenticated reachability probe 2026-10-13 17:12:31 +00:00
2b723760d1 Retain injected client diagnostic evidence 2026-10-13 17:05:40 +00:00
c5aed7a451 Carry evidence through built-in generation retries 2026-10-13 16:56:56 +00:00
d91afc61a4 Retain generation response evidence 2026-10-13 16:47:26 +00:00
6eeb0ee9c0 Capture destination-specific HTTP evidence 2026-10-13 16:38:29 +00:00
b9c232ee63 Adopt tagged diagnostic catalogs 2026-10-13 16:28:41 +00:00
7291d23587 Record RakestrawHome diagnostic catalog candidate 2026-10-13 16:21:20 +00:00
da1d196dc5 Record OpenRouter diagnostic catalog candidate 2026-10-13 16:19:32 +00:00
b0de49a87a Preserve exact smoke-test selections 2026-10-13 16:16:40 +00:00
27ff580dae Resolve smoke-test targets 2026-10-13 16:14:30 +00:00
ece764f19e Load backend smoke-test metadata 2026-10-13 15:53:27 +00:00
f91a02548e Add smoke diagnostic domain values 2026-10-13 15:36:10 +00:00
8fbd058b90 Plan smoke test diagnostics implementation 2026-10-13 15:21:58 +00:00
c8f9a4def9 Plan smoke test integration with locally tagged catalogs 2026-10-13 14:53:55 +00:00
d8b6c0b471 Make smoke test delivery self-contained in one sprint 2026-10-13 14:48:47 +00:00
52cd94fcd1 Finalize smoke test roadmap decisions and evidence contracts 2026-10-13 14:42:24 +00:00
f1f72dfe8f Define factual smoke test diagnostics roadmap 2026-10-13 14:22:35 +00:00
d4676bc870 Record smoke test design decisions and profile precedence 2026-10-13 13:33:12 +00:00
64769883ae Draft smoke test capability proposal 2026-10-13 00:33:21 +00:00
635674c066 Prepare v0.15.0 release documentation 2026-10-12 20:17:52 +00:00
50bc35516a Consolidate request validation parity assertions 2026-10-12 19:31:54 +00:00
18309f50db Record request validation review finding 2026-10-12 19:05:38 +00:00
60784e7875 Document request validation workflow 2026-10-12 17:47:04 +00:00
f3148bfe52 Verify request validation parity 2026-10-12 17:45:18 +00:00
f315620bf7 Add request validation API 2026-10-12 17:43:11 +00:00
97357d62b9 Plan request validation in focused stages 2026-10-12 17:35:25 +00:00
0c3651fceb Clarify request validation roadmap and CLI guidance 2026-10-12 17:26:50 +00:00
5a5d47d36e Define request preflight validation roadmap 2026-10-12 16:56:33 +00:00
143c6e9b3a Prepare v0.14.0 release notes 2026-10-11 01:19:17 +00:00
48e4784fcd Retire completed configuration comparison roadmaps 2026-10-11 01:18:30 +00:00
78af5069c1 Update ConfigKit to v0.7.1 2026-10-11 01:17:10 +00:00
18963266d7 Upgrade ReleaseKit to v0.4.1 2026-10-11 01:11:19 +00:00
d039a019cb Document captured configuration comparison 2026-10-09 16:06:09 +00:00
d1c0281053 Verify captured configuration diff concurrency 2026-10-09 16:01:24 +00:00
870e4ca4e6 Verify configuration diff disclosure semantics 2026-10-09 15:59:58 +00:00
fd82524977 Add captured configuration comparison 2026-10-09 15:58:04 +00:00
3f76c75919 Adopt ConfigKit v0.7.0 2026-10-09 15:56:10 +00:00
fe764f72d8 Plan captured configuration comparison in focused stages 2026-10-09 15:46:41 +00:00
26ab566f38 Define captured configuration comparison roadmap 2026-10-09 15:41:53 +00:00
18ca21fa45 Retire completed roadmap documentation 2026-10-09 04:50:10 +00:00
e32fdc22f4 Document the v0.13.0 release 2026-10-09 04:49:55 +00:00
b02b802427 Upgrade ConfigKit to v0.6.0 2026-10-09 04:45:10 +00:00
301192b974 Document profile and prompt identity listings 2026-10-09 04:34:45 +00:00
f2aeb5eb30 Verify effective listing ownership and concurrency 2026-10-09 04:31:31 +00:00
a27ee8f7da Expose profile and prompt identity listings 2026-10-09 04:28:23 +00:00
4d1895c223 Capture profile and prompt listing identities 2026-10-09 04:27:00 +00:00
d7fe70520a Plan profile and prompt listing in focused stages 2026-10-09 04:23:27 +00:00
7325a34c07 Define profile and prompt listing 2026-10-09 04:19:09 +00:00
03c44923c8 Upgrade the release tooling to ReleaseKit v0.3.0 2026-10-09 03:48:52 +00:00
dc0adf5751 Align the release policy with Go compatibility guidance 2026-10-09 03:45:39 +00:00
eae21797f7 Describe the ReleaseKit upgrade scope 2026-10-09 03:41:56 +00:00
ea6c7d5f0f Release unused file buffers for empty artifacts 2026-10-09 01:52:06 +00:00
389ef26db3 Record the remaining file buffer cleanup issue 2026-10-09 01:38:14 +00:00
8a5c0de162 Finalize computational efficiency evidence 2026-10-09 01:16:14 +00:00
80aecb646d Reuse prepared execution snapshot ownership 2026-10-09 01:08:46 +00:00
b7b4000aab Record prepared snapshot ownership evidence 2026-10-09 00:57:23 +00:00
11ecb923ba Add prepared snapshot ownership benchmarks 2026-10-09 00:56:29 +00:00
f46d5a4591 Record assembled workflow benchmark results 2026-10-09 00:50:41 +00:00
743e78d514 Add assembled workflow benchmark baseline 2026-10-09 00:49:46 +00:00
3743221aa5 Correct roadmap progress summary 2026-10-09 00:46:00 +00:00
a8863d49d0 Record prompt inspection metadata results 2026-10-09 00:45:54 +00:00
c5f733af45 Select prompt metadata for inspection 2026-10-09 00:45:25 +00:00
c09d2c6e20 Record requirement inspection identity results 2026-10-09 00:38:19 +00:00
6c5dec761c Inspect requirements by profile identity 2026-10-09 00:38:01 +00:00
74b91fae99 Record lightweight inspection lookup results 2026-10-09 00:35:59 +00:00
b7ff37118c Add lightweight configuration lookups 2026-10-09 00:35:30 +00:00
ef012ea1e5 Record requirement inspection benchmarks 2026-10-09 00:31:52 +00:00
04d5d695e0 Add requirement inspection benchmarks 2026-10-09 00:30:51 +00:00
78bff91395 Encode generation payloads once on success 2026-10-09 00:27:24 +00:00
c63ca5352f Record request preparation baseline 2026-10-09 00:25:39 +00:00
3f157150e4 Add request preparation baseline and boundary tests 2026-10-09 00:25:21 +00:00
f1e1a45a05 Reserve capacity for regular file artifacts 2026-10-09 00:23:27 +00:00
996295d5c1 Add artifact file-read benchmarks 2026-10-09 00:20:15 +00:00
58a5bbe790 Use local builders for rendered templates 2026-10-09 00:18:39 +00:00
dd89be3b03 Record basic validation allocation savings 2026-10-09 00:15:06 +00:00
ed7c27324d Avoid copying output during basic validation 2026-10-09 00:13:23 +00:00
05fe13d80b Record validation and rendering baselines 2026-10-09 00:11:00 +00:00
0eb009ad5a Add validation and rendering benchmarks 2026-10-09 00:09:15 +00:00
3fde1f6cbb Plan the performance improvements in focused stages 2026-10-09 00:02:21 +00:00
481e2d494c Mark the ConfigKit performance request complete 2026-10-08 23:50:38 +00:00
eb830bb613 Refine performance roadmap after review 2026-10-08 23:44:12 +00:00
dd6d4591ae Plan computational efficiency improvements 2026-10-08 22:02:18 +00:00
de6443f3ae Align completed ReleaseKit documentation 2026-10-08 21:21:37 +00:00
5a63b649f9 Clarify inspected input identifier disclosure 2026-10-08 21:19:24 +00:00
c7e22c0dbe Avoid rescanning unsupported input names 2026-10-08 21:18:45 +00:00
e69a847cf2 Preserve caller scope during requirement inspection 2026-10-08 21:15:56 +00:00
21815ad7c0 Record requirement inspection review findings 2026-10-08 20:59:21 +00:00
a051edd82d Document prompt requirement inspection 2026-10-08 20:48:10 +00:00
ee0ad99e17 Test public prompt requirement inspection 2026-10-08 20:43:48 +00:00
bc5a2da0f2 Expose prompt requirement inspection 2026-10-08 20:40:16 +00:00
089e824a92 Resolve direct prompt input names 2026-10-08 20:36:17 +00:00
93e8874813 Classify simple prompt requirement branches 2026-10-08 20:31:45 +00:00
cc0b6183bf Enumerate prompt input and variable dependencies 2026-10-08 20:25:17 +00:00
bf472dad8a Capture prompt requirement metadata and templates 2026-10-08 20:17:34 +00:00
a6e0ffea8f Simplify requirement inspection plan 2026-10-08 19:58:57 +00:00
e459c7d54b Settle requirement inspection scope 2026-10-08 17:54:20 +00:00
71f6683fda Plan effective requirement inspection implementation 2026-10-08 17:28:04 +00:00
7827ca043b Plan effective prompt requirement inspection 2026-10-08 16:57:58 +00:00
b595dca8c7 Record ReleaseKit adoption review findings 2026-10-08 16:47:23 +00:00
d5f34642d7 Verify ReleaseKit adoption on final source snapshot 2026-10-08 16:42:21 +00:00
b1866946d1 Delegate Promptkit releases to ReleaseKit 2026-10-08 16:40:34 +00:00
05082c6fdf Test ReleaseKit validation failure handling 2026-10-08 16:35:39 +00:00
51b072965c Add successful ReleaseKit publication fixture 2026-10-08 16:33:43 +00:00
d5940856be Add native ReleaseKit bootstrap 2026-10-08 16:23:26 +00:00
0d163f8165 Extract example checks into release profile 2026-10-08 16:20:51 +00:00
1889915a81 Separate the Go minimum from the release toolchain 2026-10-08 16:14:50 +00:00
3ca01ae0b0 Add ReleaseKit as a release tool 2026-10-08 16:10:23 +00:00
6e283d16c3 Plan the ReleaseKit adoption 2026-10-08 13:56:36 +00:00
41b2021d8c Resolve the ReleaseKit adoption decisions 2026-10-08 13:36:01 +00:00
3add7024db Align release preparation and tooling with the current Go version 2026-10-08 13:35:06 +00:00
a148a2b1f4 Define the ReleaseKit adoption roadmap 2026-10-08 13:13:10 +00:00
7310ef34a4 Assess ReleaseKit adoption and shared release validation 2026-10-07 16:08:26 +00:00
160 changed files with 17348 additions and 1544 deletions

4
.gitignore vendored
View File

@@ -22,6 +22,9 @@
go.work
go.work.sum
# Local ReleaseKit executable
.releasekit/
# env file
.env
@@ -67,4 +70,3 @@ Icon
.AppleDesktop
Network Trash Folder
Temporary Items
.apdisk

View File

@@ -21,15 +21,21 @@ Run the maintained offline preparation example from the repository root:
go run ./examples/go-library/prepare
```
It loads a repository-local prompt, supplies a programmatic profile and inline
input, inspects the captured configuration, and prints deterministic
preparation metadata without contacting a provider or requiring credentials.
It composes two explicitly selected synthetic collections, imports
source-relative prompt content, supplies an inline input, and prepares through
a scoped view. The example also demonstrates collection inventories and
namespace-filtered discovery, without contacting a provider or requiring
credentials.
The [consumer guide](docs/consumers/pkg-promptkit.md) also shows a local backend,
ordered file/embedded sources, programmatic mappings, and how to replace an
engine when source files change. Read the
[example source](examples/go-library/prepare/main.go), the
engine when source files change. Applications adapting the prior keyed
configuration or versioned-prompt format can consult the
[configuration migration guide](docs/consumers/configuration-migration.md).
See the [example source](examples/go-library/prepare/main.go), the
[Go package consumer guide](docs/consumers/pkg-promptkit.md), and the
[framework format reference](docs/formats.md) to build a consumer workflow.
The consumer guide also covers explicit, live smoke diagnostics and interpreting
their factual reports; they are separate from the offline quickstart.
Contributors should start with the [development guide](docs/development.md).
The [architecture policy](docs/policy/architecture.md) defines the library
@@ -37,6 +43,14 @@ boundary and constraints that framework work must preserve.
## Release Guidance
The [v0.16.0 release notes](docs/releases/v0.16.0.md) cover explicit smoke
diagnostics, configuration collections, scoped engine views, and discovery.
Upgrading consumers should review the breaking changes to bundled configuration
and prompt version selection and follow the linked migration guide.
The [v0.15.0 release notes](docs/releases/v0.15.0.md) cover error-only
validation of concrete requests with `Engine.ValidateRequest`.
Consumers upgrading to `v0.12.0` should read the
[v0.12.0 release notes](docs/releases/v0.12.0.md) for the intentional ConfigKit
configuration and public construction API migration.

View File

@@ -0,0 +1,238 @@
package promptkit_test
import (
"context"
"encoding/json"
"fmt"
"strings"
"testing"
"testing/fstest"
"gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/promptkit"
)
var (
assembledPreparedSink *promptkit.PreparedRun
assembledResultSink *promptkit.RunResult
assembledDetailsSink promptkit.PreparedRun
assembledHandleSink *promptkit.PreparedExecution
)
type assembledWorkflowClient struct{ content string }
func (client assembledWorkflowClient) Generate(
context.Context,
promptkit.GenerateRequest,
) (*promptkit.GenerateResponse, error) {
return &promptkit.GenerateResponse{Content: client.content}, nil
}
type assembledWorkflowFixture struct {
engine *promptkit.Engine
request promptkit.RunRequest
expectedMessage string
response string
}
func assembledWorkflowSchema(b *testing.B) []byte {
b.Helper()
properties := make(map[string]any, 256)
for index := range 256 {
properties[fmt.Sprintf("property-%03d", index)] = map[string]any{"type": "string"}
}
data, err := json.Marshal(map[string]any{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"type": "object",
"properties": properties,
"additionalProperties": false,
})
if err != nil {
b.Fatal(err)
}
return data
}
func assembledWorkflowExtras(entries int) map[string]any {
providers := make(map[string]any, entries)
for index := range entries {
providers[fmt.Sprintf("provider-%03d", index)] = map[string]any{
"enabled": true,
"labels": []any{fmt.Sprintf("tier-%d", index%4), fmt.Sprintf("region-%d", index%3)},
"limits": map[string]any{"burst": index + 1, "period": "minute"},
"routes": []any{map[string]any{"weight": index + 1, "fallback": true}},
}
}
return map[string]any{"provider_options": providers}
}
func newAssembledWorkflowFixture(b *testing.B, name string) assembledWorkflowFixture {
b.Helper()
message := "Hello {{.name}}"
response := "ok"
output := map[string]any{"format": "text", "validation_mode": "none"}
profile := map[string]any{"endpoint": "http://example.test/v1", "model": "benchmark-model"}
options := []promptkit.Option{promptkit.WithLLMClient(assembledWorkflowClient{content: response})}
var expectedMessage string
switch name {
case "small":
expectedMessage = "Hello Ada"
case "large_message":
prefix := strings.Repeat("x", 256*1024)
message = prefix + "{{.name}}"
expectedMessage = prefix + "Ada"
case "mutable_payload":
message = `{"subject":"{{.name}}"}`
expectedMessage = `{"subject":"Ada"}`
response = "{}"
profile["extra_params"] = assembledWorkflowExtras(64)
output = map[string]any{
"format": "json", "validation_mode": "json_schema", "schema_path": "workflow-schema.json",
}
options[0] = promptkit.WithLLMClient(assembledWorkflowClient{content: response})
options = append(options, promptkit.WithSchemaFS(fstest.MapFS{
"workflow-schema.json": &fstest.MapFile{Data: assembledWorkflowSchema(b)},
}, "."))
default:
b.Fatalf("unknown assembled workflow fixture %q", name)
}
config := map[string]any{
"profiles": map[string]any{"profile": profile},
"prompts": map[string]any{"workflow": map[string]any{
"version": "v1",
"default_profile": "profile",
"messages": []any{map[string]any{"role": "user", "content": message}},
"output": output,
}},
}
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
Source: configkit.Mapping("assembled workflow benchmark", config),
}}}, options...)
if err != nil {
b.Fatal(err)
}
request := promptkit.RunRequest{
PromptID: "workflow", Vars: map[string]string{"name": "Ada"},
}
return assembledWorkflowFixture{
engine: engine, request: request, expectedMessage: expectedMessage, response: response,
}
}
func verifyAssembledWorkflowFixture(b *testing.B, fixture assembledWorkflowFixture) {
b.Helper()
ctx := context.Background()
prepared, err := fixture.engine.Prepare(ctx, fixture.request)
if err != nil {
b.Fatal(err)
}
if prepared.PromptID != "workflow" || prepared.PromptVersion == nil || *prepared.PromptVersion != "v1" || len(prepared.Messages) != 1 ||
prepared.Messages[0].Content != fixture.expectedMessage {
b.Fatalf("prepared fixture output is invalid: %+v", prepared)
}
handle, err := fixture.engine.PrepareExecution(ctx, fixture.request)
if err != nil {
b.Fatal(err)
}
details := handle.Details()
if details.PromptID != "workflow" || len(details.Messages) != 1 || details.Messages[0].Content != fixture.expectedMessage {
handle.Discard()
b.Fatalf("prepared execution details are invalid: %+v", details)
}
handle.Discard()
result, err := fixture.engine.Run(ctx, fixture.request)
if err != nil {
b.Fatal(err)
}
if result.RawOutput != fixture.response {
b.Fatalf("run output=%q, want %q", result.RawOutput, fixture.response)
}
if fixture.response == "{}" && result.Validation.Status != promptkit.ValidationPassed {
b.Fatalf("structured response validation=%+v, want passed", result.Validation)
}
}
func BenchmarkEnginePreparation(b *testing.B) {
for _, name := range []string{"small", "large_message", "mutable_payload"} {
b.Run(name, func(b *testing.B) {
fixture := newAssembledWorkflowFixture(b, name)
verifyAssembledWorkflowFixture(b, fixture)
ctx := context.Background()
b.Run("prepare", func(b *testing.B) {
b.ReportAllocs()
b.ResetTimer()
for b.Loop() {
prepared, err := fixture.engine.Prepare(ctx, fixture.request)
if err != nil {
b.Fatal(err)
}
assembledPreparedSink = prepared
}
b.StopTimer()
assembledPreparedSink = nil
})
b.Run("prepared_execution_creation_and_discard", func(b *testing.B) {
b.ReportAllocs()
b.ResetTimer()
for b.Loop() {
handle, err := fixture.engine.PrepareExecution(ctx, fixture.request)
if err != nil {
b.Fatal(err)
}
handle.Discard()
assembledHandleSink = handle
}
b.StopTimer()
assembledHandleSink = nil
})
})
}
}
func BenchmarkEngineRun(b *testing.B) {
for _, name := range []string{"small", "large_message", "mutable_payload"} {
b.Run(name, func(b *testing.B) {
fixture := newAssembledWorkflowFixture(b, name)
verifyAssembledWorkflowFixture(b, fixture)
ctx := context.Background()
b.ReportAllocs()
b.ResetTimer()
for b.Loop() {
result, err := fixture.engine.Run(ctx, fixture.request)
if err != nil {
b.Fatal(err)
}
assembledResultSink = result
}
b.StopTimer()
assembledResultSink = nil
})
}
}
func BenchmarkPreparedExecutionDetails(b *testing.B) {
for _, name := range []string{"small", "large_message", "mutable_payload"} {
b.Run(name, func(b *testing.B) {
fixture := newAssembledWorkflowFixture(b, name)
verifyAssembledWorkflowFixture(b, fixture)
handle, err := fixture.engine.PrepareExecution(context.Background(), fixture.request)
if err != nil {
b.Fatal(err)
}
defer handle.Discard()
initial := handle.Details()
if len(initial.Messages) != 1 || initial.Messages[0].Content != fixture.expectedMessage {
b.Fatalf("prepared details fixture is invalid: %+v", initial)
}
b.ReportAllocs()
b.ResetTimer()
for b.Loop() {
assembledDetailsSink = handle.Details()
}
b.StopTimer()
assembledDetailsSink = promptkit.PreparedRun{}
})
}
}

View File

@@ -5,10 +5,10 @@ import (
"gitea.maximumdirect.net/eric/promptkit/internal/backend"
)
// BackendOpenRouter identifies the maintained OpenRouter backend.
// BackendOpenRouter is the conventional ID for an OpenRouter backend.
const BackendOpenRouter = backend.OpenRouterID
// BackendRakestrawHome identifies the maintained Rakestrawhome backend.
// BackendRakestrawHome is the conventional ID for a Rakestrawhome backend.
const BackendRakestrawHome = backend.RakestrawHomeID
// BackendLocal is a conventional ID for a caller-configured local backend.
@@ -17,8 +17,9 @@ const BackendLocal = "local"
// BackendConfig is a presence-aware convenience value for a keyed backend
// definition. Nil pointer fields are omitted, while non-nil pointers preserve
// explicit zero or blank values. A nil ExtraParams map is omitted; a non-nil
// empty map supplies an empty mapping. Use raw ConfigKit Mapping inputs for
// explicit null. BackendConfig has no stable JSON representation.
// empty map supplies an empty mapping. A nil SmokeTests pointer is omitted; a
// non-nil pointer supplies a smoke_tests mapping. Use raw ConfigKit Mapping
// inputs for explicit null. BackendConfig has no stable JSON representation.
type BackendConfig struct {
// Endpoint is the required absolute HTTP(S) base URL after all layers merge.
Endpoint *string
@@ -31,6 +32,43 @@ type BackendConfig struct {
QueueCapacity *int
// ExtraParams supplies JSON-compatible provider request defaults.
ExtraParams map[string]any
// SmokeTests supplies optional diagnostic defaults and HTTP probe settings.
SmokeTests *BackendSmokeTestConfig
}
// BackendSmokeTestConfig contains optional diagnostic settings associated with
// a backend. Nil nested fields are omitted by BackendSource.
type BackendSmokeTestConfig struct {
// DefaultProfile is the exact profile ID selected as the backend's
// diagnostic inference default. A supplied value must not be blank; the ID
// is resolved only when that default is selected.
DefaultProfile *string
// Reachability optionally selects a full HTTP destination for reachability
// diagnostics. A nil endpoint uses the effective backend endpoint.
Reachability *ReachabilityProbeConfig
// Authentication optionally declares a named authentication diagnostic.
// A supplied object requires Kind.
Authentication *AuthenticationProbeConfig
}
// ReachabilityProbeConfig configures an optional reachability destination.
type ReachabilityProbeConfig struct {
// Endpoint is an absolute HTTP(S) URL, including any desired path or query.
// Nil selects the effective backend endpoint; an explicit blank value is
// invalid.
Endpoint *string
}
// AuthenticationProbeConfig configures an optional named authentication
// diagnostic.
type AuthenticationProbeConfig struct {
// Kind selects the authentication adapter. Supported values are "none" and
// "openrouter_key"; surrounding whitespace and case are normalized.
Kind *string
// Endpoint is an absolute HTTP(S) URL, including any desired path or query.
// Nil selects the adapter's backend-relative destination; an explicit blank
// value is invalid.
Endpoint *string
}
// BackendSource creates a labeled ConfigKit mapping from keyed backend values.
@@ -57,6 +95,30 @@ func BackendSource(label string, backends map[string]BackendConfig) configkit.So
if config.ExtraParams != nil {
fields["extra_params"] = providerMapForMapping(config.ExtraParams)
}
if config.SmokeTests != nil {
smokeTests := make(map[string]any)
if config.SmokeTests.DefaultProfile != nil {
smokeTests["default_profile"] = *config.SmokeTests.DefaultProfile
}
if config.SmokeTests.Reachability != nil {
reachability := make(map[string]any)
if config.SmokeTests.Reachability.Endpoint != nil {
reachability["endpoint"] = *config.SmokeTests.Reachability.Endpoint
}
smokeTests["reachability"] = reachability
}
if config.SmokeTests.Authentication != nil {
authentication := make(map[string]any)
if config.SmokeTests.Authentication.Kind != nil {
authentication["kind"] = *config.SmokeTests.Authentication.Kind
}
if config.SmokeTests.Authentication.Endpoint != nil {
authentication["endpoint"] = *config.SmokeTests.Authentication.Endpoint
}
smokeTests["authentication"] = authentication
}
fields["smoke_tests"] = smokeTests
}
definitions[id] = fields
}
return configkit.Mapping(label, map[string]any{"backends": definitions})

View File

@@ -267,7 +267,7 @@ func newBackendCapacityEngine(
Source: configkit.Mapping("capacity fixture", map[string]any{
"backends": map[string]any{"limited": backend},
"profiles": map[string]any{"profile": map[string]any{"backend": "limited", "model": "model"}},
"prompts": map[string]any{promptID: map[string]any{"1": definition}},
"prompts": map[string]any{promptID: promptDefinitionWithVersion("1", definition)},
}),
}}}, options...)
if err != nil {

View File

@@ -0,0 +1,243 @@
package promptkit_test
import (
"errors"
"strings"
"testing"
"gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/promptkit"
)
func TestConfigurationInspectionDiffsCapturedValues(t *testing.T) {
left := comparisonInspection(t, "earlier-model")
right := comparisonInspection(t, "later-model")
differences, err := left.Diff(right)
if err != nil {
t.Fatalf("compare captured configurations: %v", err)
}
if len(differences) != 1 {
t.Fatalf("difference count = %d, want 1: %+v", len(differences), differences)
}
difference := differences[0]
if got, want := difference.Path.String(), "/profiles/offline/model"; got != want {
t.Fatalf("difference path = %q, want %q", got, want)
}
if !strings.Contains(difference.Left.YAML, "earlier-model") || !strings.Contains(difference.Right.YAML, "later-model") {
t.Fatalf("difference snapshots = (%q, %q)", difference.Left.YAML, difference.Right.YAML)
}
reversed, err := right.Diff(left)
if err != nil {
t.Fatalf("compare reversed configurations: %v", err)
}
if len(reversed) != 1 || reversed[0].Path.String() != difference.Path.String() ||
!strings.Contains(reversed[0].Left.YAML, "later-model") ||
!strings.Contains(reversed[0].Right.YAML, "earlier-model") {
t.Fatalf("reversed difference = %+v", reversed)
}
equal := comparisonInspection(t, "earlier-model")
if equal, err := left.Diff(equal); err != nil || len(equal) != 0 {
t.Fatalf("compare equal configurations = (%+v, %v), want no differences", equal, err)
}
if equal, err := left.Diff(left); err != nil || len(equal) != 0 {
t.Fatalf("compare view with itself = (%+v, %v), want no differences", equal, err)
}
}
func TestConfigurationInspectionDiffRejectsZeroViews(t *testing.T) {
valid := comparisonInspection(t, "current-model")
var zero promptkit.ConfigurationInspection
for _, comparison := range []struct {
name string
left, right promptkit.ConfigurationInspection
}{
{name: "zero left", left: zero, right: valid},
{name: "zero right", left: valid, right: zero},
{name: "both zero", left: zero, right: zero},
} {
t.Run(comparison.name, func(t *testing.T) {
differences, err := comparison.left.Diff(comparison.right)
if !errors.Is(err, promptkit.ErrInvalidConfig) || len(differences) != 0 {
t.Fatalf("Diff() = (%+v, %v), want ErrInvalidConfig and no records", differences, err)
}
})
}
}
func TestConfigurationInspectionDiffIgnoresSourceHistory(t *testing.T) {
left := comparisonInspectionFrom(t, "earlier source", "same-model", nil)
right := comparisonInspectionFrom(t, "later source", "same-model", nil)
if !hasSourceLabel(left, "earlier source") || !hasSourceLabel(right, "later source") {
t.Fatal("comparison fixtures did not retain their distinct source labels")
}
differences, err := left.Diff(right)
if err != nil || len(differences) != 0 {
t.Fatalf("compare equal effective values with different sources = (%+v, %v), want no differences", differences, err)
}
}
func TestConfigurationInspectionDiffReportsPresenceAndSequences(t *testing.T) {
left := comparisonInspectionFrom(t, "before", "same-model", map[string]any{
"sequence": []any{"earlier", "unchanged"},
})
right := comparisonInspectionFrom(t, "after", "same-model", map[string]any{
"nullable": nil,
"sequence": []any{"later", "unchanged"},
})
differences, err := left.Diff(right)
if err != nil {
t.Fatalf("compare presence and sequence values: %v", err)
}
if len(differences) != 2 {
t.Fatalf("difference count = %d, want 2: %+v", len(differences), differences)
}
byPath := differencesByPath(differences)
nullable, ok := byPath["/backends/local/extra_params/nullable"]
if !ok || nullable.Left.Present || !nullable.Right.Present || strings.TrimSpace(nullable.Right.YAML) != "null" {
t.Fatalf("absent-to-null difference = (%+v, %t)", nullable, ok)
}
sequence, ok := byPath["/backends/local/extra_params/sequence"]
if !ok || !sequence.Left.Present || !sequence.Right.Present ||
!strings.Contains(sequence.Left.YAML, "earlier") || !strings.Contains(sequence.Right.YAML, "later") {
t.Fatalf("atomic sequence difference = (%+v, %t)", sequence, ok)
}
}
func TestConfigurationInspectionDiffProtectsSensitiveValuesAndReturnedRecords(t *testing.T) {
left := comparisonInspectionFrom(t, "before", "same-model", map[string]any{
"token": "left-secret-sentinel",
"visible": "before-value",
})
right := comparisonInspectionFrom(t, "after", "same-model", map[string]any{
"token": "right-secret-sentinel",
"visible": "after-value",
})
differences, err := left.Diff(right)
if err != nil {
t.Fatalf("compare sensitive values: %v", err)
}
if len(differences) != 2 {
t.Fatalf("difference count = %d, want 2: %+v", len(differences), differences)
}
byPath := differencesByPath(differences)
secret, ok := byPath["/backends/local/extra_params/token"]
if !ok || !secret.Left.Redacted || !secret.Right.Redacted || !secret.HiddenChanged {
t.Fatalf("sensitive difference = (%+v, %t)", secret, ok)
}
if strings.Contains(secret.Left.YAML, "left-secret-sentinel") || strings.Contains(secret.Left.YAML, "right-secret-sentinel") ||
strings.Contains(secret.Right.YAML, "left-secret-sentinel") || strings.Contains(secret.Right.YAML, "right-secret-sentinel") {
t.Fatalf("sensitive snapshots disclosed a value: %+v", secret)
}
visible, ok := byPath["/backends/local/extra_params/visible"]
if !ok || visible.Left.Redacted || visible.Right.Redacted || visible.HiddenChanged ||
!strings.Contains(visible.Left.YAML, "before-value") || !strings.Contains(visible.Right.YAML, "after-value") {
t.Fatalf("public sibling difference = (%+v, %t)", visible, ok)
}
differences[0].Path = configkit.Keys("caller", "mutation")
differences[0].Left.YAML = "caller mutation"
differences[0].Right.YAML = "caller mutation"
again, err := left.Diff(right)
if err != nil {
t.Fatalf("repeat comparison after mutating returned records: %v", err)
}
if len(again) != 2 {
t.Fatalf("repeat difference count = %d, want 2: %+v", len(again), again)
}
againByPath := differencesByPath(again)
if _, exists := againByPath["/caller/mutation"]; exists {
t.Fatal("mutating returned path reached retained inspection")
}
if _, exists := againByPath["/backends/local/extra_params/token"]; !exists {
t.Fatalf("original sensitive path was lost after mutation: %+v", again)
}
if _, exists := againByPath["/backends/local/extra_params/visible"]; !exists {
t.Fatalf("original public path was lost after mutation: %+v", again)
}
for _, difference := range again {
if difference.Left.YAML == "caller mutation" || difference.Right.YAML == "caller mutation" {
t.Fatalf("mutating returned snapshots reached retained inspection: %+v", difference)
}
}
}
func TestConfigurationInspectionDiffMasksSensitiveShapeReplacement(t *testing.T) {
left := comparisonInspectionFrom(t, "before", "same-model", map[string]any{
"details": map[string]any{
"token": "nested-secret-sentinel",
"label": "public detail",
},
})
right := comparisonInspectionFrom(t, "after", "same-model", map[string]any{
"details": "replacement",
})
differences, err := left.Diff(right)
if err != nil {
t.Fatalf("compare sensitive shape replacement: %v", err)
}
if len(differences) != 1 || differences[0].Path.String() != "/backends/local/extra_params/details" {
t.Fatalf("shape replacement differences = %+v", differences)
}
difference := differences[0]
if !difference.Left.Redacted || !difference.Right.Redacted || !difference.HiddenChanged {
t.Fatalf("shape replacement masking = %+v", difference)
}
if strings.Contains(difference.Left.YAML, "nested-secret-sentinel") || strings.Contains(difference.Right.YAML, "nested-secret-sentinel") {
t.Fatalf("shape replacement disclosed a sensitive value: %+v", difference)
}
}
func comparisonInspection(t *testing.T, model string) promptkit.ConfigurationInspection {
t.Helper()
return comparisonInspectionFrom(t, "comparison fixture", model, nil)
}
func comparisonInspectionFrom(t *testing.T, source, model string, extraParams map[string]any) promptkit.ConfigurationInspection {
t.Helper()
backend := map[string]any{"endpoint": "https://example.invalid/v1"}
if extraParams != nil {
backend["extra_params"] = extraParams
}
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: configkit.Mapping(source, map[string]any{
"backends": map[string]any{"local": backend},
"profiles": map[string]any{"offline": map[string]any{
"backend": "local",
"model": model,
}},
})}}}},
}})
if err != nil {
t.Fatalf("construct comparison engine: %v", err)
}
inspection, err := engine.InspectConfiguration()
if err != nil {
t.Fatalf("inspect comparison engine: %v", err)
}
return inspection
}
func differencesByPath(differences []configkit.Difference) map[string]configkit.Difference {
byPath := make(map[string]configkit.Difference, len(differences))
for _, difference := range differences {
byPath[difference.Path.String()] = difference
}
return byPath
}
func hasSourceLabel(view promptkit.ConfigurationInspection, label string) bool {
for _, source := range view.Sources() {
if source.Label == label {
return true
}
}
return false
}

View File

@@ -27,6 +27,23 @@ func (e *Engine) InspectConfiguration() (ConfigurationInspection, error) {
return ConfigurationInspection{source: e.internal.InspectConfiguration()}, nil
}
// Diff compares this view's captured effective configuration with other. This
// view is the earlier (left) value and other is the later (right) value. The
// comparison delegates to ConfigKit's frozen effective-value semantics, so
// source or resolution history alone does not create differences. It does not
// reread external sources or compare referenced file contents, resolved
// credential contents, later execution preparation, or request-specific
// overrides. Either zero view returns ErrInvalidConfig and no records; other
// ConfigKit errors are returned unchanged. An empty diff is not evidence of
// equivalent engine behavior or execution. Records can contain ordinary
// application data, so callers choose where and to whom to disclose them.
func (v ConfigurationInspection) Diff(other ConfigurationInspection) ([]configkit.Difference, error) {
if v.source == nil || other.source == nil {
return nil, ErrInvalidConfig
}
return configuration.Diff(v.source, other.source)
}
// Metadata returns detached effective presence and history records.
func (v ConfigurationInspection) Metadata() configkit.Metadata {
if v.source == nil {

View File

@@ -23,13 +23,19 @@ func TestConfigurationInspectionReportsLiteralKeysAndRedactedHistory(t *testing.
"endpoint": "https://example.invalid/v1", "api_key_env": "OPTIONAL_KEY",
"extra_params": map[string]any{"token": "private-sentinel", "visible": "public-value"},
}},
"profiles": map[string]any{
"base.v1": map[string]any{"backend": "local", "model": "default-model"},
"literal/id": map[string]any{"extends": "base.v1"},
},
"collections": map[string]any{"literal": map[string]any{
"backends": map[string]any{"local": map[string]any{
"endpoint": "https://example.invalid/v1",
"extra_params": map[string]any{"token": "namespaced-private-sentinel"},
}},
"profiles": map[string]any{
"base.v1": map[string]any{"backend": "local", "model": "default-model"},
"id": map[string]any{"extends": "base.v1"},
},
}},
})}}},
{Inputs: []configkit.Input{{Source: configkit.Mapping("operator", map[string]any{
"profiles": map[string]any{"base.v1": map[string]any{"model": "operator-model"}},
"collections": map[string]any{"literal": map[string]any{"profiles": map[string]any{"base.v1": map[string]any{"model": "operator-model"}}}},
})}}},
},
}})
@@ -40,17 +46,17 @@ func TestConfigurationInspectionReportsLiteralKeysAndRedactedHistory(t *testing.
if err != nil {
t.Fatalf("inspect configuration: %v", err)
}
modelPath := configkit.Keys("profiles", "base.v1", "model")
modelPath := configkit.Keys("collections", "literal", "profiles", "base.v1", "model")
model, found := view.Explain(modelPath)
if !found || !strings.Contains(model.Effective.YAML, "operator-model") || len(model.History) < 2 {
t.Fatalf("layered model explanation = (%+v, %t)", model, found)
}
inheritedPath := configkit.Keys("profiles", "literal/id", "model")
inheritedPath := configkit.Keys("collections", "literal", "profiles", "id", "model")
inherited, found := view.Explain(inheritedPath)
if !found || !strings.Contains(inherited.Effective.YAML, "operator-model") || len(inherited.History) == 0 || !view.Metadata().IsExplicit(inheritedPath) {
t.Fatalf("literal-key inheritance explanation = (%+v, %t)", inherited, found)
}
if inheritedPath.String() != "/profiles/literal~1id/model" {
if inheritedPath.String() != "/collections/literal/profiles/id/model" {
t.Fatalf("literal-key path = %q", inheritedPath)
}
secretPath := configkit.Keys("backends", "local", "extra_params", "token")
@@ -58,6 +64,11 @@ func TestConfigurationInspectionReportsLiteralKeysAndRedactedHistory(t *testing.
if !found || !secret.Effective.Redacted || strings.Contains(secret.Effective.YAML, "private-sentinel") {
t.Fatalf("credential-shaped provider value was not redacted: (%+v, %t)", secret, found)
}
namespacedSecretPath := configkit.Keys("collections", "literal", "backends", "local", "extra_params", "token")
namespacedSecret, found := view.Explain(namespacedSecretPath)
if !found || !namespacedSecret.Effective.Redacted || strings.Contains(namespacedSecret.Effective.YAML, "namespaced-private-sentinel") {
t.Fatalf("namespaced provider value was not redacted: (%+v, %t)", namespacedSecret, found)
}
var effective bytes.Buffer
if err := view.WriteEffective(&effective); err != nil {
t.Fatalf("write effective configuration: %v", err)
@@ -66,7 +77,7 @@ func TestConfigurationInspectionReportsLiteralKeysAndRedactedHistory(t *testing.
t.Fatalf("unexpected effective report: %s", effective.String())
}
var explanation bytes.Buffer
if err := view.WriteExplanation(&explanation, secretPath); err != nil || strings.Contains(explanation.String(), "private-sentinel") {
if err := view.WriteExplanation(&explanation, namespacedSecretPath); err != nil || strings.Contains(explanation.String(), "namespaced-private-sentinel") {
t.Fatalf("secret explanation = (%q, %v)", explanation.String(), err)
}
@@ -93,6 +104,28 @@ func TestConfigurationInspectionReportsLiteralKeysAndRedactedHistory(t *testing.
}
}
func TestConfigurationInspectionRetainsMainSpecificationDeclaration(t *testing.T) {
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
Source: configkit.Mapping("main", map[string]any{"version": "1"}),
}}})
if err != nil {
t.Fatalf("construct engine with accepted declaration: %v", err)
}
inspection, err := engine.InspectConfiguration()
if err != nil {
t.Fatalf("inspect configuration: %v", err)
}
path := configkit.Keys("version")
field, found := inspection.Explain(path)
if !found || !inspection.Metadata().IsExplicit(path) || !strings.Contains(field.Effective.YAML, "1") {
t.Fatalf("main specification declaration was not retained: (%+v, %t)", field, found)
}
var effective bytes.Buffer
if err := inspection.WriteEffective(&effective); err != nil || !strings.Contains(effective.String(), "version") {
t.Fatalf("effective inspection = (%q, %v), want retained version", effective.String(), err)
}
}
type failingInspectionWriter struct{ err error }
func (writer failingInspectionWriter) Write([]byte) (int, error) { return 0, writer.err }
@@ -117,7 +150,13 @@ func TestConfigurationInspectionZeroAndWriterErrors(t *testing.T) {
t.Fatalf("zero explanation writer = %v", err)
}
valid, err := promptkit.NewEngine(promptkit.Config{})
endpoint, model := "https://configured.example/v1", "configured-model"
profile := configkit.Input{Source: promptkit.ProfileSource("inspection test profile", map[string]promptkit.ProfileConfig{
"inspection-profile": {Endpoint: &endpoint, Model: &model},
})}
valid, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{profile}}},
}})
if err != nil {
t.Fatalf("construct engine: %v", err)
}
@@ -130,7 +169,7 @@ func TestConfigurationInspectionZeroAndWriterErrors(t *testing.T) {
if err := view.WriteEffective(writer); !errors.Is(err, writeErr) {
t.Fatalf("effective writer error = %v", err)
}
if err := view.WriteExplanation(writer, configkit.Keys("profiles", "mistral-small-3", "model")); !errors.Is(err, writeErr) {
if err := view.WriteExplanation(writer, configkit.Keys("profiles", "inspection-profile", "model")); !errors.Is(err, writeErr) {
t.Fatalf("explanation writer error = %v", err)
}
}
@@ -144,9 +183,17 @@ func TestConfigurationInspectionUsesCapturedSourcesWithoutProviderCalls(t *testi
if err != nil {
t.Fatalf("select inputs: %v", err)
}
otherSource := &unreadableAfterConstructionFS{FS: fstest.MapFS{
"config/prompt.yaml": &fstest.MapFile{Data: []byte(capturedPromptDocument)},
"config/body.tmpl": &fstest.MapFile{Data: []byte("hello")},
}}
otherInputs, err := promptkit.FilesystemInputs("comparison source", otherSource, "config")
if err != nil {
t.Fatalf("select comparison inputs: %v", err)
}
var modelCalls atomic.Int64
client := countingLLMClient{calls: &modelCalls}
endpoint, model := "https://example.invalid/v1", "offline-model"
endpoint, model, otherModel := "https://example.invalid/v1", "offline-model", "comparison-model"
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
Underlays: []configkit.Underlay{{Inputs: inputs}, {Inputs: []configkit.Input{{Source: promptkit.ProfileSource("offline profile", map[string]promptkit.ProfileConfig{
"offline": {Endpoint: &endpoint, Model: &model},
@@ -155,36 +202,75 @@ func TestConfigurationInspectionUsesCapturedSourcesWithoutProviderCalls(t *testi
if err != nil {
t.Fatalf("construct engine: %v", err)
}
otherEngine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
Underlays: []configkit.Underlay{{Inputs: otherInputs}, {Inputs: []configkit.Input{{Source: promptkit.ProfileSource("comparison profile", map[string]promptkit.ProfileConfig{
"offline": {Endpoint: &endpoint, Model: &otherModel},
})}}}},
}}, promptkit.WithLLMClient(client))
if err != nil {
t.Fatalf("construct comparison engine: %v", err)
}
source.closed.Store(true)
otherSource.closed.Store(true)
view, err := engine.InspectConfiguration()
if err != nil {
t.Fatalf("inspect configuration: %v", err)
}
if _, found := view.Explain(configkit.Keys("prompts", "captured", "1", "messages").Index(0).Key("content_file")); !found {
otherView, err := otherEngine.InspectConfiguration()
if err != nil {
t.Fatalf("inspect comparison configuration: %v", err)
}
var expected configkit.Difference
for index := 0; index < 3; index++ {
differences, err := view.Diff(otherView)
if err != nil || len(differences) != 1 {
t.Fatalf("compare captured configurations = (%+v, %v), want one difference", differences, err)
}
if differences[0].Path.String() != "/profiles/offline/model" ||
!strings.Contains(differences[0].Left.YAML, model) ||
!strings.Contains(differences[0].Right.YAML, otherModel) {
t.Fatalf("captured difference = %+v", differences[0])
}
if index == 0 {
expected = differences[0]
} else if differences[0].Path.String() != expected.Path.String() ||
differences[0].Left.YAML != expected.Left.YAML || differences[0].Right.YAML != expected.Right.YAML {
t.Fatalf("captured difference changed between reads: first=%+v next=%+v", expected, differences[0])
}
}
if _, found := view.Explain(configkit.Keys("prompts", "captured", "messages").Index(0).Key("content_file")); !found {
t.Fatal("captured prompt content path was not inspectable")
}
var report bytes.Buffer
if err := view.WriteEffective(&report); err != nil || !strings.Contains(report.String(), "captured") {
t.Fatalf("write captured report = (%q, %v)", report.String(), err)
}
if source.reads.Load() != 0 || modelCalls.Load() != 0 {
t.Fatalf("inspection touched source or provider: reads=%d requests=%d", source.reads.Load(), modelCalls.Load())
if source.reads.Load() != 0 || otherSource.reads.Load() != 0 || modelCalls.Load() != 0 {
t.Fatalf("inspection or comparison touched source or provider: reads=(%d,%d) requests=%d", source.reads.Load(), otherSource.reads.Load(), modelCalls.Load())
}
const calls = 16
const calls = 24
var wait sync.WaitGroup
errs := make(chan error, calls)
for index := 0; index < calls; index++ {
wait.Add(1)
go func(index int) {
defer wait.Done()
if index%2 == 0 {
switch index % 3 {
case 0:
_, err := engine.InspectConfiguration()
errs <- err
return
case 1:
_, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "captured", SessionID: "direct"})
errs <- err
case 2:
differences, err := view.Diff(otherView)
if err == nil && (len(differences) != 1 || differences[0].Path.String() != expected.Path.String() ||
differences[0].Left.YAML != expected.Left.YAML || differences[0].Right.YAML != expected.Right.YAML) {
err = errors.New("concurrent comparison returned an unexpected difference")
}
errs <- err
}
_, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "captured", SessionID: "direct"})
errs <- err
}(index)
}
wait.Wait()
@@ -194,7 +280,7 @@ func TestConfigurationInspectionUsesCapturedSourcesWithoutProviderCalls(t *testi
t.Fatal(err)
}
}
if source.reads.Load() != 0 || modelCalls.Load() != calls/2 {
t.Fatalf("concurrent inspection/execution = source reads %d, model calls %d", source.reads.Load(), modelCalls.Load())
if source.reads.Load() != 0 || otherSource.reads.Load() != 0 || modelCalls.Load() != calls/3 {
t.Fatalf("concurrent inspection/comparison/execution = source reads (%d,%d), model calls %d", source.reads.Load(), otherSource.reads.Load(), modelCalls.Load())
}
}

View File

@@ -8,9 +8,26 @@ import (
"path/filepath"
"gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
"gitea.maximumdirect.net/eric/promptkit/internal/filecatalog"
)
// CollectionInput mounts source at the selected collection's configuration
// destination, collections/<namespace>, and enables ConfigKit imports for that
// input. An invalid namespace returns an error matching ErrInvalidConfig. The
// source is not read or validated until ConfigKit loads it. Callers may adjust
// the returned input, including its filesystem context or imports setting.
func CollectionInput(namespace string, source configkit.Source) (configkit.Input, error) {
if !domain.IsValidResourceNamespace(namespace) {
return configkit.Input{}, fmt.Errorf("%w: invalid collection namespace %q", ErrInvalidConfig, namespace)
}
return configkit.Input{
Source: source,
Destination: configkit.Keys("collections", namespace),
Imports: true,
}, nil
}
// DirectoryInputs selects the YAML files beneath an explicitly chosen host
// directory as one strict-composition group. The returned order is deterministic;
// it does not assign precedence among those files. Discovery is recursive over

View File

@@ -1,6 +1,8 @@
package promptkit_test
import (
"context"
"errors"
"io/fs"
"os"
"path/filepath"
@@ -16,6 +18,78 @@ type sourceDocument struct {
Second string `yaml:"second"`
}
func TestCollectionInputValidatesWithoutReadingSource(t *testing.T) {
input, err := promptkit.CollectionInput("docs", configkit.File(filepath.Join(t.TempDir(), "not-read.yaml")))
if err != nil {
t.Fatal(err)
}
if input.Destination.String() != "/collections/docs" || !input.Imports {
t.Fatalf("collection input = %+v, want destination /collections/docs with imports enabled", input)
}
for _, namespace := range []string{"", " ", "docs/api"} {
if _, err := promptkit.CollectionInput(namespace, configkit.NoSource()); !errors.Is(err, promptkit.ErrInvalidConfig) {
t.Errorf("namespace %q error = %v, want ErrInvalidConfig", namespace, err)
}
}
}
func TestCollectionInputLoadsNativeImportsAndLaterOverride(t *testing.T) {
directory := t.TempDir()
mainPath := filepath.Join(directory, "collection.yaml")
defaultsPath := filepath.Join(directory, "defaults.yaml")
if err := os.WriteFile(mainPath, []byte("configkit:\n imports:\n - path: defaults.yaml\n"), 0o600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(defaultsPath, []byte("prompts:\n greeting:\n version: imported\n messages:\n - role: user\n content: Hello\n output:\n format: text\n validation_mode: none\n"), 0o600); err != nil {
t.Fatal(err)
}
input, err := promptkit.CollectionInput("docs", configkit.File(mainPath))
if err != nil {
t.Fatal(err)
}
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{input}}},
Main: &configkit.Input{
Source: configkit.Mapping("later collection override", map[string]any{"prompts": map[string]any{"greeting": map[string]any{"version": "overridden"}}}),
Destination: configkit.Keys("collections", "docs"),
},
}})
if err != nil {
t.Fatalf("construct engine with native collection import: %v", err)
}
inspection, err := engine.InspectPrompt(context.Background(), "docs/greeting")
if err != nil || inspection.PromptVersion == nil || *inspection.PromptVersion != "overridden" {
t.Fatalf("inspect imported and overlaid prompt = (%+v, %v)", inspection, err)
}
}
func TestCollectionInputLoadsEmbeddedImportsAndLaterOverride(t *testing.T) {
files := fstest.MapFS{
"collection.yaml": {Data: []byte("configkit:\n imports:\n - path: defaults.yaml\n")},
"defaults.yaml": {Data: []byte("prompts:\n greeting:\n version: imported\n messages:\n - role: user\n content: Hello\n output:\n format: text\n validation_mode: none\n")},
}
filesystem := configkit.NewFilesystem("embedded collection", files)
input, err := promptkit.CollectionInput("docs", filesystem.File("collection.yaml"))
if err != nil {
t.Fatal(err)
}
input.Filesystem = &configkit.FilesystemContext{FS: &filesystem, Root: "."}
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{input}}},
Main: &configkit.Input{
Source: configkit.Mapping("later collection override", map[string]any{"prompts": map[string]any{"greeting": map[string]any{"version": "overridden"}}}),
Destination: configkit.Keys("collections", "docs"),
},
}})
if err != nil {
t.Fatalf("construct engine with embedded collection import: %v", err)
}
inspection, err := engine.InspectPrompt(context.Background(), "docs/greeting")
if err != nil || inspection.PromptVersion == nil || *inspection.PromptVersion != "overridden" {
t.Fatalf("inspect imported and overlaid prompt = (%+v, %v)", inspection, err)
}
}
func TestDirectoryInputsSelectsSortedFilesWithRootContext(t *testing.T) {
root := t.TempDir()
if err := os.Mkdir(filepath.Join(root, "nested"), 0o755); err != nil {

View File

@@ -1,6 +1,7 @@
package promptkit_test
import (
"bytes"
"context"
"errors"
"io"
@@ -72,11 +73,12 @@ func TestProfileEnvironmentLookupAcceptsNonShellNames(t *testing.T) {
func TestPublicConstructionWithLocalBackendAndProgrammaticProfile(t *testing.T) {
backendID, model := "workstation", "offline-model"
prompt := configkit.Input{Source: configkit.Mapping("prompt", map[string]any{
"prompts": map[string]any{"hello": map[string]any{"1": map[string]any{
"prompts": map[string]any{"hello": map[string]any{
"version": "1",
"default_profile": "offline",
"messages": []any{map[string]any{"role": "user", "content": "Hello"}},
"output": map[string]any{"format": "text", "validation_mode": "none"},
}}},
}},
})}
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "reply"}}
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
@@ -105,6 +107,36 @@ func TestPublicConstructionWithLocalBackendAndProgrammaticProfile(t *testing.T)
}
}
func TestFormerCatalogBackendIDsWorkWhenConfigured(t *testing.T) {
openRouterID, rakestrawhomeID := promptkit.BackendOpenRouter, promptkit.BackendRakestrawHome
openRouterEndpoint, rakestrawhomeEndpoint := "https://openrouter.example/v1", "https://inference.example/v1"
model := "configured-model"
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{
{Source: promptkit.BackendSource("application backends", map[string]promptkit.BackendConfig{
openRouterID: {Endpoint: &openRouterEndpoint},
rakestrawhomeID: {Endpoint: &rakestrawhomeEndpoint},
})},
{Source: promptkit.ProfileSource("application profiles", map[string]promptkit.ProfileConfig{
"openrouter-profile": {Backend: &openRouterID, Model: &model},
"rakestrawhome-profile": {Backend: &rakestrawhomeID, Model: &model},
})},
}}},
}})
if err != nil {
t.Fatalf("construct engine with explicitly configured provider IDs: %v", err)
}
for profileID, backendID := range map[string]string{
"openrouter-profile": openRouterID,
"rakestrawhome-profile": rakestrawhomeID,
} {
inspection, err := engine.InspectProfile(context.Background(), profileID)
if err != nil || inspection.EffectiveModelParams.BackendID != backendID {
t.Fatalf("inspect %q = (%+v, %v), want backend %q", profileID, inspection, err, backendID)
}
}
}
func TestPublicConstructionCombinesEmbeddedOperatorAndProgrammaticInputs(t *testing.T) {
embedded, err := promptkit.FilesystemInputs("embedded defaults", fstest.MapFS{
"config/default.yaml": &fstest.MapFile{Data: []byte(`
@@ -114,14 +146,14 @@ profiles:
model: default-model
prompts:
hello:
"1":
default_profile: application
messages:
- role: user
content: Hello
output:
format: text
validation_mode: none
version: "1"
default_profile: application
messages:
- role: user
content: Hello
output:
format: text
validation_mode: none
`)},
}, "config")
if err != nil {
@@ -189,43 +221,94 @@ func TestPublicConstructionRejectsSuppliedInvalidMainInput(t *testing.T) {
}
}
func TestPublicConstructionCanOverlayMaintainedBackend(t *testing.T) {
endpoint := "http://local-openrouter.example/v1"
func TestPublicConstructionUsesProgrammaticBackendAndProfileSources(t *testing.T) {
endpoint, backendID, model := "http://application.example/v1", "application-backend", "synthetic-model"
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: promptkit.BackendSource("operator backend", map[string]promptkit.BackendConfig{
promptkit.BackendOpenRouter: {Endpoint: &endpoint},
})}}}},
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{
{Source: promptkit.BackendSource("operator backend", map[string]promptkit.BackendConfig{
backendID: {Endpoint: &endpoint},
})},
{Source: promptkit.ProfileSource("operator profile", map[string]promptkit.ProfileConfig{
"application-profile": {Backend: &backendID, Model: &model},
})},
}}},
}})
if err != nil {
t.Fatalf("construct engine with backend overlay: %v", err)
}
inspection, err := engine.InspectProfile(context.Background(), "mistral-small-3")
inspection, err := engine.InspectProfile(context.Background(), "application-profile")
if err != nil {
t.Fatalf("inspect catalog profile: %v", err)
t.Fatalf("inspect configured profile: %v", err)
}
if inspection.EffectiveModelParams.Endpoint != endpoint {
t.Fatalf("effective endpoint = %q, want %q", inspection.EffectiveModelParams.Endpoint, endpoint)
}
}
func TestPreparationRejectsAmbiguousPromptVersion(t *testing.T) {
definition := func(content string) map[string]any {
return map[string]any{
"messages": []any{map[string]any{"role": "user", "content": content}},
"output": map[string]any{"format": "text", "validation_mode": "none"},
}
}
func TestBackendSourceOwnsSmokeTestValuesAndDefersDefaultLookup(t *testing.T) {
backendEndpoint := "https://backend.example/v1"
defaultProfile := "not-yet-listed"
reachabilityEndpoint := " https://probe.example/ready?check=1 "
authenticationKind := " OPENROUTER_KEY "
authenticationEndpoint := "https://probe.example/key?scope=current"
source := promptkit.BackendSource("diagnostic backend", map[string]promptkit.BackendConfig{
"local": {
Endpoint: &backendEndpoint,
SmokeTests: &promptkit.BackendSmokeTestConfig{
DefaultProfile: &defaultProfile,
Reachability: &promptkit.ReachabilityProbeConfig{Endpoint: &reachabilityEndpoint},
Authentication: &promptkit.AuthenticationProbeConfig{
Kind: &authenticationKind, Endpoint: &authenticationEndpoint,
},
},
},
})
backendEndpoint = "https://mutated.example/v1"
defaultProfile = "mutated-profile"
reachabilityEndpoint = "https://mutated.example/ready"
authenticationKind = "none"
authenticationEndpoint = "https://mutated.example/key"
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
Main: &configkit.Input{Source: configkit.Mapping("versions", map[string]any{
"prompts": map[string]any{"hello": map[string]any{
"1": definition("first"), "2": definition("second"),
}},
})},
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: source}}}},
}})
if err != nil {
t.Fatalf("construct engine: %v", err)
t.Fatalf("construct engine with dangling diagnostic default: %v", err)
}
if result, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "hello"}); result != nil || !errors.Is(err, promptkit.ErrPromptLoad) {
t.Fatalf("ambiguous preparation = (%#v, %v), want ErrPromptLoad", result, err)
inspection, err := engine.InspectConfiguration()
if err != nil {
t.Fatal(err)
}
var effective bytes.Buffer
if err := inspection.WriteEffective(&effective); err != nil {
t.Fatal(err)
}
contents := effective.String()
for _, want := range []string{
"https://backend.example/v1",
"not-yet-listed",
"https://probe.example/ready?check=1",
"openrouter_key",
"https://probe.example/key?scope=current",
} {
if !strings.Contains(contents, want) {
t.Errorf("effective configuration does not contain %q:\n%s", want, contents)
}
}
for _, unwanted := range []string{
"mutated.example",
"mutated-profile",
} {
if strings.Contains(contents, unwanted) {
t.Errorf("effective configuration unexpectedly contains %q:\n%s", unwanted, contents)
}
}
}
func TestPreparationSelectsSinglePromptDefinitionByID(t *testing.T) {
engine := newSimpleEngine(t, "selected")
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
if err != nil || prepared == nil || len(prepared.Messages) != 1 || prepared.Messages[0].Content != "selected" {
t.Fatalf("prepare prompt by ID = (%+v, %v)", prepared, err)
}
}

View File

@@ -63,11 +63,12 @@ func simpleConfig(content string) promptkit.Config {
"profiles": map[string]any{"profile": map[string]any{
"endpoint": "http://example.test/v1", "model": "model",
}},
"prompts": map[string]any{"prompt": map[string]any{"1": map[string]any{
"prompts": map[string]any{"prompt": map[string]any{
"version": "1",
"default_profile": "profile",
"messages": []any{map[string]any{"role": "user", "content": content}},
"output": map[string]any{"format": "text", "validation_mode": "none"},
}}},
}},
})},
}}
}

View File

@@ -25,7 +25,6 @@ func toDomainRunRequest(req RunRequest) (domain.RunRequest, error) {
}
return domain.RunRequest{
PromptID: req.PromptID,
PromptVersion: req.PromptVersion,
ProfileID: req.ProfileID,
SessionID: req.SessionID,
APIKey: req.APIKey,
@@ -73,7 +72,7 @@ func fromDomainPreparedRun(prepared *domain.PreparedRun) *PreparedRun {
}
return &PreparedRun{
PromptID: prepared.PromptID,
PromptVersion: prepared.PromptVersion,
PromptVersion: copyStringPtr(prepared.PromptVersion),
PromptHash: prepared.PromptHash,
SelectedProfileID: prepared.SelectedProfileID,
SelectedBackendID: prepared.SelectedBackendID,
@@ -100,7 +99,7 @@ func fromDomainRunResult(result *domain.RunResult) *RunResult {
RawOutput: result.RawOutput,
Validation: fromDomainValidationResult(result.Validation),
PromptID: result.PromptID,
PromptVersion: result.PromptVersion,
PromptVersion: copyStringPtr(result.PromptVersion),
PromptHash: result.PromptHash,
SessionID: result.SessionID,
RenderedPromptHash: result.RenderedPromptHash,
@@ -211,6 +210,12 @@ func toDomainExecutionTargetOverride(override *ExecutionTargetOverride) (*domain
}
func fromDomainExecutionTarget(target domain.ExecutionTarget) ExecutionTarget {
return fromDomainExecutionTargetWithOwnedParams(target, copyAnyMap(target.ExtraParams))
}
// fromDomainExecutionTargetWithOwnedParams projects scalar settings and takes
// ownership of extraParams, which the caller has already copied.
func fromDomainExecutionTargetWithOwnedParams(target domain.ExecutionTarget, extraParams map[string]any) ExecutionTarget {
return ExecutionTarget{
BackendID: target.BackendID,
Endpoint: target.Endpoint,
@@ -222,7 +227,7 @@ func fromDomainExecutionTarget(target domain.ExecutionTarget) ExecutionTarget {
ServiceTier: target.ServiceTier,
ReasoningEffort: target.ReasoningEffort,
APIKeyEnv: target.APIKeyEnv,
ExtraParams: copyAnyMap(target.ExtraParams),
ExtraParams: extraParams,
}
}
@@ -252,7 +257,7 @@ func fromDomainPromptInspection(inspection *domain.PromptInspection) *PromptInsp
}
return &PromptInspection{
PromptID: inspection.PromptID,
PromptVersion: inspection.PromptVersion,
PromptVersion: copyStringPtr(inspection.PromptVersion),
PromptHash: inspection.PromptHash,
DefaultProfileID: inspection.DefaultProfileID,
Inputs: inputs,

25
doc.go
View File

@@ -4,15 +4,16 @@
// Applications construct an [Engine] with [NewEngine], select ConfigKit
// sources for prompts, profiles, and backends, and
// call [Engine.InspectConfiguration], [Engine.InspectPrompt],
// [Engine.InspectProfile], [Engine.Prepare],
// [Engine.PrepareExecution], [Engine.Run], or [Engine.RunPrepared]. Concrete
// registries, repositories, validators, and the built-in OpenAI-compatible
// client remain internal implementation details.
// [Engine.InspectRequirements], [Engine.InspectProfile], [Engine.ListProfiles],
// [Engine.ListPrompts], [Engine.Prepare], [Engine.PrepareExecution],
// [Engine.Run], or [Engine.RunPrepared]. Concrete registries, repositories,
// validators, and the built-in OpenAI-compatible client remain internal
// implementation details.
//
// # Concurrency and ownership
//
// An Engine supports concurrent InspectConfiguration, InspectPrompt,
// InspectProfile, Prepare,
// InspectRequirements, InspectProfile, ListProfiles, ListPrompts, Prepare,
// PrepareExecution, Run, and RunPrepared calls. Engine-local backend policies
// bound admitted Run and RunPrepared calls and model generations where
// configured, while different backend pools and unlimited backends continue
@@ -20,11 +21,12 @@
// still receive concurrent calls and must be safe for that use.
//
// ConfigKit captures the selected configuration when NewEngine constructs an
// Engine. Prepare,
// PrepareExecution, and Run copy request maps, slices, pointer values, and
// JSON-compatible extra parameters before using them. InspectPrompt and
// InspectProfile return copied inspection values. Returned values and values
// passed to extension interfaces are likewise isolated from engine state.
// Engine. Prepare, PrepareExecution, and Run copy request maps, slices, pointer
// values, and JSON-compatible extra parameters before using them. InspectPrompt,
// InspectRequirements, and InspectProfile return copied inspection values.
// ListProfiles and ListPrompts return caller-owned identity slices.
// Returned values and values passed to extension interfaces are likewise
// isolated from engine state.
// Callers own those copies and may mutate them after the call that supplied or
// returned them. [CapacityError] values are caller-owned and may be mutated
// without affecting engine state or another error. Immutable [GenerationError]
@@ -58,7 +60,8 @@
// [ConfigurationSources], [ConfigurationInspection], [BackendConfig],
// [ProfileConfig], [RunRequest],
// [ArtifactRef], [ExecutionTargetOverride], [ProfileInspection],
// [PromptInputDefinition], [PromptInspection], [PreparedExecution],
// [PromptIdentity], [PromptInputDefinition], [PromptInspection],
// [RequirementInspectionRequest], [RequirementInspection], [PreparedExecution],
// [CapacityError], [GenerationError], [RetryExhaustedError], and
// [ExecutionEvent], do not have stable JSON
// representations. Direct API keys are nevertheless excluded from JSON for

View File

@@ -0,0 +1,165 @@
# Configuration And API Migration
Promptkit no longer installs a provider catalog or selects prompt definitions
by descriptive version labels. Applications now choose configuration inputs
explicitly, and execution selects one prompt by its exact ID. This guide
explains the shape changes and decisions an application must make; it is not an
automatic migration tool. The [framework format reference](../formats.md) owns
the current configuration contract, and the root package GoDoc owns exact API
signatures and errors.
## Select And Compose The Configuration You Need
Supply each backend, profile, prompt, schema, and collection source that the
application intends to use. Promptkit does not discover directories or install
provider definitions implicitly. An application may select an explicit
collection input with `CollectionInput`:
```go
input, err := promptkit.CollectionInput(
"openrouter",
configkit.File("config/openrouter.yaml"),
)
if err != nil {
return err
}
engine, err := promptkit.NewEngine(promptkit.Config{
Sources: promptkit.ConfigurationSources{
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{input}}},
},
})
```
Collection resource keys are local keys, while their runtime identities are
qualified. For example, `collections.openrouter.profiles.gemini-flash-latest`
is selected as `openrouter/gemini-flash-latest`. Stored backend, prompt-default,
and other resource references must use their exact intended IDs; Promptkit does
not rewrite references when a source is mounted. Profile `extends` remains a
sibling local key in the same collection, not a cross-collection reference.
Review collisions and all references when moving resources into collections.
The root `version` is an optional ConfigKit source header. A collection's
`release_version` and a prompt's `version` are optional descriptive labels;
neither selects a resource or changes execution. Preserve a label only when it
is useful to your application. A formerly version-keyed prompt mapping must be
reduced to one deliberately chosen definition per prompt ID, with its old label
retained as prompt metadata if desired. Promptkit cannot choose a winner from a
consumer's version history.
For example, change a version-keyed definition:
```yaml
prompts:
report:
"2.0":
messages:
- role: user
content: Review this report.
output:
format: text
validation_mode: none
```
to one direct definition. Its optional `version` is only a label:
```yaml
prompts:
report:
version: "2.0"
messages:
- role: user
content: Review this report.
output:
format: text
validation_mode: none
```
## Identity Mapping
The reviewed catalog identities have slash-free profile keys. Their documented
mapping is:
| Previous identity | Namespaced identity or action |
| --- | --- |
| Backend `openrouter` | `openrouter/default` |
| Backend `rakestrawhome` | `rakestrawhome/default` |
| Each OpenRouter catalog profile `p` | `openrouter/p`, retaining the exact local key `p` |
| Profile `rakestrawhome-gemma-4-31b` | `rakestrawhome/rakestrawhome-gemma-4-31b` |
| Built-in `promptkit-openrouter-standard` | No automatic replacement; optionally define caller profile `openrouter/standard` |
| Built-in `promptkit-openrouter-auto` | No automatic replacement; optionally define caller profile `openrouter/auto` |
These are conversion examples, not aliases or bundled resources. Mounting old
`/v3` assets does not update backend references, profile inheritance, prompt
defaults, or diagnostic defaults inside them. Edit those values to the exact
qualified IDs you chose.
For example, an optional caller-owned standard profile may extend a selected
Gemini profile in the same `openrouter` collection:
```yaml
collections:
openrouter:
profiles:
standard:
extends: gemini-flash-latest
```
An optional auto profile may instead select the exact backend and model values:
```yaml
collections:
openrouter:
profiles:
auto:
backend: openrouter/default
model: openrouter/auto
```
Neither optional profile name is a Promptkit contract. Keep provider model
strings and settings as selected by the application, including any meaningful
prefixes. If a backend declares diagnostic defaults, update them to exact
qualified IDs; an unselected diagnostic default is not automatically made
resolvable by migration.
## Prompt Selection API
Prompt selection is ID-only. Remove version selectors from calls to `Run`,
`Prepare`, `PrepareExecution`, `ValidateRequest`, `InspectRequirements`, and
`SmokeTest`. Remove the `PromptVersion` field from `RunRequest`,
`RequirementInspectionRequest`, and `SmokeTestRequest`; `InspectPrompt` now
has the form `InspectPrompt(ctx, promptID)`. For example, select the prompt
explicitly in the request rather than setting a version selector:
```go
request := promptkit.RunRequest{
PromptID: "support/report",
ProfileID: "support/standard",
}
_, err = engine.Run(ctx, request)
if err != nil {
return err
}
```
If the former format contained multiple versions under one ID, choose the one
definition the application should install, and optionally keep its label in
the prompt's `version` field. That label remains descriptive metadata and is
reported by inspection or results where applicable; it never chooses the
definition.
The public inventory APIs can help an operator inspect a captured effective
configuration: `ListProfiles`, `ListPrompts`, and `ListBackends` return exact
identities, while `ListNamespaces` and `ListCollections` expose resource-bearing
namespaces and effective collection declarations. Inventory does not select
which configuration version the application should deploy.
## Deployment Selection Remains With The Application
Applications decide which checked-in, embedded, operator-supplied, or
programmatic inputs to compose for each deployment. They also decide when to
construct a replacement engine after changing those sources. Promptkit does
not fetch catalog updates, select provider availability, validate upstream
catalog release status, or automatically migrate arbitrary consumer files.
The maintained [offline collection example](../../examples/go-library/prepare/main.go)
shows explicit source selection, local imports, layering, a scoped view, and
operator inventory without a provider call.

View File

@@ -15,10 +15,30 @@ Import the package as:
import "gitea.maximumdirect.net/eric/promptkit"
```
The maintained [offline preparation example](../../examples/go-library/prepare/main.go)
demonstrates explicit collection composition and inventory. Applications
adapting older provider or versioned-prompt configuration can use the
[configuration migration guide](configuration-migration.md) for the required
caller-owned choices.
The following Go fragments are illustrative and omit surrounding package,
import, and error-handling code. Use the maintained examples for complete
programs.
## Go Compatibility
The `go` directive in the selected release's [module metadata](../../go.mod)
declares its minimum supported consumer Go version. Maintainers build and
validate releases with the locally selected Go patch release; the minimum
does not simply track that patch version.
Successful release validation on the selected patch is sufficient to presume
support across its Go series, subject to the declared minimum and concrete
evidence of incompatibility. Specific compatibility concerns are investigated
as they arise. Minor releases may raise the minimum; patch releases preserve
it. Release notes explain increases and any required consumer action. See the
[release policy](../policy/release.md#consumer-minimum-and-release-toolchain).
## Construct An Engine
Create an engine with [`NewEngine`](../../engine.go). Select each directory
@@ -49,6 +69,34 @@ new engine to observe configuration edits. See the [constructor GoDoc](../../eng
for the exact public contract and the [framework format reference](../formats.md)
for definition fields.
### Use Scoped Engine Views
Create a view when an application wants short request IDs to resolve within a
selected collection:
```go
scoped, err := engine.WithNamespaces(promptkit.Namespaces{
Prompt: "support",
Profile: "support",
})
if err != nil { return err }
prepared, err := scoped.Prepare(ctx, promptkit.RunRequest{
PromptID: "answer",
ProfileID: "concise",
})
```
Execution, preparation, prompt/profile inspection, requirement inspection, and
smoke diagnostics qualify slashless caller selections with their respective
namespaces. Already-qualified IDs pass through unchanged; configured defaults
and references are not rewritten. Each call replaces the view's scope defaults,
including clearing a scope with an empty field, without changing the receiver
or rebuilding its captured configuration and execution collaborators.
Namespace syntax is checked when creating the view; resource existence is
checked when selected. Configuration inspection remains global. See
[`WithNamespaces`](../../namespaces.go) for the exact public contract.
## Supply Embedded Application Defaults
Use `FilesystemInputs` for embedded defaults and `DirectoryInputs` for an
@@ -95,8 +143,8 @@ input := configkit.Input{Source: document}
```
`ProfileSource` and `BackendSource` are smaller typed conveniences when null
is not needed. A caller underlay can overlay catalog backend fields just as it
can overlay profiles; there is no registration-only backend namespace.
is not needed. Backend and profile definitions are supplied by the caller;
there is no registration-only backend namespace.
When configuration or message files change, build and validate a new engine,
then switch the application to that engine for subsequent work. Existing
@@ -104,6 +152,53 @@ engines and prepared handles retain their captured definitions; schema and
request artifact resources still follow their runtime lifecycle. Inspection
does not watch or reload sources automatically.
## Discover Available Resources And Collections
Use [`Engine.ListProfiles`](../../listing.go),
[`Engine.ListPrompts`](../../listing.go), and
[`Engine.ListBackends`](../../listing.go) to discover exact identities from the
engine's final captured configuration. A scoped view filters by its respective
namespace by default; pass `NamespaceNamed`, `NamespaceUnqualified`, or
`NamespaceAll` to override that listing filter without changing execution
scope. Returned IDs remain full identities and can be used for exact selection:
```go
profiles, err := engine.ListProfiles(ctx)
if err != nil {
return err
}
if len(profiles) > 0 {
_, err := engine.InspectProfile(ctx, profiles[0])
if err != nil {
return err
}
}
prompts, err := engine.ListPrompts(ctx)
if err != nil {
return err
}
if len(prompts) > 0 {
selected := prompts[0]
inspection, err := engine.InspectPrompt(ctx, selected.ID)
if err != nil {
return err
}
_ = inspection
}
```
Use [`Engine.ListNamespaces`](../../listing.go) to discover resource-bearing
namespaces across the entire captured configuration, independent of view
scopes. [`Engine.ListCollections`](../../listing.go) reports effective
collection declarations, including metadata-only entries; it does not infer
collection metadata from resource membership or source history.
These lists describe effective membership, not ConfigKit contribution history,
and do not promise that a profile is ready to execute. Use inspection for
details and preparation to resolve request resources. The linked method GoDoc
owns ordering, ownership, concurrency, and error behavior; identifiers may be
sensitive application data.
## Inspect The Captured Configuration
Use [`Engine.InspectConfiguration`](../../configuration_inspection.go) for an
@@ -130,6 +225,104 @@ shows backend-completed execution settings for one profile, and `Prepare` or
`PreparedExecution.Details` shows request-specific rendered messages, inputs,
and validation resources. None of those views is a substitute for the others.
To review changes between two independently captured configurations, compare
their inspection views:
```go
earlier, err := earlierEngine.InspectConfiguration()
if err != nil {
return err
}
later, err := candidateEngine.InspectConfiguration()
if err != nil {
return err
}
differences, err := earlier.Diff(later)
if err != nil {
return err
}
for _, difference := range differences {
if difference.HiddenChanged {
// Report a protected change without inferring its value.
}
if difference.Left.Redacted || difference.Right.Redacted {
// Apply the application's disclosure policy before presenting snapshots.
}
}
```
See the [`ConfigurationInspection.Diff` GoDoc](../../configuration_inspection.go)
and ConfigKit's [v0.8.0 effective-configuration diff guidance](https://gitea.maximumdirect.net/eric/ConfigKit/src/tag/v0.8.0/docs/config.md#effective-configuration-diffs)
for the exact comparison contract. The operation compares only frozen effective
configuration values: it does not reread external files or compare their
contents, resolved credential contents, later execution preparation, or
request-specific overrides. An empty diff is not proof of equivalent execution.
Records can contain ordinary application data, so use the application's
disclosure policy when presenting them.
## Run Explicit Smoke Diagnostics
[`Engine.SmokeTest`](../../smoke_diagnostics.go) runs one explicit diagnostic
using a target resolved from the engine's captured configuration. It does not
execute the selected application prompt or its inputs, session, artifacts, or
output schema. Unlike the offline examples and default test suite, calling it
can contact the selected provider and use configured credentials. It is a
consumer operation, not an automatic readiness verdict or a release check.
For example, a caller can request the built-in text inference check and treat
its optional fact as an observation rather than assuming that a nil Go error
means success:
```go
report, err := engine.SmokeTest(ctx, promptkit.SmokeTestRequest{
Kind: promptkit.SmokeTestTextInference,
ProfileID: profileID,
APIKey: apiKey,
})
if err != nil {
if report != nil {
retainDisclosureSafeSummary(report)
}
return err
}
if report != nil {
matched := report.Facts.ExpectedTextMatched
switch {
case matched == nil:
// The comparison was not evaluated or is unknown/inapplicable.
case *matched:
// The private expected-text comparison matched.
default:
// It did not match; the operation may still have concluded normally.
}
}
```
A nil Go error means the diagnostic concluded, not that its finding was
positive. Fact pointers are tri-state: nil means unknown, not evaluated, or
inapplicable. Reports contain factual observations and effective settings, not
generated text or provider bodies; identifiers and settings can still be
sensitive. The
[`SmokeTest` GoDoc](../../smoke_diagnostics.go) owns selection precedence,
exact fields, timeout and retry behavior, supported transports, and disclosure
details.
Caller cancellation can return both an error and a partial report. Preserve the
context error identity and inspect any available observations before returning:
```go
report, err := engine.SmokeTest(ctx, request)
if report != nil {
retainDisclosureSafeSummary(report)
}
if err != nil {
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
// The report may describe work completed before interruption.
}
return err
}
```
## Inspect A Prompt Before Preparation
Use [`Engine.InspectPrompt`](../../engine.go) to check one configured prompt's
@@ -137,7 +330,7 @@ declared inputs and output workflow without creating placeholder inputs or
resolving a profile:
```go
inspection, err := engine.InspectPrompt(ctx, "meeting.summary", "")
inspection, err := engine.InspectPrompt(ctx, "meeting.summary")
if err != nil {
return err
}
@@ -154,6 +347,85 @@ rendered messages, and use prepared execution when that work must remain tied
to later execution. The method's [GoDoc](../../engine.go) owns exact fields,
hash, ownership, and error semantics.
## Inspect Requirements Before Preparation
Use [`Engine.InspectRequirements`](../../requirements.go) when the application
needs the prompt's recognizable input and caller-variable dependencies for a
particular context before it has input artifacts. Unlike `InspectPrompt`, it
also walks captured template syntax and selects a profile identity; unlike
`Prepare`, it does not load artifacts or render messages.
```go
requirements, err := engine.InspectRequirements(ctx, promptkit.RequirementInspectionRequest{
PromptID: "meeting.summary",
Vars: map[string]string{"audience": "project leads"},
})
if err != nil {
return err
}
for _, input := range requirements.Inputs {
if input.Kind == promptkit.RequirementRequired {
// Plan to bind this input before preparation.
}
if input.Kind == promptkit.RequirementOptional && input.Explanation != "" {
// A recognizable reference may occur only on a known skipped path.
}
}
for _, variable := range requirements.Variables {
if variable.Kind == promptkit.RequirementRequired && !variable.Supplied {
// The key is required even though this inspection did not receive it.
}
}
if !requirements.Complete {
// Preserve the partial inventory and account for its discovery issues.
}
```
Optional entries can include recognizable inputs and variables found only in
inactive branches. A missing required variable is reported separately from
discovery completeness: missing a value used only for direct text substitution
does not by itself make the inventory incomplete. Conversely, an unresolved
dependency-bearing branch can make discovery incomplete while retaining known
conditional entries. Do not treat an empty partial inventory as proof that no
input bindings are needed. See the method's [GoDoc](../../requirements.go) for
the exact classifications, supported syntax, ownership, and error contract.
## Validate A Concrete Request
Use [`Engine.ValidateRequest`](../../engine.go) when the application has a
complete `RunRequest` and needs to know whether that concrete invocation passes
preparation:
```go
request := promptkit.RunRequest{
PromptID: "meeting.summary",
Vars: map[string]string{"audience": "project leads"},
Inputs: map[string]promptkit.ArtifactRef{
"note": promptkit.Inline("Synthetic meeting notes"),
},
}
if err := engine.ValidateRequest(ctx, request); err != nil {
return err
}
```
Unlike `InspectRequirements`, this checks supplied values, the selected template
branches, and request bindings through the existing preparation path. It checks
only this invocation, does not aggregate errors, and loads every supplied input
according to ordinary preparation behavior, even if a branch does not use it.
It makes no model call and returns no prepared snapshot, but artifact and schema
resource I/O and its time and memory costs remain. Success does not certify
provider acceptance or later execution success. See the method's GoDoc for its
exact error and context behavior.
Scriptorium can map the returned error to its own diagnostics and exit status.
Its `validate` and `run` commands should construct equivalent requests for
equivalent caller inputs so their preparation behavior agrees. Separate
subprocess calls prepare independently; an opaque `PrepareExecution` handle
cannot be transferred from `validate` to `run`. Use `PrepareExecution` when one
process and engine must execute the exact prepared snapshot.
## Prepare Without Model Execution
[`Engine.Prepare`](../../engine.go) resolves the selected prompt and profile,
@@ -174,8 +446,11 @@ prepared, err := engine.Prepare(ctx, promptkit.RunRequest{
The maintained
[offline preparation example](../../examples/go-library/prepare/main.go)
shows a complete runnable setup with a prompt file, in-memory profile, and
inline input. Exact request requirements and prepared-result fields belong to
the [`RunRequest` and `PreparedRun` GoDoc](../../types.go).
inline input. It demonstrates validation and preparation with the same request
to make the two operations concrete; this performs preparation twice and is
illustrative, not a requirement to validate before `Prepare` or `Run`. Exact
request requirements and prepared-result fields belong to the
[`RunRequest` and `PreparedRun` GoDoc](../../types.go).
## Prepare Now And Execute The Same Snapshot Later
@@ -362,96 +637,28 @@ are keys in configuration, not registrations. Choose caller underlays in
fallback-to-operator-to-programmatic order; use `Sources.Main` for one final
highest-priority input.
Move embedded `id` and `version` fields into keyed `profiles` and `prompts`
maps; convert `base_profile` to ConfigKit `extends`. IDs and references are
exact literal keys, without trimming. A supplied zero, false, or empty string
is now distinguishable from omission; null can clear or suppress an inherited
field according to the [field table](../formats.md#defaults-and-overrides).
Maps merge recursively across inheritance and layers. `{}` does not clear an
inherited map: use null to clear the whole map. At request time, a literal
`extra_params["extends"]` is ordinary provider data, not an inheritance
instruction; a nonempty request ExtraParams map replaces configured params.
Changing a model or backend does not clear unrelated explicit fields, though
backend-derived defaults are applied after inheritance to the final backend.
Invalid final definitions, including unused ones, now fail `NewEngine`; only
request-dependent failures remain at preparation or execution. Prompt text
and static templates are captured at construction, not automatically reloaded
between runs. Replace the engine to pick up changed definitions or message
files. The [format reference](../formats.md) owns exact field and presence
rules; the maintained examples show complete runnable workflows.
Move embedded IDs into keyed `profiles` and `prompts` maps; each prompt ID now
contains one installed definition, with any descriptive `version` label inside
that definition. Convert `base_profile` to ConfigKit `extends`. For the current
collection identity mapping, ID-only prompt selection, and required
caller-owned choices, see the
[configuration migration guide](configuration-migration.md). Exact current
format, reference, inheritance, and presence semantics belong to the
[framework format reference](../formats.md). Prompt text and static templates
are captured at construction, not automatically reloaded between runs. Replace
the engine to pick up changed definitions or message files.
### Choose A Profile Ownership Model
Choose the profile source that matches the application's continuity needs:
- Select `promptkit-openrouter-standard` or `promptkit-openrouter-auto` when
the maintained literal name is the desired consumer contract. Their current
selections are in the [format reference](../formats.md#built-in-profile-catalog).
- Select an external catalog profile directly when the application accepts that
profile's catalog volatility. The selected external inventories are linked
from the [format reference](../formats.md#external-profile-inventories).
- Define a complete application-owned profile with an explicit backend or
endpoint, model, and settings when the application owns those choices.
An application profile that inherits an external base follows later changes to
that base and fails if the base disappears; it is not a frozen copy. Promptkit
tests the catalog versions selected in its module graph. Independently advancing
a catalog dependency can remove an inherited base and produce an untested
combination. Pinning the resolved module graph preserves the embedded data for
a build, but cannot preserve provider availability. Running binaries neither
download catalog updates nor automatically replace withdrawn models. See the
[release policy](../policy/release.md#catalog-profile-compatibility) for the exact
cross-version guarantees.
### Alias A Built-In Profile
Give an application-owned profile ID a built-in base when prompts should select
the application ID while inheriting the built-in target. The child can override
only the setting it owns:
```go
profile := promptkit.ProfileSource("application profiles", map[string]promptkit.ProfileConfig{
"weather-light": {
Extends: &standardID,
ReasoningEffort: &high,
},
})
```
Place `profile` in a ConfigKit underlay, with `standardID` set to
`"promptkit-openrouter-standard"` and `high` to `"high"`. Select
`weather-light` in a prompt or `RunRequest.ProfileID`; it remains the
reported selected profile. Definition IDs and inheritance references are exact
keys. If the base is an external profile, apply the volatility guidance
above when deciding whether to use an application-owned complete definition
instead.
### Use The Rakestrawhome Built-In Profile
Set `RAKESTRAWHOME_INFERENCE_API_KEY` in the application environment, then
select `rakestrawhome-gemma-4-31b` as an ordinary profile ID. For example, a
prepared result identifies the selected built-in through
`BackendRakestrawHome`:
```go
prepared, err := engine.Prepare(ctx, promptkit.RunRequest{
PromptID: "meeting.summary",
ProfileID: "rakestrawhome-gemma-4-31b",
Inputs: inputs,
})
if err != nil {
return err
}
if prepared.SelectedBackendID != promptkit.BackendRakestrawHome {
return fmt.Errorf("unexpected backend %q", prepared.SelectedBackendID)
}
```
The maintained catalog supplies this backend, and callers may overlay its
effective configuration through a later ConfigKit source. Direct request credentials
and runtime endpoint overrides remain supported under their ordinary GoDoc and
format contracts.
Supply backend and profile definitions through the configuration sources your
application selects. PromptKit does not install provider catalogs or built-in
profile inventories; conventional backend ID constants are names only. Define
the model, endpoint, credentials lookup, and settings that your application
intends to support. A profile may inherit a sibling profile by its local key in
the same root or collection mapping; its backend reference uses the exact
configured identity. Caller-selected source layers determine the final values. The
[format reference](../formats.md) describes the definition fields and
composition rules.
### Inspect A Profile Before Prompt Work

View File

@@ -43,13 +43,13 @@ Before starting work:
| Root public API | The [architecture policy](policy/architecture.md), [consumer guide](consumers/pkg-promptkit.md), [testing policy](policy/testing.md), and existing GoDoc. |
| Prompt, profile, or schema formats | The [framework format reference](formats.md), owning parser or validator package, and [documentation policy](policy/documentation.md). |
| Source loading or validation | The [framework format reference](formats.md), [internal source document](internal/sources.md), and owning package tests. |
| Maintained external catalogs or built-ins | The [internal source document](internal/sources.md), [framework format reference](formats.md), [testing policy](policy/testing.md), both catalog module repositories, each catalog's release procedure, and the selected module versions in `go.mod`. |
| Caller-supplied configuration sources | The [internal source document](internal/sources.md), [framework format reference](formats.md), and [testing policy](policy/testing.md). |
| Model-client behavior | The [OpenAI-compatible integration contract](integrations/openai-compatible-chat.md), [internal model-client document](internal/llm.md), and owning package tests. |
| Internal package implementation | The [architecture policy](policy/architecture.md), [internal component overview](internal/overview.md), and focused internal document listed for that package. |
| Tests or test fixtures | The [testing policy](policy/testing.md), owning package, and focused internal document listed by the component overview. |
| Maintained example | The [preparation](../examples/go-library/prepare/main.go), [execution](../examples/go-library/run/main.go), and [observation](../examples/go-library/observe/main.go) examples, [consumer guide](consumers/pkg-promptkit.md), [framework format reference](formats.md), and [documentation policy](policy/documentation.md). |
| Documentation | The [documentation policy](policy/documentation.md) and canonical owner of every affected contract. |
| Licensing, dependencies/catalog updates, copied material, or attribution | The [OSS policy](policy/oss.md), [project grant](../COPYRIGHT), [license texts](../LICENSE), and [attribution entries](oss/attribution.md); inspect the selected upstream source's notices. |
| Licensing, dependencies, copied material, or attribution | The [OSS policy](policy/oss.md), [project grant](../COPYRIGHT), [license texts](../LICENSE), and [attribution entries](oss/attribution.md); inspect the selected upstream source's notices. |
| Release preparation or publication, including release notes | The [release policy](policy/release.md), [release procedure](release.md), and [OSS discoverability rules](policy/oss.md#discoverability). |
For cross-cutting changes, follow every applicable row. Do not create
@@ -59,37 +59,43 @@ exist.
## Maintainer Validation
For ordinary work, select validation proportionate to the change. Do not repeat
successful checks for unchanged content merely to create a commit. The full
release-candidate checks have one executable entry point from the repository root:
successful checks for unchanged content merely to create a commit. During
development, run relevant Go tests and vet, format changed Go files, and execute
the maintained examples when they are affected:
```sh
./scripts/release-check.sh
go test ./...
go vet ./...
gofmt -w path/to/changed.go
python3 scripts/check-examples.py
git diff --check
```
The [script](../scripts/release-check.sh) owns the executable checks:
race-enabled tests, vet, the three offline consumer examples with semantic output
assertions, tracked Go formatting, local Markdown links, module/workspace
hygiene, whitespace, and the credential-pattern scan. It does not publish,
change files, validate release-note structure, or retire roadmaps. It requires Go,
Git, a POSIX shell, and Python 3. Tests and examples require no provider
credentials or live inference calls.
Release candidates use `./scripts/release.sh VERSION`, which runs the selected
profile checks and publishes the tag. The [release procedure](release.md)
describes its requirements and recovery. For an optional diagnostic of a
committed candidate without publication, run `./scripts/releasekit.sh check
VERSION`; this is not a required pre-publication step.
The script can run on intended uncommitted changes during development.
Formatting and documentation checks use tracked files; stage intended new
files before final validation so those checks include them. The release
procedure separately enforces a clean, committed candidate.
The standalone `python3 scripts/check-examples.py` command checks semantic
expectations for the three offline consumer examples. The focused
`python3 scripts/test-release.py` command exercises the release tooling against
disposable local repositories; use it when changing that tooling, not as a
recurring release gate. Tests and examples require no provider credentials or
live inference calls.
Ordinary validation can run on intended uncommitted work. Release validation
uses committed candidate contents and the selected local Go installation.
### Change Review
Inspect `git status --short --untracked-files=all` and the complete diff before
accepting a change. Reject unintended files, credentials, private data,
generated binaries, test output, downloaded assets, and template residue.
The automated credential scan does not replace review of the actual change.
For releases, follow the [release procedure](release.md). Its publication
command runs the full local checks once on the final committed candidate and
automates release-note structure and Git checks. Do not repeat ordinary change
review as a release sign-off. Roadmap retirement remains preparation housekeeping.
For releases, follow the [release procedure](release.md). Do not repeat its
candidate checks as a separate sign-off. Roadmap retirement remains preparation
housekeeping.
Follow the [agent commit policy](policy/agents.md#git-commit-policy) for local
commits. Ordinary commits may leave unrelated work in the workspace; preserve

View File

@@ -10,12 +10,14 @@ owns outbound wire behavior. [ConfigKit](https://gitea.maximumdirect.net/eric/Co
owns the generic YAML grammar, imports, layering, inheritance, and diagnostics.
This reference describes what Promptkit does with the final effective values.
A configuration document has keyed top-level `backends`, `profiles`, and
`prompts` mappings. Their keys are literal IDs, not filename-derived identities.
Prompt versions are a second keyed mapping. Keys and references must be
nonblank and remain exact: Promptkit does not trim or re-key them. Old embedded
`id`, `version`, and `base_profile` fields are invalid. Unknown fields are
errors.
A configuration document may contain keyed top-level `backends`, `profiles`,
and `prompts` mappings, plus a `collections` mapping of namespaced resource
groups. Resource keys are literal IDs, not filename-derived identities. Each
prompt ID maps directly to one definition; an optional `version` string is
descriptive metadata, not a selection key. Keys and references must be nonblank
and remain exact: Promptkit does not trim or re-key them. Old embedded `id` and
`base_profile` fields are invalid. Unknown fields are errors. Promptkit installs
no provider or profile catalog; callers select the definitions they need.
```yaml
backends:
@@ -28,16 +30,74 @@ profiles:
model: example-model
prompts:
meeting.summary:
"1.0.0":
default_profile: local-summary
messages:
- role: user
content: Summarize this note.
output:
format: text
validation_mode: none
version: "1.0.0"
default_profile: local-summary
messages:
- role: user
content: Summarize this note.
output:
format: text
validation_mode: none
collections:
support:
release_version: "1.4.0"
backends:
offline:
endpoint: https://example.invalid/v1
profiles:
base:
backend: support/offline
model: offline-model
fast:
extends: base
timeout_seconds: 30
prompts:
answer:
default_profile: support/fast
messages:
- role: user
content: Answer the selected caller input.
output:
format: text
validation_mode: none
```
## Source Headers, Collections, And Resource IDs
An input may omit a source header. When present, the source-root `version` is
reserved for ConfigKit's specification header and accepts only the exact string
`"1"` or strict decimal integer `1`; null and other values are invalid. This
header is distinct from a prompt's nested `version` label and a collection's
`release_version`. ConfigKit checks and consumes accepted headers on non-main
inputs; a declaration on the main input remains visible in explicit
configuration inspection. Headerless inputs still undergo ordinary final-value
validation. ConfigKit owns generic import and source-history behavior.
The root resource mappings contain unqualified IDs. Each `collections` entry
has a namespace key and may contain `release_version` plus the same three
resource mappings. Collection resource keys are local keys; their runtime IDs
are `<namespace>/<local-key>`. Root and collection keys may be the same. A
namespace and local key must be nonblank and slash-free, and references must
name their intended exact ID; mounting a source does not rewrite references.
For example, the `support/fast` profile above stores `support/offline` as its
backend ID. In `collections.support.profiles.fast`, ConfigKit `extends: base`
refers to a sibling profile in that same mapping. It is not a qualified or
cross-collection reference.
`release_version` and prompt `version` are optional opaque labels. Absent or
null means no label; an explicitly empty string is retained as a declared
label. Neither label selects a resource or changes execution. Runtime lookups
use the captured effective resource indexes. `InspectConfiguration` is an
explicit operator view of effective settings and provenance; source history is
not a second runtime resolver, and the retained effective shape is not flattened
or rewritten into those indexes.
For examples of explicitly selecting native collection inputs and inspecting
inventories, see the [consumer guide](consumers/pkg-promptkit.md) and its
[maintained preparation example](../examples/go-library/prepare/main.go). For
breaking identity and prompt-selection changes, see the
[configuration migration guide](consumers/configuration-migration.md).
Every effective backend, profile, and prompt definition is validated when
`NewEngine` constructs the engine, even if it is never selected. A profile may
be partial: an endpoint and model need only be complete after request overrides
@@ -51,36 +111,37 @@ message files on ordinary runs; construct a new engine to observe edits.
Schema and request artifact reads still happen during preparation. Use
`InspectConfiguration` for an explicit effective-configuration or provenance
report, `InspectProfile` for backend-completed profile settings, and preparation
for request-specific rendered details.
for request-specific rendered details. For identity discovery before inspection,
see the consumer guide's [resource and collection listing](consumers/pkg-promptkit.md#discover-available-resources-and-collections).
## Prompt Definitions
The `prompts` mapping selects a literal prompt ID, then a string version key:
The `prompts` mapping selects a literal prompt ID whose value is the definition:
```yaml
prompts:
meeting.summary:
"1.0.0":
default_profile: local-summary
description: Summarize a synthetic meeting note.
session_id: '{{.session}}'
inputs:
- name: note
required: true
content_type: text/plain
description: Meeting note to summarize.
messages:
- role: system
content: Return a concise summary.
cache_control:
type: ephemeral
ttl: 1h
- role: user
content_file: ./summary.user.md
output:
format: markdown
validation_mode: basic
repair_attempts: 0
version: "1.0.0"
default_profile: local-summary
description: Summarize a synthetic meeting note.
session_id: '{{.session}}'
inputs:
- name: note
required: true
content_type: text/plain
description: Meeting note to summarize.
messages:
- role: system
content: Return a concise summary.
cache_control:
type: ephemeral
ttl: 1h
- role: user
content_file: ./summary.user.md
output:
format: markdown
validation_mode: basic
repair_attempts: 0
```
`messages` must be nonempty and `output` must be a valid mapping. Optional
@@ -88,9 +149,8 @@ prompts:
fields may be absent or cleared with null where appropriate. A null whole
definition, required role, output format, or validation mode is invalid.
`required` and `repair_attempts` default to false and zero, respectively, and
are not nullable. A request that omits a prompt version succeeds only if that
prompt ID has exactly one effective version. `InspectPrompt` selects from the
same captured definitions without rendering or resolving a profile.
are not nullable. `InspectPrompt` selects the same captured definition without
rendering or resolving a profile.
### Inputs
@@ -124,6 +184,12 @@ Optional `session_id` uses the same template variables and input helper. Its
rendered value is trimmed, omitted if empty, and limited to 256 Unicode code
points. A nonblank direct request session ID bypasses the template.
`InspectRequirements` can enumerate recognizable dependencies from captured
templates before artifact content is available. Its conservative syntax
coverage does not change template validity or rendering; see the [consumer
guide](consumers/pkg-promptkit.md#inspect-requirements-before-preparation) and
the method's [GoDoc](../requirements.go) for that inspection boundary.
### Cache Control
Optional `cache_control` has required `type: ephemeral` and optional `ttl`,
@@ -149,28 +215,64 @@ last candidate and its failed validation result. Operational errors remain
errors. A request-level `OutputContract` replaces the whole configured one,
not individual fields; an empty request format defaults to `text`.
## Built-In Backends
## Backend Definitions
The selected catalog modules contribute these backend definitions as the
earliest underlay. Their settings can be overlaid by caller sources; these IDs
are not reserved against configuration changes.
| ID | Catalog endpoint | API-key environment name | Active generation limit |
| --- | --- | --- | ---: |
| `openrouter` | `https://openrouter.ai/api/v1` | `OPENROUTER_API_KEY` | 16 |
| `rakestrawhome` | `https://inference.ai.rakestrawhome.com/v1` | `RAKESTRAWHOME_INFERENCE_API_KEY` | 4 |
Backend definitions are supplied through caller-selected configuration
sources. The conventional backend IDs are names only; PromptKit does not
install definitions for them.
A backend definition may have `endpoint`, `api_key_env`,
`concurrency_limit`, `queue_capacity`, and `extra_params`. The final endpoint
is required and must be an absolute HTTP(S) URL with a host, no user
information, query, or fragment. An omitted or null API-key environment name
means no lookup. Omitted/null concurrency is zero (unlimited); a supplied
`concurrency_limit`, `queue_capacity`, `extra_params`, and `smoke_tests`. The
final endpoint is required and must be an absolute HTTP(S) URL with a host, no
user information, query, or fragment. An omitted or null API-key environment
name means no lookup. Omitted/null concurrency is zero (unlimited); a supplied
negative value is invalid. For positive concurrency, omitted/null queue
capacity defaults to 1024. An explicit queue capacity, including zero,
requires positive concurrency and must be nonnegative. Unlimited concurrency
has no queue. Provider maps contain JSON-compatible values; null entries are
ordinary JSON null. Unknown fields, including a raw `api_key`, fail.
`smoke_tests` is optional backend metadata for explicit diagnostic operations.
It may contain `default_profile`, `reachability`, and `authentication`:
```yaml
backends:
openrouter:
smoke_tests:
default_profile: application-profile
reachability:
endpoint: https://status.example/ready?check=1
authentication:
kind: openrouter_key
endpoint: https://openrouter.ai/api/v1/key
```
An omitted or null `smoke_tests` block disables these metadata defaults.
`default_profile` is preserved as an exact, nonblank profile ID; its existence
is checked only if an inference diagnostic selects that default. Omitted/null
`default_profile` means there is no inference default. Omitted/null
`reachability` or its endpoint uses the effective backend endpoint. An explicit
reachability endpoint is trimmed and must be an absolute HTTP(S) URL with a
host; unlike the backend base endpoint, its path and query are allowed. Null
means fallback, while an explicit blank value is invalid.
Omitted/null `authentication` means no authentication adapter is declared. A
non-null object requires `kind`, which is trimmed and case-normalized; the
recognized values are `none` and `openrouter_key`. Its omitted/null endpoint
uses the adapter's backend-relative destination (`openrouter_key` appends
`/key` to the effective backend base); an explicit endpoint follows the same
full-URL rules as reachability. `none` declares authentication inapplicable.
These values are configuration only: engine construction does not contact a
backend.
These fields are resolved through the ordinary ConfigKit source layers before
Promptkit validates the final effective values. A higher-priority mapping can
override individual nested fields; an explicit null can clear the whole
`smoke_tests` block or one of its nested objects. An endpoint set to null uses
its documented fallback, while an explicit blank endpoint is invalid. The
metadata does not change ordinary generation credential requirements or
provider settings.
## Profile Definitions
Profiles are keyed by literal ID and may inherit a named parent with
@@ -231,57 +333,36 @@ does not disable the caller context or HTTP transport timeout.
### Source And Profile Precedence
`NewEngine` composes catalog definitions first, Promptkit-owned definitions
next, then caller underlays in their given order, with an optional main input
last. Later tiers overlay earlier ones. Inputs within one underlay group are
strictly composed: conflicting contributions are errors, not priority
signals. The directory helpers discover `.yaml` and `.yml` files recursively
`NewEngine` composes caller underlays in their given order, followed by an
optional main input. Later layers overlay earlier ones. Inputs within one
underlay group are strictly composed: conflicting contributions are errors,
not priority signals. The directory helpers discover `.yaml` and `.yml` files recursively
in sorted order, but that order does not give one file higher precedence.
An explicitly selected missing/unreadable root fails; an empty root contributes
nothing. Imports are disabled by these helpers unless the caller enables them
on an input. Single files can use ConfigKit inputs directly.
Backends, profiles, and prompts all follow these same source rules. A later
profile or backend can change individual fields of a lower definition. A
request `ProfileID` takes precedence over a prompt `default_profile`; if
Root backends, profiles, and prompts, as well as collection declarations and
their resource mappings, follow these same source rules. `CollectionInput`
mounts one explicitly selected source at a collection destination and enables
ConfigKit imports for that input; it does not discover or select sources.
A later profile or backend can change individual fields of a lower definition.
A request `ProfileID` takes precedence over a prompt `default_profile`; if
neither is selected, preparation fails. Exact profile inspection selects one
explicit ID and does not consult a prompt default.
### Profile Inheritance
ConfigKit resolves `extends` on effective profile definitions. One profile
may inherit another keyed profile; presence, explicit null, and recursively
merged maps determine the final fields. References are literal IDs, including
IDs containing punctuation. There is no Promptkit-specific depth limit or
second base-chain resolver. Missing references and cycles fail at construction.
For generic inheritance and layer mechanics, consult
may inherit a sibling profile by local key in the same root or collection
mapping; collection inheritance cannot cross namespaces. Presence, explicit
null, and recursively merged maps determine the final fields. Other resource
references are exact IDs and use their qualified form when they select a
collection resource. There is no Promptkit-specific depth limit or second
base-chain resolver. Missing references and cycles fail at construction. For
generic inheritance and layer mechanics, consult
[ConfigKit](https://gitea.maximumdirect.net/eric/ConfigKit).
## Built-In Profile Catalog
Promptkit contributes two opt-in definitions; neither is selected by default:
| ID | Current definition |
| --- | --- |
| `promptkit-openrouter-standard` | Extends the selected OpenRouter catalog's `gemini-flash-latest` profile. |
| `promptkit-openrouter-auto` | Selects backend `openrouter` and model `openrouter/auto`. |
Select one through request `ProfileID` or prompt `default_profile`.
Caller layers may overlay these definitions or their catalog bases. The
[release policy](policy/release.md#catalog-profile-compatibility) owns their
cross-version compatibility boundary.
## External Profile Inventories
The selected catalog modules, not this document, own their profile inventories:
- [OpenRouter catalog v3.20260930.0 profiles](https://gitea.maximumdirect.net/eric/promptkit-backend-openrouter/src/tag/v3.20260930.0/catalog/profiles)
- [Rakestrawhome catalog v3.20260930.0 profiles](https://gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/src/tag/v3.20260930.0/catalog/profiles)
These are tested local data, not a promise of live provider availability.
The [release policy](policy/release.md#catalog-profile-compatibility) owns
the compatibility boundary.
## Schemas
Schemas are JSON documents selected by a prompt or request `schema_path`.

View File

@@ -45,6 +45,45 @@ precedence over the backend default. Only the name reaches prepared metadata;
the environment value is read just before the provider call and is never added
to the JSON body.
## Explicit Smoke Diagnostics
Promptkit's explicit smoke diagnostics are separate consumer operations; they
are not part of ordinary `Run`, the offline examples, or release validation.
With the built-in client, inference diagnostics use the same configured client
and `/chat/completions` request as generation. Their content cases are private
to Promptkit and are not application prompt, schema, or provider protocol
extensions. The public
[`Engine.SmokeTest` GoDoc](../../smoke_diagnostics.go) owns request selection,
report facts, and injected-client behavior.
HTTP-only diagnostics use the configured built-in HTTP client. Reachability is
one unauthenticated `GET` to the selected diagnostic URL; an observed response
counts as reachable even when its status is non-2xx. Authentication behavior
is selected from backend metadata, not inferred from the host. The
`openrouter_key` adapter makes one `GET` to `/key` relative to the effective
backend base, unless configuration supplies a full diagnostic endpoint (see
the [backend format reference](../formats.md#backend-definitions)). It may send
the available bearer credential, but can dispatch without one. A 401 is
recognized as rejection. A 200 is recognized as acceptance only when its JSON
object has a `data` object containing string `label`, Boolean `is_free_tier`,
and numeric `usage`; unrelated fields and account limits are ignored. This is
only a minimal identifying subset of OpenRouter's documented
[current-key response](https://github.com/OpenRouterTeam/docs/blob/main/api_reference/limits.mdx#checking-your-limits),
not validation of the full account document. Other statuses and unrecognized
200 responses do not establish acceptance.
Diagnostic HTTP evidence describes the last request observed at Promptkit's
RoundTripper boundary and any response associated with it. It includes the
selected destination without URL user information, response status, and
credential presence when observed. Redirects can change the last observed
destination or strip credentials. This evidence does not prove that bytes
reached a peer or reveal dispatch hidden inside a custom transport. Diagnostic
reports do not retain headers, credentials, provider
bodies, or generated text. A consumer-injected `LLMClient` can support inference
diagnostics, but does not establish HTTP status, destination, reachability, or
upstream credential-supply facts; Promptkit does not fall back to a built-in
transport for HTTP-only diagnostics.
## Request Body
The request body always contains `model` and `messages`. The execution
@@ -78,9 +117,8 @@ The client conditionally includes:
The engine resolves backend, profile, and request extra-parameter maps by
whole-map replacement rather than key merging. The resulting effective map is
then merged directly into the top-level body after JSON serialization is
verified. Empty keys and collisions with these reserved fields are rejected
before any provider call:
merged into the top-level body. Empty keys, values that cannot be encoded, and
collisions with these reserved fields are rejected before any provider call:
- `model`
- `session_id`

View File

@@ -88,6 +88,17 @@ release, including canceled acquisition without dispatch. A retry wait holds
the logical admission lease but no active permit. Observer callbacks must not
block on work that requires the same run's admission slot.
## Smoke-Test Capacity
The engine's smoke tester shares this manager and generation coordinator with
the ordinary runner. Inference diagnostics reserve the same logical admission
slot and acquire the same backend-scoped active permits for actual model
attempts, including retries. They therefore participate in the same configured
backend limits as `Run`. Reachability and authentication HTTP probes do not
enter generation admission or consume active permits. The public
[`Engine.SmokeTest` GoDoc](../../smoke_diagnostics.go) owns the caller-visible
operation and injected-client limits.
## Cancellation And Release
Admission checks the caller context before reserving a slot. A call canceled
@@ -127,4 +138,7 @@ cover shared and independent backend pools during retry and repair. The
own the public prepared-capacity boundary. The
[root error-boundary tests](../../errors_internal_test.go) own preservation of
the public generation category and context identity when generation is
canceled.
canceled. The assembled
[smoke diagnostic contract tests](../../smoke_integration_contract_test.go)
verify that inference diagnostics share the engine's Run admission and recover
capacity after cancellation or failure.

View File

@@ -41,9 +41,22 @@ original error chain. This keeps injected attempt classifications consistent
with terminal observations without parsing diagnostic text or retrying injected
clients.
The diagnostic path uses the same private attempt and retry implementation as
ordinary generation. It retains invocation-local transport, response,
generation, usage, capacity-wait, and retry-delay evidence before normal
result-discard and cancellation checks. Built-in preparation also supplies its
frozen request URL to the evidence path before attempts begin; injected
generation has no equivalent HTTP destination. This evidence is returned to
diagnostic orchestration; it is not added to public `ExecutionEvent` values or ordinary
observer accounting. The [model-client document](llm.md) describes the
transport-observation boundary, and the root
[`SmokeTestReport` GoDoc](../../smoke_diagnostics.go) owns report semantics.
Focused coordinator tests use controlled transport and wait functions. They
cover eligibility, exhaustion, cancellation, backoff, request freezing, permit
release, and the injected-client path. The capacity manager tests own permit
release, and the injected-client path. The
[evidence tests](../../internal/generation/evidence_test.go) own diagnostic
attempt retention and retry evidence. The capacity manager tests own permit
acquisition and idempotent release. Public
[integration tests](../../reliable_execution_contract_test.go) cover retry,
repair, observation, frozen prepared execution, and shared-backend behavior

View File

@@ -34,9 +34,13 @@ resolved request target may supply the endpoint. Generation preparation then:
1. validates shared execution-setting invariants and the final selected base
endpoint;
2. maps the internal request into the OpenAI-compatible chat payload;
3. validates and merges extra parameters;
3. checks extra-parameter keys against empty and reserved names, then merges
their values into the payload;
4. composes `/chat/completions` through parsed URL path operations;
5. resolves authentication and freezes the URL, body, and optional credential.
5. encodes the complete body once on success. If encoding fails, extra values
are checked individually to retain invalid-request classification; otherwise
the complete-request failure remains a request failure;
6. resolves authentication and freezes the URL, body, and optional credential.
Each execution of that prepared request creates a fresh HTTP request, body
reader, and attempt timeout context. It closes the response body and returns
@@ -55,6 +59,33 @@ This client does not schedule retries; the [generation coordinator](generation.m
does so around its prepared attempts. The client has no tool-call support,
provider catalog, inbound HTTP behavior, or durable session store.
## Diagnostic HTTP Evidence
Built-in diagnostics wrap the selected HTTP client's `RoundTripper` for one
invocation using a shallow client copy, preserving its timeout, redirect policy,
cookie jar, and underlying transport without mutating the configured client.
Evidence distinguishes a diagnostic dispatch invocation from an observed
request and response. A newly observed request replaces the previous
destination-specific response evidence, so redirects retain only the last
observed hop. A response is recorded before envelope decoding or later context
checks can discard a generation result.
The collector keeps only the last observed destination (without URL user
information), status, credential-presence, response-availability, decoded
generation, and presence-aware usage facts needed by the diagnostic. It does
not retain headers, credentials, raw request or response bodies, or a redirect
trace. These are observations at Promptkit's RoundTripper boundary, not proof
that bytes reached a peer or that a custom transport did no hidden dispatch.
Reachability and named authentication probes use the same configured built-in
client but discard their response bodies after local recognition. The
[integration contract](../integrations/openai-compatible-chat.md#explicit-smoke-diagnostics)
owns their observable HTTP semantics.
An injected `LLMClient` does not pass through this HTTP boundary. Its returned
response can establish response and inference facts, but not an HTTP
destination, status, reachability, or upstream credential-supply fact. The
facade does not silently install a built-in transport for those diagnostics.
## Prepared Generation
For [`RunPrepared`](../../engine.go), the runner supplies the model client with
@@ -133,7 +164,10 @@ own configuration, client cloning, deterministic deadline precedence,
authentication, request and response mapping, malformed data, error identity,
cancellation, endpoint selection and composition, pre-transport rejection, and
bounded single-document successful-response framing, closure, and
response-body suppression. The focused
response-body suppression. The [HTTP evidence tests](../../internal/llm/http_evidence_test.go)
own invocation-local last-hop request/response facts, while the
[prepared-generation evidence tests](../../internal/llm/prepared_generation_evidence_test.go)
own retention across decoding and cancellation. The focused
[provider HTTP error tests](../../internal/llm/provider_http_error_test.go)
own envelope parsing, normalization, and bounded-reader cases; their
[transport tests](../../internal/llm/provider_http_error_transport_test.go)

View File

@@ -23,6 +23,10 @@ callback. The caller supplies normalized failure metadata; the recorder never
parses an error string or receives provider bodies, credentials, headers, or
other content. Events are values with no mutable containers. Callback panics
propagate to the caller.
Resolved prompt and profile IDs are the exact runtime identities selected for
the invocation, including namespace qualification when supplied by an engine
view. A prompt's optional version label is descriptive metadata, not a
selection key.
Per-operation callbacks are ordered and synchronous; different operations may
invoke the same callback concurrently. The recorder marks each measured phase's
end before delivery, so callback work does not inflate provider duration.
@@ -41,6 +45,11 @@ attempts above the first in a generation, and repair generations with at least
one dispatch. A terminal event is an aggregate and must not be added to its
attempt events.
Smoke diagnostics mask inherited execution-recorder state while invoking
generation or HTTP collaborators. Their private attempt evidence is assembled
into a `SmokeTestReport`; it does not create ordinary run events or enter an
enclosing run's observation totals.
The [recorder tests](../../internal/observation/recorder_test.go) own usage
presence, phase sums, event ordering, empty-run accounting, identity snapshots,
and callback panic behavior. The [generation coordinator](generation.md) owns

View File

@@ -11,28 +11,27 @@ contributor workflow and validation.
| Component | Implemented responsibility | References |
| --- | --- | --- |
| Root `promptkit` package | Provides the supported engine facade, ConfigKit source helpers, profile and backend mapping conveniences, injection options, public requests and results, explicit inspection, observation values, opaque prepared-execution handles, value conversion, redacted formatting, and typed capacity and generation error mapping. | [Package GoDoc](../../doc.go), [observations](../../observations.go), [prepared execution](../../prepared_execution.go), [backend API](../../backends.go), [engine facade](../../engine.go) |
| `examples/go-library/prepare` | Demonstrates offline preparation and explicit configuration inspection with a prompt file, programmatic profile, and inline input. It is not a public library package. | [Example program](../../examples/go-library/prepare/main.go) |
| Root `promptkit` package | Provides the supported engine facade, ConfigKit source helpers, collection and resource mapping conveniences, injection options, public requests and results, prompt/profile/requirement inspection and identity inventories, concrete-request validation, explicit smoke diagnostics, observation values, opaque prepared-execution handles, value conversion, redacted formatting, and typed capacity and generation error mapping. | [Package GoDoc](../../doc.go), [observations](../../observations.go), [prepared execution](../../prepared_execution.go), [backend API](../../backends.go), [engine facade](../../engine.go), [smoke diagnostics](../../smoke_diagnostics.go), [identity listing](../../listing.go), [requirement facade](../../requirements.go) |
| `examples/go-library/prepare` | Demonstrates offline preparation from two explicitly selected synthetic collections, source-relative imported content, a scoped view, filtered listings, and operator inventory. It is not a public library package. | [Example program](../../examples/go-library/prepare/main.go), [collection inputs](../../examples/go-library/prepare/collections/support/collection.yaml) |
| `examples/go-library/run` | Demonstrates an offline downstream consumer using a prompt file, programmatic profile, inline input, an injected deterministic model client, and `Run`. It is not a public library package. | [Example program](../../examples/go-library/run/main.go) |
| `examples/go-library/observe` | Demonstrates offline built-in HTTP retry and synchronous metadata observation across structured proposal and assessment runs using embedded sources, two profiles on one finite backend, and an application decision. It is not a public library package. | [Example program](../../examples/go-library/observe/main.go) |
| `internal/backend` | Holds a defensive, immutable lookup and capacity-policy index for final backend definitions validated by the configuration adapter. | [Backend registry](../../internal/backend/registry.go) |
| `internal/catalog` | Selects released keyed catalog assets and PromptKit-owned profile definitions as ConfigKit underlays; effective validation belongs to the configuration loader. | [Catalog adapter](../../internal/catalog/catalog.go), [internal sources](sources.md#profiles-and-built-ins) |
| `internal/configuration` | Loads keyed backend, profile, and prompt configuration through ConfigKit; records effective field presence, validates final values, captures indexed prompt definitions and referenced message content, constructs backend and presence-aware profile snapshots, and retains an explicit read-only inspection view. | [Typed loader](../../internal/configuration/load.go), [internal sources](sources.md#typed-configuration-loading) |
| `internal/backend` | Holds a defensive, immutable lookup of final backend definitions, including diagnostic metadata, plus capacity policies validated by the configuration adapter. | [Backend registry](../../internal/backend/registry.go) |
| `internal/configuration` | Loads root and namespaced collection resources through ConfigKit; records effective field presence, validates final values including exact references and optional diagnostic metadata, captures indexed prompt definitions and source-relative message content, compiled templates, input metadata snapshots, sorted backend/profile/prompt identities, resource namespace sets, effective collection declarations, backend and presence-aware profile snapshots, and the unchanged read-only inspection view. | [Typed loader](../../internal/configuration/load.go), [identity listing](../../internal/configuration/listing.go), [internal sources](sources.md#typed-configuration-loading) |
| `internal/capacity` | Owns engine-local bounded execution admission and FIFO model-generation permits for limited backend IDs, including cancellation-safe waiter removal and client wrapping. | [Internal capacity management](capacity.md) |
| `internal/domain` | Defines internal framework values for requests, artifacts, prompt definitions, profiles, execution targets, rendering, generation, and validation, and owns source-neutral invariants for shared execution settings, OpenAI-compatible base endpoints, session identifiers, and output contracts. Source parsing, required fields, other source-specific normalization, defaulting, and boundary-specific error classification remain with their callers. | [Domain declarations](../../internal/domain/domain.go), [endpoint invariant](../../internal/domain/endpoint.go) |
| `internal/engine` | Assembles selected catalog and caller configuration into one snapshot, then constructs capacity, rendering, validation, generation, repair, and the use-case runner for each engine. | [Engine assembly](../../internal/engine/engine.go), [internal sources](sources.md#profiles-and-built-ins), [internal runner](runner.md) |
| `internal/generation` | Coordinates frozen built-in HTTP attempts and one-shot injected calls through active backend permits, including bounded status retries, cancellable backoff, and attempt observations. | [Internal generation coordination](generation.md) |
| `internal/domain` | Defines internal framework values for requests, artifacts, prompt definitions, requirement inspection, profiles, execution targets, rendering, generation, validation, and source-neutral smoke diagnostics, and owns source-neutral invariants for shared execution settings, OpenAI-compatible base endpoints, session identifiers, output contracts, and resource-identity syntax and qualification. Source parsing, required fields, other source-specific normalization, defaulting, and boundary-specific error classification remain with their callers. | [Domain declarations](../../internal/domain/domain.go), [requirement values](../../internal/domain/requirements.go), [endpoint invariant](../../internal/domain/endpoint.go) |
| `internal/engine` | Assembles caller-selected configuration into one snapshot, then constructs shared capacity, rendering, validation, generation, repair, the ordinary runner, and the smoke tester for each engine. Immutable namespace views share those collaborators and qualify caller-supplied execution, inspection, and diagnostic selections without rewriting captured references. Built-in engines provide the diagnostic HTTP collaborator; injected-client engines do not. It delegates identity listing directly to the captured configuration inventory rather than the runner. | [Engine assembly](../../internal/engine/engine.go), [internal sources](sources.md#typed-configuration-loading), [internal runner](runner.md) |
| `internal/generation` | Coordinates frozen built-in HTTP attempts and one-shot injected calls through active backend permits, including bounded status retries, cancellable backoff, ordinary attempt observations, and invocation-local evidence for diagnostics. | [Internal generation coordination](generation.md) |
| `internal/defaults` | Defines application-neutral framework constants and constructs the default execution target. It contains no CLI, server, or inbound HTTP limits. | [Framework defaults](../../internal/defaults/defaults.go) |
| `internal/filecatalog` | Provides deterministic YAML discovery and path helpers for operating-system filesystems and `fs.FS` sources. | [File catalog](../../internal/filecatalog/catalog.go) |
| `internal/jsonvalue` | Validates and deeply copies bounded JSON-compatible extra-parameter and prepared-schema trees while preserving supported concrete value types and rejecting cycles or excessive depth and work. | [JSON values](../../internal/jsonvalue/jsonvalue.go) |
| `internal/promptdef` | Provides a narrow captured-prompt lookup contract, not-found identity, and execution-ready definition hash. | [Prompt lookup](../../internal/promptdef/repository.go), [framework formats](../formats.md) |
| `internal/profile` | Provides a narrow captured-profile lookup contract and not-found identity. | [Profile lookup](../../internal/profile/repository.go), [framework formats](../formats.md) |
| `internal/prompt` | Renders prompt messages from Go templates with artifact, variable, session, and cache-control data. | [Go-template renderer](../../internal/prompt/go_renderer.go) |
| `internal/prompt` | Renders prompt messages from Go templates with artifact, variable, session, and cache-control data, and conservatively enumerates dependencies from captured template syntax. | [Go-template renderer](../../internal/prompt/go_renderer.go), [requirement discovery](../../internal/prompt/requirements.go) |
| `internal/artifact` | Resolves ordinary inline and unrestricted caller-selected file references into copied artifacts with metadata and hashes. | [Internal sources and validation](sources.md) |
| `internal/validate` | Validates basic, JSON, and JSON Schema output using operating-system filesystem or `fs.FS` schema sources and creates operation-local validation plans with canonical contained schema resources. | [Framework formats](../formats.md#schemas), [internal sources and validation](sources.md) |
| `internal/llm` | Defines the internal generation boundary and implements outbound OpenAI-compatible chat requests from resolved execution targets, including bounded structured non-success response decoding, successful-response decoding, authentication, deadline handling, and ownership of the OpenAI-compatible reserved request-field policy. | [Internal model client](llm.md) |
| `internal/observation` | Owns invocation-local run IDs, presence-aware usage and phase accounting, and synchronous metadata event delivery for ordinary and claimed prepared runs and generation attempts. | [Internal execution observation](observation.md) |
| `internal/usecase` | Selects captured prompts and profiles, applies request overrides and completion checks, and coordinates ordinary and single-claim prepared execution across rendering, artifact loading, operation-local validation plans, generation, capacity, and bounded repair. | [Internal runner](runner.md), [prepared-execution implementation](../../internal/usecase/prepared_execution.go) |
| `internal/llm` | Defines the internal generation boundary and implements outbound OpenAI-compatible chat requests from resolved execution targets, including bounded response decoding, authentication, deadline handling, per-invocation HTTP evidence and built-in diagnostic probes, and ownership of the OpenAI-compatible reserved request-field policy. | [Internal model client](llm.md) |
| `internal/observation` | Owns invocation-local run IDs, presence-aware usage and phase accounting, and synchronous metadata event delivery for ordinary and claimed prepared runs and generation attempts; smoke diagnostics keep their private evidence outside ordinary observer events. | [Internal execution observation](observation.md) |
| `internal/usecase` | Selects captured prompts and profiles, enumerates requirements from captured prompt syntax, applies request overrides and completion checks, coordinates ordinary and single-claim prepared execution across rendering, artifact loading, operation-local validation plans, generation, capacity, and bounded repair, and runs explicit smoke diagnostics with private cases, shared generation/capacity, and optional built-in HTTP probes. | [Internal runner](runner.md), [requirement inspection](../../internal/usecase/requirements_inspection.go), [prepared-execution implementation](../../internal/usecase/prepared_execution.go) |
The internal engine assembles these internal components without exposing their
representations. Consumers depend only on the root facade.

View File

@@ -32,16 +32,36 @@ The runner has no durable run or session store.
## Shared Prompt Selection
The runner uses one prompt-selection and hashing boundary for ordinary
preparation and exact prompt inspection. Preparation retains its early
request-ID check before direct-session normalization; both operations select
one captured definition and its construction-time hash. Referenced message
content and static template syntax were captured when the engine was built.
The internal engine applies each namespace view to slashless caller selections
before they reach the runner. The runner therefore selects captured prompts and
profiles by the resulting exact runtime IDs. Already-qualified selections pass
through, while configured defaults and stored references are used as authored
and are not requalified. Prompt version labels are returned as optional
descriptive metadata; they are not prompt selectors.
Inspection stops after that structural lookup. It does not render templates or
touch profile, artifact, schema, renderer, validator, admission, or model
collaborators. The root [`Engine.InspectPrompt`](../../engine.go) GoDoc owns
the public operation's exact contract.
Ordinary preparation selects the full prompt definition and construction-time
hash. It retains its early request-ID check before direct-session
normalization. Referenced message content and static template syntax were
captured when the engine was built.
`InspectPrompt` selects captured metadata through an optional repository
capability, falling back to the full definition and existing hash resolution
for internal repositories without it. It returns owned metadata without
rendering templates or touching profile, artifact, schema, renderer, validator,
admission, or model collaborators. The root
[`Engine.InspectPrompt`](../../engine.go) GoDoc owns the public operation's
exact contract.
`InspectRequirements` uses the same metadata selection to resolve the
request-profile/default identity, then obtains the captured requirement
snapshot by the selected prompt ID. The analyzer receives only
those identity fields rather than an execution definition. It analyzes the
snapshot with the supplied caller variables and normalized direct-session
bypass; it does not resolve target readiness or enter the preparation
pipeline. Missing snapshot capability is a prompt-load failure rather than a
source fallback. The root
[`Engine.InspectRequirements`](../../requirements.go) GoDoc owns the public
operation's exact contract.
## Shared Profile Selection
@@ -57,6 +77,37 @@ artifact, schema, rendering, admission, or model-client work. The root
[`Engine.InspectProfile`](../../engine.go) GoDoc owns the public operation's
exact contract.
## Smoke-Test Diagnostics
The separate `SmokeTester` in `internal/usecase` selects from the same captured
configuration snapshot and backend/profile collaborators as the runner. The
internal engine applies namespace-view qualification to slashless caller IDs
before diagnostic selection; captured defaults and references remain exact.
Explicit profile selection takes precedence over prompt-default selection and a
backend diagnostic default; HTTP-only selection does not resolve a default
inference profile. Exact public selection rules belong to the root
[`Engine.SmokeTest` GoDoc](../../smoke_diagnostics.go).
Diagnostics do not call `Run`, `Prepare`, or the prompt preparation pipeline.
They do not render application messages, load application artifacts, use
application schemas, or inspect session templates. Private text and structured
cases and the structured case's schema remain internal to the diagnostic
operation; they are not entries in prompt/profile listings or inspection.
`InspectConfiguration` remains a separate, explicit operator-oriented view of
effective settings and source history. A diagnostic report may include selected
effective settings and identifiers, so consumers still apply their disclosure
policy.
The engine assembles the smoke tester with the runner's generation coordinator
and capacity manager. Inference holds the ordinary logical admission lease and
uses the same active permits, retries, timeout, and request evidence path as
generation. Reachability and authentication probes bypass generation capacity
and do not require a model. An engine with an injected `LLMClient` can run
inference diagnostics but has no built-in HTTP probe collaborator; HTTP-only
diagnostics report that limitation instead of falling back to another client.
The public GoDoc owns the caller-visible timeout, cancellation, report, and
transport semantics.
## Shared Preparation Pipeline
`Prepare` and `Run` share one private preparation pipeline split at the point
@@ -92,8 +143,17 @@ one execution-precedence and error-ordering implementation while allowing a
full backend pool to reject work before expensive schema, artifact, and
rendering operations. `Prepare` discards the plan after returning its public
metadata. `Run` retains the plan through initial and repaired-output validation
and discards it when the operation ends. Prepared execution stores the same
kind of plan only in its private payload.
and discards it when the operation ends. Prepared execution transfers the
completed preparation's owned message, input-hash, and copied-schema values
into its private payload, while deep-copying mutable provider extras and
creating an independent details snapshot. Claim completion or discard clears
the execution payload without removing the details snapshot.
The root facade's `Engine.ValidateRequest` uses the same complete preparation
path as `Prepare` and discards the resulting internal prepared value without
converting it into a public snapshot. It does not add a runner operation or
validator component; request conversion and public error mapping remain at the
facade boundary.
Pointer-based numeric overrides preserve an explicit zero. Invalid negative or
out-of-range values fail as invalid requests. Endpoint overrides do not change
@@ -234,4 +294,8 @@ Changes to orchestration should continue to use the existing package
interfaces, keep request state local to an invocation, and preserve the shared
resolution and completion pipeline. Source, renderer, validator, or
model-client contract changes belong first in their owning package and
document.
document. The
[smoke selection tests](../../internal/usecase/smoke_selection_test.go),
[diagnostic orchestration tests](../../internal/usecase/smoke_tester_test.go),
and [private inference-case tests](../../internal/usecase/smoke_inference_test.go)
own diagnostic selection, report assembly, and inference findings.

View File

@@ -14,61 +14,149 @@ source semantics. Internal packages are not consumer APIs.
effective configuration to execution-owned definitions. It performs one
layered load, captures effective presence (including explicit null), records
normalization and defaults through resolution, and validates all effective
backend, profile, and prompt values. Replaced lower-layer domain values are
not separately validated. It builds an immutable backend index and
presence-aware profile targets, applying final backend defaults only after
profile inheritance. It retains ConfigKit's result privately for explicit
inspection, not for per-run resolution. The runtime/domain packages do not
depend on ConfigKit.
root and collection backend, profile, and prompt values. Replaced lower-layer
domain values are not separately validated. It indexes root resources under
their unqualified IDs and collection resources under their full
`namespace/local-key` IDs while retaining ConfigKit's effective tree in its
authored shape. It builds an immutable backend index and presence-aware profile
targets, applying final backend defaults only after profile inheritance. Runtime
lookups use those final indexes and exact stored references; they do not rewrite
or resolve resources from ConfigKit source history. The runtime/domain packages
do not depend on ConfigKit.
ConfigKit owns source headers, imports, composition, and source evidence.
Promptkit does not preparse contributions: it validates the final effective
values once, captures the indexed identities and their original resource
contexts, and retains ConfigKit's result for explicit operator inspection.
Accepted non-main source headers are consumed during composition; the main
source declaration remains available in that inspection view. Prompt and
collection release labels are descriptive metadata, not selection inputs. The
[format reference](../formats.md#source-headers-collections-and-resource-ids)
owns the exact source-header and collection contracts.
The `ConfigurationInspection.Diff` facade method compares two retained results
through a private wrapper and a narrow `internal/configuration.Diff` bridge.
The wrapper keeps `Result[document]` inside the adapter; the bridge delegates
once to ConfigKit v0.8.0 and returns its public difference records. Comparison
uses only frozen effective values and does not reopen sources, resolve runtime
credentials, or enter the execution path. ConfigKit applies sensitivity from
both results; reports may still contain ordinary application data. See the
[consumer comparison guide](../consumers/pkg-promptkit.md#inspect-the-captured-configuration)
and ConfigKit's [v0.8.0 comparison contract](https://gitea.maximumdirect.net/eric/ConfigKit/src/tag/v0.8.0/docs/config.md#effective-configuration-diffs).
After resource materialization and effective-profile snapshot construction, the
adapter captures lexically ordered identity inventories for backends, profiles,
and prompts, per-kind resource namespace sets, and metadata for effective
collection declarations. Listing methods return caller-owned copies from those
captured values without consulting ConfigKit or copying resource definitions.
The root `DirectoryInputs` and `FilesystemInputs` helpers discover sorted YAML
files under an explicit root, provide resource context, and leave imports
disabled. Each returned slice belongs to one strict-composition group; file
order does not confer priority. The engine prepends selected catalog and
Promptkit-owned definitions, then caller underlays and optional main. The
order does not confer priority. Caller underlays are composed in their given
order, followed by the optional main input. The
[configuration tests](../../internal/configuration/load_test.go) exercise the
real loader and final-value handoff.
The public `CollectionInput` helper mounts one caller-selected source at
`collections/<namespace>` and enables ConfigKit imports for that input. The
helper does not read the source; ConfigKit retains responsibility for loading,
composition, imports, and source-local resource context. Imported or inherited
`content_file` values continue resolving relative to the source that contributed
the effective field, not to a flattened runtime identity. Explicit
`InspectConfiguration` views expose the retained effective configuration and
provenance for operator review; runtime selection and inventories do not derive
from source history.
## Prompt Definitions
The configuration adapter indexes each prompt by exact ID and version,
The configuration adapter indexes each prompt by exact ID,
materializes file-backed message content using the exact effective field's
origin and resource context, and checks static message/session template syntax
at construction. It stores captured content, compiled templates, declared
input metadata, and a definition hash. It closes content-file handles before
input metadata, and a definition hash. Alongside the unchanged hashed
definition, it retains content-type and description presence for each declared
input and provides an internal requirement snapshot with those declarations
and the captured parses. Metadata values come from the final normalized
declaration; absent and null fields remain distinguishable from explicit
strings, including empty strings. It closes content-file handles before
returning and retains no source capability for public inspection.
Role and cache-control normalization is applied to execution-owned message
copies. The retained ConfigKit configuration keeps effective input message
metadata and its resource origins: recording a sequence-wide normalization in
ConfigKit v0.5.0 would replace origins of unchanged content-file fields.
Prompt materialization shares a lazy metadata snapshot across root and collection
resources during construction, avoiding a full metadata copy per collection.
The runner selects from this index for preparation and `InspectPrompt`.
Neither path rescans configuration or reopens message files. `InspectPrompt`
Role and cache-control normalization is applied to execution-owned message
copies. ConfigKit v0.8.0 retains effective input message metadata and its
resource origins; recording a sequence-wide normalization in the retained
configuration would replace origins of unchanged content-file fields.
The runner selects full definitions from this index for preparation and uses
captured prompt metadata for `InspectPrompt` and `InspectRequirements`; the
internal requirement snapshot lookup uses the selected prompt ID alone.
These paths do not rescan configuration or reopen message files. `InspectPrompt`
returns declared metadata without rendering or resolving profile, input, or
schema resources. Runtime rendering still checks request variables and input
helpers. The [prompt adapter tests](../../internal/configuration/prompts_test.go)
schema resources, and its public shape and prompt hashes do not include the
captured presence state. Runtime rendering still checks request variables and
input helpers. The [prompt adapter tests](../../internal/configuration/prompts_test.go)
cover resource origins, context containment, capture, and selection.
## Profiles And Built-Ins
The configuration adapter shares ID-only selection across its internal prompt
lookups. The execution snapshot lookup returns an owned prompt
definition and cached hash; metadata inspection returns owned input metadata
without copying execution templates. Profile selection likewise has an
identity-only lookup for consumers that do not need the execution target,
while the full profile lookup continues to return an owned target and deep copy
of provider extras. These lookups use captured indexes and do not query
ConfigKit again.
`internal/catalog` selects the released keyed OpenRouter and Rakestrawhome
assets as one ConfigKit underlay and PromptKit-owned profiles as the next.
It discovers YAML inputs and supplies passive filesystem context without
parsing or pre-validating catalog definitions. The configuration loader
validates only the final effective backend and profile values, including
consumer overlays of maintained bases or backends. There is no catalog-only
preflight, reserved source namespace, or replay of backend registration.
The `internal/prompt` requirement walker uses the captured parse trees and
declaration metadata without executing templates or reading artifacts. It
collects literal `input` names, direct root caller keys, and literal-key
`index` lookups. For `if` and `with`, it classifies branches when their
conditions are simple supported values: direct values, literals, negation, or
two-string equality/inequality. It uses Go template truthiness, so a supplied
nonempty string such as `false` is true. Missing decision values and
content-dependent conditions leave dependency-bearing branches conditional
and report incomplete discovery; unresolved conditions with no branch
dependencies do not by themselves make discovery incomplete. Recognizable
references on known skipped paths remain optional inventory entries with an
inactive-use explanation, unless another applicable use establishes a
stronger classification. Skipped expressions are scanned only for
syntactically recognizable names and are not evaluated. A direct caller value
or literal-key lookup can also supply an input name, including through simple
parentheses or a direct pipeline to `input`; names are used exactly as
supplied. Unresolved names and unsupported dependency-bearing expressions
produce structured limitations while retaining recognizable references.
Where an unresolved name or scope could affect existing optional entries, those
entries are reported as conditional rather than inactive-only or definitively
optional. Direct root-map `with` forms retain caller-map identity. Known scalar
values remain separate from unknown scopes; accesses through untracked local
maps report incomplete discovery rather than silently losing caller keys.
Unsupported name expressions reuse value analysis's conservative syntax scan
instead of recursively rescanning the same subtree.
Invoked named templates are scanned conservatively, and
dependency-bearing range scopes make discovery incomplete. Uncalled definitions
and bypassed session templates are not traversed. This internal walker does not
change rendering behavior.
ConfigKit resolves profile inheritance before the handoff. The configuration
adapter validates effective references and field values, then builds target
## Profiles
ConfigKit resolves profile inheritance before the handoff. Within a collection,
`extends` names a sibling local key; it is not a cross-collection reference.
The configuration adapter validates final effective resource references and
field values, then builds target
snapshots with backend defaults and presence/suppression state. A profile can
remain partial until runtime overrides complete it. `InspectProfile` selects
that captured target without prompt, artifact, schema, credential lookup, or
provider work. Preparation applies runtime overrides and then checks target
completeness. The [target tests](../../internal/configuration/targets_test.go)
cover effective target interpretation, while the [catalog tests](../../internal/catalog/catalog_test.go)
cover selected released assets.
completeness. Identity-only profile selection can identify a partial captured
profile without copying its target or requiring readiness. The
[target tests](../../internal/configuration/targets_test.go) cover effective
target interpretation.
The internal engine delegates listing through a narrow identity-inventory
interface directly to the configuration snapshot. The execution runner has no
listing responsibility.
## Ordinary Artifacts
@@ -79,10 +167,15 @@ and after opening the file. It copies content into an artifact, records
metadata and an opaque content-equality value, and applies a content-type
fallback.
Regular files are read synchronously in bounded chunks. Cancellation is
checked before opening, before and after every read, and before publishing the
artifact, so a canceled read never publishes partial content. The ordinary
reader does not detach file reads into background goroutines.
Regular files are read synchronously in bounded chunks. The opened file's size
is only an initial-capacity hint, capped at 1 MiB; reads continue to EOF, and
the returned body determines artifact size and hash. If an overreported hint
leaves unused capacity greater than the actual body length, the completed body
is copied to release the excess backing storage, including when the body is
empty. Cancellation is checked before opening, before and after every read,
and before publishing the artifact, so a
canceled read never publishes partial content. The ordinary reader does not
detach file reads into background goroutines.
This ordinary reader does not implement an inbound HTTP security boundary. In
particular, it does not constrain files to an application root or impose an
@@ -106,7 +199,9 @@ between artifact conversion chunks, around each message, and before publishing
the complete prompt. It cannot interrupt template work already in progress and
never publishes a partial prompt after observing cancellation. It validates and
canonicalizes message roles before carrying roles, session IDs, and cache
control into the rendered prompt. The
control into the rendered prompt. Message and session templates execute into
per-call string builders, so their rendered strings do not require a final
buffer-to-string copy. The
[renderer tests](../../internal/prompt/renderer_test.go) own rendering behavior.
## Schemas And Output Validation
@@ -114,7 +209,8 @@ control into the rendered prompt. The
`internal/validate` provides validators backed by an operating-system
filesystem or an `fs.FS`. Invalid generated content is returned as a validation
result; inability to load, register, or compile a schema is an operational
error.
error. Basic validation checks body emptiness on the byte slice directly
instead of first making a response-sized string copy.
Every preparation operation creates one operation-local validation plan. None,
basic, and JSON modes retain the effective output contract without source

View File

@@ -6,59 +6,37 @@ Material: policy documents under `docs/policy/` and the Policy Reading Guide in
`docs/development.md`, adapted for this repository; root `AGENTS.md`, copied
without modification from upstream `agents/AGENTS.md`.
The release policy uses upstream commit
`e833faddf120c789a9b45348edaf75720de16348`; the remaining material uses
`d84a38ebab26f048cafbe59a2246700c35d29843`.
Copyright (C) 2026 Eric Rakestraw
SPDX-License-Identifier: CC-BY-SA-4.0
License and notices: [CC BY-SA 4.0](licenses/CC-BY-SA-4.0.txt) |
[COPYRIGHT](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/COPYRIGHT) |
[LICENSE](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/LICENSE).
[LICENSE](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/LICENSE) |
[COPYRIGHT](https://gitea.maximumdirect.net/eric/policy/src/commit/e833faddf120c789a9b45348edaf75720de16348/COPYRIGHT) |
[LICENSE](https://gitea.maximumdirect.net/eric/policy/src/commit/e833faddf120c789a9b45348edaf75720de16348/LICENSE).
[Source](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843).
[Source](https://gitea.maximumdirect.net/eric/policy/src/commit/e833faddf120c789a9b45348edaf75720de16348).
## ConfigKit
Selected module: `gitea.maximumdirect.net/eric/ConfigKit` at `v0.5.0`.
Selected module: `gitea.maximumdirect.net/eric/ConfigKit` at `v0.8.0`.
Copyright (C) 2026 Eric Rakestraw
SPDX-License-Identifier: LGPL-3.0-or-later
License and notices: [LGPL 3.0](../../LICENSE) |
[COPYRIGHT](https://gitea.maximumdirect.net/eric/ConfigKit/src/commit/dc09959cca4e105f49ec316c6d4ba585bd2340bd/COPYRIGHT) |
[LICENSE](https://gitea.maximumdirect.net/eric/ConfigKit/src/commit/dc09959cca4e105f49ec316c6d4ba585bd2340bd/LICENSE).
[COPYRIGHT](https://gitea.maximumdirect.net/eric/ConfigKit/src/commit/137769f3f18a370c68444242fb5d88e620e307e5/COPYRIGHT) |
[LICENSE](https://gitea.maximumdirect.net/eric/ConfigKit/src/commit/137769f3f18a370c68444242fb5d88e620e307e5/LICENSE).
[Source](https://gitea.maximumdirect.net/eric/ConfigKit/src/commit/dc09959cca4e105f49ec316c6d4ba585bd2340bd).
## Promptkit OpenRouter Catalog v3
Selected module: `gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v3`
at `v3.20260930.0`.
Copyright (C) 2026 Eric Rakestraw
SPDX-License-Identifier: MIT
License and notices: [MIT](licenses/MIT.txt) |
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit-backend-openrouter/src/commit/f17d208cb7ad1fb043cda2266cd3011d11b40aa0/COPYRIGHT) |
[LICENSE](https://gitea.maximumdirect.net/eric/promptkit-backend-openrouter/src/commit/f17d208cb7ad1fb043cda2266cd3011d11b40aa0/LICENSE).
[Source](https://gitea.maximumdirect.net/eric/promptkit-backend-openrouter/src/commit/f17d208cb7ad1fb043cda2266cd3011d11b40aa0).
## Promptkit Rakestrawhome Catalog v3
Selected module: `gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v3`
at `v3.20260930.0`.
Copyright (C) 2026 Eric Rakestraw
SPDX-License-Identifier: MIT
License and notices: [MIT](licenses/MIT.txt) |
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/src/commit/993443d88c451901f4c917fa00c204fd3b54d28d/COPYRIGHT) |
[LICENSE](https://gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/src/commit/993443d88c451901f4c917fa00c204fd3b54d28d/LICENSE).
[Source](https://gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/src/commit/993443d88c451901f4c917fa00c204fd3b54d28d).
[Source](https://gitea.maximumdirect.net/eric/ConfigKit/src/commit/137769f3f18a370c68444242fb5d88e620e307e5).
## jsonschema

View File

@@ -51,8 +51,8 @@ The [internal component overview](../internal/overview.md) owns the inventory
of implemented packages and their concrete responsibilities. This policy owns
the boundaries and dependency rules those components must preserve.
The internal engine composes selected catalog and caller ConfigKit inputs once,
then assembles the captured definitions, capacity manager, renderer, validator,
The internal engine composes caller-selected ConfigKit inputs once, then
assembles the captured definitions, capacity manager, renderer, validator,
outbound client, and use-case runner. The configuration adapter interprets and
validates only the final effective values, including presence, null, and
resource context. It retains source history for explicit operator inspection,
@@ -97,7 +97,8 @@ on consumers or on Scriptorium.
`internal/domain` owns source-neutral invariants for values shared across
multiple input and execution boundaries, including execution-setting bounds,
OpenAI-compatible base endpoints, session identifiers, and output-contract
legality. Callers retain source parsing, required-field rules, other
legality, plus shared resource-identity syntax and qualification. Callers retain
source parsing, required-field rules, other
source-specific normalization, defaulting, error classification, and policy
specific to their own boundary.

View File

@@ -88,7 +88,7 @@ secret values.
| Release-note history | `docs/releases/` | One checked-in summary per source release under the procedure; the note at its immutable tag is the release record. | Current commands, behavior, contracts, and compatibility definitions. |
| Complete copyable artifacts | `examples/` | Maintained configuration, inputs, and other copyable or runnable files. | Field-by-field reference, command reference, prose explanation. |
| Exact exported Go API | Go declarations and GoDoc, as APIs are implemented | Exported names, signatures, types, values, errors, and exact behavioral contracts. | Task-oriented consumer walkthroughs, implementation details, and future API proposals. |
| Framework file formats | `docs/formats.md` | Prompt-definition and profile YAML fields, schema references, defaults, validation modes, built-in profiles, credentials, and file-to-request precedence. | Exported Go declarations, outbound wire behavior, internal parsing mechanics, and application configuration. |
| Framework file formats | `docs/formats.md` | Prompt-definition and profile YAML fields, schema references, defaults, validation modes, credentials, and file-to-request precedence. | Exported Go declarations, outbound wire behavior, internal parsing mechanics, and application configuration. |
Create supporting documents for existing interfaces and responsibilities when
substantive content is available.

View File

@@ -1,10 +1,10 @@
# Release Policy
**Base Policy Version:** 1.4
**Base Policy Version:** 1.6
**Last Updated**: 2026-09-27
**Last Updated**: 2026-10-08
**Upstream Source:** [release.md](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/docs/policy/release.md)
**Upstream Source:** [release.md](https://gitea.maximumdirect.net/eric/policy/src/commit/e833faddf120c789a9b45348edaf75720de16348/docs/policy/release.md)
## Purpose
@@ -12,6 +12,9 @@ This policy defines a proportionate workflow for validating and publishing an
intended source revision under a fresh version. Releases should require little
routine intervention, avoid repeated work, and preserve published contents.
Adopt stronger verification or reproducibility requirements through an explicit
project-specific decision justified by the intended audience and support needs.
The repository-specific commands belong in `docs/release.md`. Historical release
summaries belong in `docs/releases/`. The procedure must agree with the adopted
policy and link to it rather than introduce additional release requirements.
@@ -59,7 +62,8 @@ requires a deliberate update to the policy and procedure where needed.
For releases at or after `v1.0.0`:
- a major release may make intentional incompatible changes;
- a minor release may add backward-compatible behavior; and
- a minor release may add backward-compatible behavior or raise the Go consumer
minimum as described below; and
- a patch release may make backward-compatible fixes.
Before `v1.0.0`, a minor release may change a documented public, operational,
@@ -71,14 +75,44 @@ Additional compatibility promises and deliberate exceptions belong in the
adopted policy or canonical contract documentation. Release notes explain
changes; they do not redefine current compatibility contracts.
## Release Documentation
## Consumer Minimum And Release Toolchain
Distinguish the minimum toolchain supported for consumers from the toolchain
maintainers use to build and validate a release. Record the consumer minimum in
the language's standard metadata and explain its support meaning in canonical
documentation. Define the selected release toolchain and its setup commands in
`docs/release.md`. Toolchain selection and upgrade cadence are repository-specific
choices.
For Go repositories, normally declare the initial stable release of the selected
series as the consumer minimum, such as `go 1.26.0`. Use a higher patch minimum
when a dependency or an identified correctness fix requires it, and document
the reason.
For release-policy purposes, successful build and release validation on the
selected Go patch release are sufficient to presume support across that Go
series, subject to the declared consumer minimum and concrete evidence of
incompatibility. Investigate specific compatibility concerns as they arise.
Use the selected release toolchain for candidate validation and release builds,
including CI jobs performing those tasks. A newer validation patch version alone
leaves the consumer minimum unchanged.
A minor release may raise the Go consumer minimum, including from one Go series
to the next, both before and after `v1.0.0`. This is an explicit exception to the
backward-compatibility requirement for minor releases. Patch releases preserve
the consumer minimum. Explain an increase's impact and required consumer action
in the release note. Other incompatible changes remain subject to the versioning
rules above.
## Release Documentation And Preparation
Every new release must include `docs/releases/<version>.md` in the tagged
source tree. Use the complete version tag as the filename and begin with a
title containing the product name and that exact version.
Include a short summary and any relevant compatibility impact or upgrade
instructions. Use the Promptkit section structure below and keep the note
instructions. Choose sections suited to the release and keep the note
proportionate to the change.
Keep exact current contracts in their canonical documents and update affected
@@ -186,7 +220,8 @@ release without replacing its contents or reusing its version.
Maintain `docs/release.md` as a short, executable maintainer runbook. It must
identify:
1. prerequisites, primary branch, and authoritative remote;
1. prerequisites, selected release toolchain, primary branch, and authoritative
remote;
2. version and release-note preparation;
3. one publication command with its selected local checks and automated guards;
4. declared deliverables, required build targets, and automation responsibilities;
@@ -210,43 +245,36 @@ annotated stable SemVer tags. It has no binaries, hosted download pages, or
asynchronous deliverables. The source commit is pushed through main before the
tag. The [release procedure](../release.md) owns commands and recovery.
The existing publication script requires Summary, Compatibility, Upgrade, and
Changes headings beneath a title of `# Promptkit vMAJOR.MINOR.PATCH`. State
“No action required” or “None” where appropriate. This retained local structure
matches the automated release-note guard; changing it is separate work.
Release preparation uses the latest Go installation available in the development
environment to build the maintainer release executable and validate the
candidate, including the consumer examples. Keep that same local toolchain
active throughout those operations.
The [publication script](../../scripts/release.sh) runs the
[local checks](../../scripts/release-check.sh) once for the final committed
candidate: race-enabled tests, vet, three offline consumer examples, tracked
Go formatting, Markdown links, module/workspace hygiene, whitespace, and a
credential-pattern scan. These protect concurrency, assembled library use,
source distribution, and documentation integrity. A separate ordinary test run
or standalone build is not an additional release gate. Candidate, tag, and
synchronized-main checks remain automated. No CI or module-availability wait
is required. Licensing footer URLs are preparation work, not a network gate.
For a minor release, advance Promptkit's consumer minimum to the initial stable
release of that Go series, retaining a higher patch minimum when required by a
dependency or an identified correctness fix. Patch releases preserve the
existing consumer minimum. Commit any module update before validating the final
candidate. The [release procedure](../release.md#prepare-the-candidate) owns
setup commands, and the consumer guide explains
[Go compatibility](../consumers/pkg-promptkit.md#go-compatibility).
## Catalog Profile Compatibility
Use a title containing `Promptkit` and the exact version, followed by a short
summary and any relevant compatibility impact or upgrade instructions. The
note is committed with the candidate and supplies the annotated tag message.
No fixed set of section headings is required.
External catalog profile data may be added, removed, renamed, or modified
without deprecation or advance notice. This exception is limited to profile
data imported from the maintained catalog modules. It does not weaken ordinary
compatibility for Promptkit's public Go API, configuration and format semantics,
runtime behavior, catalog-module APIs, manifest schema, backend definitions, or
consumer-owned definitions.
Within the catalog versions selected and tested by a Promptkit release, and
unless a consumer overrides the name or its base, Promptkit keeps the literal
profile IDs `promptkit-openrouter-standard` and
`promptkit-openrouter-auto` available. The names do not promise a permanent
model, backend, routing behavior, settings, cost, output, or upstream
availability. The [framework format reference](../formats.md#built-in-profile-catalog)
owns their current definitions.
`v0.10.0` is the initial minor release for this compatibility transition.
Afterward, catalog-only updates and adjustments to the current selections may
ship in a patch release when all other compatibility obligations remain intact.
External profile removals can require consumer action; they are not automatic
migrations.
Catalog adoption and release validation remain local and offline. They do not
probe provider endpoints, request model lists, or make inference calls.
The [publication command](../../scripts/release.sh) delegates to the released
ReleaseKit tool selected in `go.mod`. The committed
[profile](../../releasekit.json) selects the race-enabled Go suite, full vet,
committed Go formatting, local Markdown links, workspace and filesystem
replacement inspection, and the offline semantic checks for the three
maintained consumer examples. These are the selected checks for the final
candidate; a separate ordinary test run or standalone build is not an
additional release gate. The upstream tool owns candidate, remote, and tag
publication checks. It returns native status 0 for confirmed publication, 1
for a known validation or publication failure, 2 for invalid input or profile,
and 3 when the push outcome is unknown. For status 3, resolve the exact
captured publication endpoint before retrying; the repository-specific
[procedure](../release.md#failure-recovery) links to the upstream recovery
instructions. Do not wait for CI or Go module availability. Licensing footer
URLs are preparation work, not a network gate.

View File

@@ -58,17 +58,24 @@ and repository hygiene.
Introducing hosted CI later would supplement, not silently redefine, this
documented validation model.
Full release-candidate validation includes race-enabled package tests, explicit
static analysis, and execution with automated semantic checks of the three maintained
offline consumer examples. The test run builds the packages; the examples
also exercise ordinary compilation. Separate ordinary tests and a standalone
build are not additional release gates. Ordinary tests remain useful during
development. The preparation example protects assembled preparation and
The selected ReleaseKit profile runs the race-enabled package suite and full
vet, checks committed Go formatting, local Markdown links, and module
distribution hygiene, and executes semantic assertions for the three
maintained offline consumer examples. ReleaseKit owns this shared validation
machinery and publication workflow. The test run builds the packages; the
examples also exercise ordinary compilation. Separate ordinary tests and a
standalone build are not additional release gates. Ordinary tests remain useful
during development. The preparation example protects assembled preparation and
inspection behavior. The execution example separately protects assembled
`Run`, injected-client, validation, usage, and result behavior. The observation
example protects assembled built-in retries, structured proposal and assessment
runs, and application-owned interpretation of metadata.
The standalone `python3 scripts/test-release.py` command exercises Promptkit's
ReleaseKit wiring against disposable local repositories. It is focused
development validation when release tooling changes, not part of the default
Go test suite or a recurring release check.
Tests in the default suite must be deterministic, offline, and independent of
real credentials. They must not invoke paid APIs, use live network
dependencies, or depend on mutable external services. Tests that require live

View File

@@ -1,94 +1,130 @@
# Release Procedure
This runbook implements the [release policy](policy/release.md). A successful
push of the selected annotated tag to origin completes the release. Never wait
for CI to finish or succeed.
This runbook implements the [release policy](policy/release.md). Promptkit
publishes source-only Go module tags from `main` to `origin`. A successful push
of the selected annotated tag, after its source commit is available on `main`,
completes the release. Do not wait for CI or Go module availability.
## Prepare The Candidate
Required tools are Git, Go with race-detector support, POSIX shell, and Python
3. Publication requires push access to origin. Work from the repository root.
3. Work from the repository root with push access to `origin`.
Select an unused version and create `docs/releases/<version>.md` using the
[required structure](policy/release.md#release-documentation). Describe
compatibility impact and consumer actions and update affected canonical docs.
For catalog updates, record selected catalog versions and any consumer action,
linking to the catalog policy rather than repeating its contracts.
Use the latest Go installation available locally for the release tool, the
candidate validation, and the maintained examples. Make it the active `go`
command and record its version:
Follow the [OSS policy](policy/oss.md) when preparing licensing and attribution
changes. Each new note must end with the following navigation, replacing
`vMAJOR.MINOR.PATCH` with the selected release tag:
```markdown
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit/src/tag/vMAJOR.MINOR.PATCH/COPYRIGHT) | [LICENSE](https://gitea.maximumdirect.net/eric/promptkit/src/tag/vMAJOR.MINOR.PATCH/LICENSE) | [ATTRIBUTION](https://gitea.maximumdirect.net/eric/promptkit/src/tag/vMAJOR.MINOR.PATCH/docs/oss/attribution.md)
```sh
export GOTOOLCHAIN=local
go version
```
Check these file paths against the candidate; a future tag need not resolve
remotely during preparation. The annotated tag derives from this same note,
including its footer. Promptkit publishes no separate hosted release body or
download page. If such a channel is deliberately introduced, reuse the note
and preserve its footer. No extra network checks, artifacts, or approval gates
are introduced, and published notes are not retrofitted.
For a minor release, advance Promptkit's consumer minimum to the initial stable
release of that Go series before committing the candidate:
Retire completed roadmaps, preserving planned or deferred work and repairing
links. This is housekeeping, not a separate approval gate. Review changes
through the ordinary development workflow, commit them, and push main to
origin. No repeated release-specific diff review is required.
```sh
release_go_version=$(go env GOVERSION)
release_go_series=${release_go_version#go}
release_go_series=${release_go_series%.*}
go mod edit "-go=${release_go_series}.0"
```
For example, Go 1.26.8 normally results in `go 1.26.0`. Keep a higher patch
minimum if a dependency or identified correctness fix requires it. For a patch
release, leave the consumer minimum unchanged; using a newer validation
toolchain does not itself raise that minimum. If the candidate needs a higher
minimum, prepare a minor release instead. See the
[consumer-minimum policy](policy/release.md#consumer-minimum-and-release-toolchain).
Prepare module caches before validation; this may update `go.sum` and belongs
before the candidate commit:
```sh
GOTOOLCHAIN=local go mod download
```
Choose an unused stable version and create `docs/releases/<version>.md` with a
title containing `Promptkit` and that exact version, a short summary, and any
relevant compatibility impact or upgrade instructions. No fixed section
headings are required. Describe a raised Go minimum and its compatibility
impact under the [versioning policy](policy/release.md#versioning-and-compatibility).
End each new note with the licensing footer below, using the selected version:
```markdown
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit/src/tag/vX.Y.Z/COPYRIGHT) | [LICENSE](https://gitea.maximumdirect.net/eric/promptkit/src/tag/vX.Y.Z/LICENSE) | [ATTRIBUTION](https://gitea.maximumdirect.net/eric/promptkit/src/tag/vX.Y.Z/docs/oss/attribution.md)
```
Check the footer paths against the candidate; the future tag need not resolve
remotely during preparation. Follow the [OSS policy](policy/oss.md) for these
links. Promptkit publishes no binaries, hosted download page, or other
asynchronous deliverables.
Retire completed roadmaps while preserving deferred work and repairing incoming
links. Review the intended changes through the ordinary development workflow,
commit the candidate on `main`, and push `main` to `origin` before publication.
Preparation may update module metadata; the release bootstrap and validation
preserve the committed candidate.
## Validate And Publish
Run the following with the chosen unused version (replace the placeholder):
For an optional local diagnostic of a committed candidate, run:
```sh
./scripts/releasekit.sh check vMAJOR.MINOR.PATCH
```
This is not a required step before publication. The upstream
[candidate-check contract](https://gitea.maximumdirect.net/eric/ReleaseKit/src/tag/v0.4.1/docs/consumers/check.md)
describes its requirements.
Run the single publication command with the chosen version:
```sh
./scripts/release.sh vMAJOR.MINOR.PATCH
```
The [publication script](../scripts/release.sh) checks a clean, synchronized
main candidate, the version and unused tag, and the release note's title and
required headings. It runs the [local checks](../scripts/release-check.sh)
once, rechecks candidate identity and publication state, generates the
annotation from the committed note plus the validated commit, and pushes only
the selected annotated tag.
The familiar wrapper delegates to the native ReleaseKit executable selected in
`go.mod`. The committed [profile](../releasekit.json) selects the race-enabled
Go suite, full vet, committed Go formatting, local Markdown links, workspace
and filesystem replacement inspection, and semantic checks for the three
maintained offline consumer examples. ReleaseKit also verifies the committed
candidate and publication state, then publishes only the selected tag. Its
checks run once for this candidate; do not repeat them as a separate sign-off.
There is no separate tag-message authoring, example-output inspection,
metadata inspection, or human sign-off. Compatibility and release-note content
remain ordinary editorial responsibilities. Development validation is useful
but does not require a second full run before invoking the publication script.
The script stops successfully after the tag push. CI completion, CI success,
remote read-back, and Go module availability are not release completion gates.
Do not wait or poll for them. Provider probes and inference calls are never
part of release validation.
## Catalog Profile Compatibility
The [release policy](policy/release.md#catalog-profile-compatibility) owns the
catalog compatibility boundary. This heading preserves older incoming links.
The native command returns status 0 for confirmed publication, 1 for a known
validation or publication failure, 2 for invalid input or profile, and 3 when
the push outcome remains unknown. Successful publication needs no remote
read-back, CI wait, or installation check. Provider probes and inference calls
are not part of validation.
## Failure Recovery
If local checks fail, fix the candidate, commit and push changes, then rerun
the publication command. It never automatically removes a tag.
For status 1, use the diagnostic to distinguish a validation failure from a
known publication conflict. Correct and commit candidate defects, push `main`,
and rerun publication with an unused version when appropriate. Status 2 means
the invocation or profile must be corrected.
If a push fails or its result is ambiguous, set `RELEASE_VERSION` to the
attempted version and inspect origin before retrying:
For status 3, do not retry or remove the local tag until the remote outcome is
resolved. ReleaseKit reports the attempted version, tag object, and recovery
state, including the captured publication endpoint when available. Use that
endpoint—not merely the current `origin` URL—to query the exact tag; follow the
upstream
[failure-recovery procedure](https://gitea.maximumdirect.net/eric/ReleaseKit/src/tag/v0.4.1/docs/release.md#resolve-failures):
```sh
git ls-remote --tags origin "refs/tags/$RELEASE_VERSION" "refs/tags/$RELEASE_VERSION^{}"
git rev-parse "refs/tags/$RELEASE_VERSION" "refs/tags/$RELEASE_VERSION^{}"
releasekit_endpoint='COPY_CAPTURED_ENDPOINT'
releasekit_version='vX.Y.Z'
test "$(git ls-remote --get-url -- "$releasekit_endpoint")" = "$releasekit_endpoint" &&
git ls-remote --refs -- "$releasekit_endpoint" "refs/tags/$releasekit_version"
```
Matching remote tag and peeled commit values confirm publication, even if the
push command reported an error. If the remote query fails, publication status
is still unknown. Do not delete or recreate the local tag until absence on
origin is confirmed. Once confirmed unpublished, an unsuccessful attempt's
local tag can be removed and the publication command rerun. Never overwrite
a published tag.
CI or module-resolution problems may be investigated separately; temporary
service, network, or authentication failures do not require another release.
Only an actual defect requiring changes to published contents warrants a new
appropriate version. See the [recovery policy](policy/release.md#immutability-and-failure-recovery).
Compare the returned object ID with the reported tag object. An exact match
confirms publication. A missing ref confirms absence only when the endpoint
identity check and query both succeed. If either check fails, the outcome
remains unknown; correct the transport configuration and resolve it before
retrying. Only after authoritative absence is confirmed may an unpublished
local tag be removed before correcting the candidate or retrying. Never move,
overwrite, or delete a published tag.
## Policy Changes

View File

@@ -15,7 +15,7 @@ the existing Go API. This refactor requires no consumer code changes.
This release selects OpenRouter `v2.20260922.1` and Rakestrawhome
`v2.20260922.0`; their exact inventories are linked from the
[format reference](../formats.md#external-profile-inventories).
[migration identity mapping](../consumers/configuration-migration.md#identity-mapping).
The maintained names provide continuity only within the catalog versions
selected and tested by a Promptkit release and when consumers do not override
@@ -37,8 +37,8 @@ an available profile or supply their own complete definition; the new
`deepseek-flash-latest` and `deepseek-pro-latest` profiles are not automatic
aliases and need not preserve the previous settings.
See the [release policy](../release.md#catalog-profile-compatibility) for the
compatibility boundary, the [format reference](../formats.md#built-in-profile-catalog)
for current Promptkit-owned definitions and selected external inventories, and
See the [release policy](../policy/release.md#versioning-and-compatibility) for the
compatibility boundary, the [migration identity mapping](../consumers/configuration-migration.md#identity-mapping)
for current resource mappings, and
the [consumer guide](../consumers/pkg-promptkit.md#choose-a-profile-ownership-model)
for configuration choices.

View File

@@ -72,5 +72,5 @@ for a structured proposal-and-assessment workflow.
Backend catalog selections are unchanged from v0.10.0: OpenRouter
`v2.20260922.1` and Rakestrawhome `v2.20260922.0`. See the
[catalog compatibility policy](../policy/release.md#catalog-profile-compatibility)
and [profile inventories](../formats.md#external-profile-inventories).
[release compatibility policy](../policy/release.md#versioning-and-compatibility)
and [migration identity mapping](../consumers/configuration-migration.md#identity-mapping).

View File

@@ -15,7 +15,7 @@ No public API, runtime behavior, or profile changes are introduced.
Promptkit remains LGPL-3.0-or-later; the catalogs are now MIT-licensed.
See the [project grant](../../COPYRIGHT),
[component attribution](../oss/attribution.md), and
[catalog compatibility policy](../policy/release.md#catalog-profile-compatibility).
[release compatibility policy](../policy/release.md#versioning-and-compatibility).
## Upgrade

View File

@@ -35,7 +35,7 @@ for API and source-selection changes.
Maintained OpenRouter and Rakestrawhome catalogs are selected as `/v3` modules,
both at `v3.20260930.0`. Their asset format changed; consumers that read catalog
assets directly must adopt the keyed format. The maintained Promptkit profile
IDs remain available under the [catalog compatibility policy](../policy/release.md#catalog-profile-compatibility),
IDs remain available under the [release compatibility policy](../policy/release.md#versioning-and-compatibility),
which does not promise fixed profile data.
## Upgrade

69
docs/releases/v0.13.0.md Normal file
View File

@@ -0,0 +1,69 @@
# Promptkit v0.13.0
This release adds prompt requirement discovery and profile/prompt listing,
reduces avoidable configuration and execution allocations, and standardizes
maintainer publication through ReleaseKit. The minimum supported Go version
increases to 1.26.0.
## Discover Requirements And Available Definitions
- `Engine.InspectRequirements` reports recognizable input and caller-variable
dependencies for a selected prompt and supplied caller context before input
artifacts exist. It classifies dependencies as required, conditional, or
optional and reports structured limitations when discovery is incomplete.
Discovery coverage is not proof that requirements are satisfied or execution
will succeed. See the [requirement inspection guide](../consumers/pkg-promptkit.md#inspect-requirements-before-preparation)
and [GoDoc](../../requirements.go).
- `Engine.ListProfiles` returns effective profile IDs, and `Engine.ListPrompts`
returns exact ID/version pairs as `PromptIdentity` values. Lists come from the
engine's captured configuration, are lexically ordered and caller-owned, and
include consumer definitions and maintained profiles that remain effective.
Listing performs no provider or execution work. See the
[discovery guide](../consumers/pkg-promptkit.md#discover-available-resources-and-collections)
and [GoDoc](../../listing.go).
## Efficiency And Tooling
- Reduce response-sized copies in basic validation and rendered-message output,
and encode valid provider extra parameters only as part of the complete
request rather than serializing them twice.
- Use bounded initial file-size reservations without limiting accepted content,
releasing excess backing storage for small or empty completed artifacts.
- Reuse owned preparation values when creating prepared-execution snapshots,
while preserving independent inspection details and caller-owned data.
- Use lightweight captured profile identities and prompt metadata for inspection
instead of copying execution targets and definitions unnecessarily.
- Upgrade ConfigKit to v0.6.0 for more efficient loading, resolution, metadata,
and inspection processing. Configuration semantics remain unchanged; these
improvements primarily benefit construction and explicit operator inspection,
not upstream inference latency.
- Adopt module-selected ReleaseKit v0.3.0 for maintainer validation and tag
publication. The [release procedure](../release.md) owns the current commands
and recovery workflow; successful tag publication remains the completion
point, without waiting for CI.
These changes preserve existing execution and ownership contracts. Benefits
depend on workload; this release introduces no fixed performance guarantee.
## Compatibility And Upgrade
Consumers upgrading from v0.12.0 need **Go 1.26.0 or newer**, rather than the
previous 1.25.5 minimum. Update application build environments and any CI images
still using Go 1.25 before upgrading:
```sh
go get gitea.maximumdirect.net/eric/promptkit@v0.13.0
```
No application-code or configuration migration is required from v0.12.0 beyond
the Go toolchain update. The new discovery methods are optional additions;
existing construction, inspection, preparation, and execution workflows remain
available. See [Go compatibility](../consumers/pkg-promptkit.md#go-compatibility)
for the distinction between the consumer minimum and maintainer toolchain.
Maintainers should follow the current release runbook rather than the retired
`scripts/release-check.sh`. The publication entry point remains
`scripts/release.sh`; ReleaseKit now owns its shared validation and publication
mechanics.
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.13.0/COPYRIGHT) | [LICENSE](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.13.0/LICENSE) | [ATTRIBUTION](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.13.0/docs/oss/attribution.md)

48
docs/releases/v0.14.0.md Normal file
View File

@@ -0,0 +1,48 @@
# Promptkit v0.14.0
This release adds comparison of captured configuration inspections and updates
ConfigKit and the maintainer release tool. The minimum supported Go version
remains 1.26.0.
## Compare Captured Configurations
`ConfigurationInspection.Diff` compares the effective configuration captured by
two engines. Consumers can use it to review a candidate configuration, inspect
an overlay's effect, or present configuration changes to operators.
Comparison delegates to ConfigKit and returns its difference records, including
presence information, paired redaction, and protected-change indicators. It
does not reload sources or contact providers. Source-history differences alone
do not produce value differences.
An empty diff is not proof of equivalent execution: referenced file contents,
resolved credentials, later execution preparation, and request-specific
overrides are outside the comparison. Records can contain ordinary application
data; callers retain responsibility for their disclosure.
See the [configuration comparison guide](../consumers/pkg-promptkit.md#inspect-the-captured-configuration)
and [GoDoc](../../configuration_inspection.go) for usage and the exact contract.
## Dependency And Tooling Updates
- Upgrade ConfigKit from v0.6.0 to v0.7.1, providing the upstream comparison API.
v0.7.1 also updates ConfigKit's maintainer tooling without changing its runtime
behavior from v0.7.0.
- Upgrade the module-selected ReleaseKit tool from v0.3.0 to v0.4.1. Promptkit
retains its existing root-module profile, publication entry point, and release
gates. See the [release procedure](../release.md) for maintainer commands.
## Compatibility And Upgrade
No application-code, configuration, or Go-toolchain migration is required from
v0.13.0. Configuration comparison is an optional API addition; existing
construction, inspection, preparation, and execution workflows remain available.
```sh
go get gitea.maximumdirect.net/eric/promptkit@v0.14.0
```
See [Go compatibility](../consumers/pkg-promptkit.md#go-compatibility) for the
distinction between the consumer minimum and maintainer toolchain.
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.14.0/COPYRIGHT) | [LICENSE](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.14.0/LICENSE) | [ATTRIBUTION](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.14.0/docs/oss/attribution.md)

34
docs/releases/v0.15.0.md Normal file
View File

@@ -0,0 +1,34 @@
# Promptkit v0.15.0
This release adds error-only validation of a concrete prompt request without
calling a model. The minimum supported Go version remains 1.26.0.
## Validate A Concrete Request
`Engine.ValidateRequest` accepts an ordinary `RunRequest` and returns nil when
it passes preparation, or the existing preparation error. Consumers can use it
for fail-fast checks, including downstream CLI validation commands, without
receiving a prepared snapshot.
The method shares `Prepare`'s request conversion and preparation path. Unlike
requirements inspection, it checks the actual supplied values and bindings.
Artifact and schema resource I/O still occurs; success does not certify provider
acceptance or later execution success. A subsequent run prepares independently.
Validation is optional, not a required step before preparation or execution.
See the [request validation guide](../consumers/pkg-promptkit.md#validate-a-concrete-request)
and [GoDoc](../../engine.go) for usage and the exact contract. Consumers that
need execution tied to a checked snapshot can use the existing
[prepared execution workflow](../consumers/pkg-promptkit.md#prepare-now-and-execute-the-same-snapshot-later).
## Compatibility And Upgrade
No application-code, configuration, or Go-toolchain migration is required from
v0.14.0. This is an optional API addition; existing inspection, preparation, and
execution workflows remain available.
```sh
go get gitea.maximumdirect.net/eric/promptkit@v0.15.0
```
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.15.0/COPYRIGHT) | [LICENSE](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.15.0/LICENSE) | [ATTRIBUTION](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.15.0/docs/oss/attribution.md)

86
docs/releases/v0.16.0.md Normal file
View File

@@ -0,0 +1,86 @@
# Promptkit v0.16.0
This release adds explicit smoke diagnostics, configuration collections,
namespace-scoped engine views, and expanded resource discovery. It also makes
intentional breaking changes to configuration ownership and prompt selection.
The minimum supported Go version remains 1.26.0.
## Configuration Collections And Discovery
Applications can compose caller-selected collections of backends, profiles,
and prompts. Collection resources have qualified identities such as
`support/answer`; root resources retain unqualified identities. `CollectionInput`
provides a convenience for mounting a selected ConfigKit input into a collection.
Promptkit now uses ConfigKit v0.8.0, including its optional source specification
header support.
`Engine.WithNamespaces` creates an immutable view with separate prompt, profile,
and backend scopes for short caller-supplied IDs. Views share the underlying
configuration, clients, capacity pools, and prepared-execution owner. Explicitly
qualified IDs and configured references keep their exact identities.
`ListProfiles` and `ListPrompts` support namespace filters. New `ListBackends`,
`ListNamespaces`, and `ListCollections` methods expose captured resource and
collection inventories. See the [collection format](../formats.md#source-headers-collections-and-resource-ids),
[scoped-view guide](../consumers/pkg-promptkit.md#use-scoped-engine-views), and
[discovery guide](../consumers/pkg-promptkit.md#discover-available-resources-and-collections).
## Explicit Smoke Diagnostics
`Engine.SmokeTest` provides backend reachability, authentication, text inference,
and structured-output inference diagnostics. Reports provide factual
observations, timing, and available usage rather than an overall readiness
verdict. Optional Boolean facts distinguish negative observations from unknown
or inapplicable checks; matching a JSON schema does not prove that a provider
enforced it.
A concluded diagnostic can report failure with a nil Go error. Invalid requests
return ordinary errors, and caller cancellation can return partial observations
with a recognizable context error. Diagnostics can contact providers and incur
inference costs only when explicitly invoked; they are not automatic execution
or release checks. See the [smoke diagnostic guide](../consumers/pkg-promptkit.md#run-explicit-smoke-diagnostics)
and [API GoDoc](../../smoke_diagnostics.go).
## Breaking Changes And Upgrade
Consumers upgrading from v0.15.0 should follow the
[configuration and API migration guide](../consumers/configuration-migration.md):
- Supply backend and profile definitions explicitly. Promptkit no longer bundles
provider catalogs or the `promptkit-openrouter-standard` and
`promptkit-openrouter-auto` profiles. Conventional backend names do not install
definitions automatically. Applications choose and maintain their sources.
- Convert version-keyed prompt mappings into one definition per prompt ID.
Prompt `version` is now an optional descriptive label, not a selector.
Choose deliberately which former definition to install.
- Remove `PromptVersion` selectors from `RunRequest`,
`RequirementInspectionRequest`, and `SmokeTestRequest`, where applicable.
Change prompt inspection calls to `InspectPrompt(ctx, promptID)`. Update code
consuming version-bearing inspection, identity, result, or observation fields
to handle optional labels, including nil and explicitly empty labels.
- When moving resources into collections, update stored backend, profile, and
diagnostic-default references to their intended qualified IDs. Mounting a
source does not rewrite references; profile inheritance remains within its
sibling mapping. Root configuration `version`, when supplied, is now a source
specification header, distinct from descriptive resource labels.
The migration guide includes identity mappings and before/after prompt examples.
The [maintained offline example](../../examples/go-library/prepare/main.go)
demonstrates explicit composition, source-relative content, scoped preparation,
and inventory without contacting a provider.
After adapting configuration and API calls:
```sh
go get gitea.maximumdirect.net/eric/promptkit@v0.16.0
```
## Correctness And Efficiency
Collection prompt materialization shares one lazy origin-metadata snapshot,
avoiding repeated full metadata copies across collections. Internal prepared
snapshots detach optional prompt labels. Smoke diagnostics preserve caller
cancellation and report actual inference destinations and known-zero usage when
no generation request was dispatched.
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.16.0/COPYRIGHT) | [LICENSE](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.16.0/LICENSE) | [ATTRIBUTION](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.16.0/docs/oss/attribution.md)

View File

@@ -63,8 +63,8 @@ Every engine now includes the reserved `rakestrawhome` backend, identified by
selects that backend. Consumers can use the maintained endpoint, credential,
capacity, and model defaults without registering either definition themselves.
See the [built-in backend and profile catalogs](../formats.md#built-in-backends)
and the [consumer adoption example](../consumers/pkg-promptkit.md#use-the-rakestrawhome-built-in-profile)
See the [backend format reference](../formats.md#backend-definitions)
and the [current identity migration mapping](../consumers/configuration-migration.md#identity-mapping)
for the current contracts.
## Structured Generation Errors
@@ -127,7 +127,7 @@ freezes the fully resolved target. The selected leaf ID remains public while
effective execution settings reflect the resolved chain.
See the [profile inheritance format reference](../formats.md#profile-inheritance),
the [consumer alias example](../consumers/pkg-promptkit.md#alias-a-built-in-profile),
the [current identity migration mapping](../consumers/configuration-migration.md#identity-mapping),
and the [`Profile` GoDoc](../../types.go) for exact merge and validation
behavior.

View File

@@ -96,9 +96,8 @@ precedence rules. Promptkit no longer keeps duplicate embedded profile assets
or hard-coded definitions for those maintained backends.
The module versions in Promptkit's `go.mod` identify the catalog releases
tested with this release. The [built-in backend and profile format
reference](../formats.md#built-in-backends) remains the canonical consumer
contract, while the [internal source documentation](../internal/sources.md#profiles-and-built-ins)
tested with this release. The [backend format reference](../formats.md#backend-definitions)
remains the canonical consumer contract, while the [internal source documentation](../internal/sources.md#profiles)
describes the dependency boundary.
## Public API Changes

View File

@@ -24,9 +24,8 @@ var (
// ErrInvalidRequest identifies a request whose required values, overrides,
// credentials, or effective settings are invalid.
ErrInvalidRequest = errors.New("invalid run request")
// ErrPromptNotFound identifies a requested prompt ID or version that is not
// present in the selected prompt source. It does not also match
// ErrPromptLoad.
// ErrPromptNotFound identifies a requested prompt ID that is not present in
// the captured configuration. It does not also match ErrPromptLoad.
ErrPromptNotFound = errors.New("prompt not found")
// ErrProfileNotFound identifies a selected profile ID that is absent from
// every configured profile source. It does not also match ErrProfileLoad.
@@ -82,16 +81,21 @@ var (
ErrValidation = errors.New("failed to validate output")
)
// Engine inspects prompts and profiles and prepares and runs Promptkit prompt
// requests.
// Engine inspects prompts and profiles, enumerates prompt requirements, and
// prepares and runs Promptkit prompt requests.
//
// An Engine is safe for concurrent calls to [Engine.InspectConfiguration],
// An Engine is safe for concurrent calls to [Engine.WithNamespaces],
// [Engine.InspectConfiguration],
// [Engine.InspectPrompt],
// [Engine.InspectProfile], [Engine.Prepare], [Engine.PrepareExecution],
// [Engine.Run], and [Engine.RunPrepared]. Each Engine owns independent
// backend-capacity pools that coordinate Run and RunPrepared admission and
// model generation. Injected collaborators may still be invoked concurrently
// across different backend pools or for unlimited backends.
// [Engine.InspectRequirements], [Engine.InspectProfile], [Engine.Prepare], [Engine.PrepareExecution],
// [Engine.Run], [Engine.RunPrepared], [Engine.SmokeTest], [Engine.ListProfiles],
// [Engine.ListPrompts], [Engine.ListBackends], [Engine.ListNamespaces], and
// [Engine.ListCollections]. Engines constructed separately by [NewEngine] own
// independent backend-capacity pools that coordinate Run, RunPrepared, and
// inference SmokeTest admission and model generation. Namespace views returned
// by [Engine.WithNamespaces] share their underlying engine's pools, clients,
// and prepared-execution owner. Injected collaborators may still be invoked
// concurrently across different backend pools or for unlimited backends.
type Engine struct {
internal *internalengine.Engine
}
@@ -208,11 +212,11 @@ func WithSchemaFile(path string) Option {
// NewEngine constructs an Engine from configuration and options.
//
// ConfigKit composes maintained catalog inputs, PromptKit-owned definitions,
// caller underlays, and the optional main input in that order. Construction
// validates the final effective backend, profile, and prompt definitions and
// captures referenced prompt content. A missing prompt collection is allowed.
// Schema documents and request artifacts remain runtime resources.
// ConfigKit composes caller-selected underlays in their given order, followed
// by the optional main input. Construction validates the final effective
// backend, profile, and prompt definitions and captures referenced prompt
// content. A missing prompt collection is allowed. Schema documents and
// request artifacts remain runtime resources.
//
// NewEngine returns an error matching ErrInvalidConfig for invalid
// configuration or options, preserving a ConfigKit LoadError when loading
@@ -241,14 +245,13 @@ func NewEngine(cfg Config, opts ...Option) (*Engine, error) {
return &Engine{internal: assembled}, nil
}
// InspectPrompt resolves one explicit prompt definition without selecting a
// profile or starting execution work.
// InspectPrompt resolves one prompt ID without selecting a profile or starting
// execution work.
//
// InspectPrompt requires a nonblank promptID. It passes nonblank promptID and
// promptVersion values unchanged to the engine's ordinary, case-sensitive
// prompt selection. An empty version succeeds only when that ID has one
// version; a nonempty version selects one exact ID/version pair from the
// captured configuration snapshot.
// InspectPrompt requires a nonblank promptID and performs an exact,
// case-sensitive lookup in the captured configuration snapshot. A
// namespace-scoped view qualifies slashless caller IDs before lookup; already
// qualified IDs pass through. Prompt version labels do not select definitions.
//
// A successful result proves that the selected definition and any referenced
// message content files were structurally loaded. Inputs are returned in
@@ -267,21 +270,19 @@ func NewEngine(cfg Config, opts ...Option) (*Engine, error) {
// or content files; construct another Engine to observe source changes.
//
// A nil Engine returns an error matching ErrInvalidConfig. A blank prompt ID
// matches ErrInvalidRequest. An absent exact ID or version matches
// ErrPromptNotFound and not ErrPromptLoad. An ambiguous omitted version matches
// ErrPromptLoad. Invalid definitions and resource failures reject construction
// instead. Cancellation during lookup preserves the context error. InspectPrompt
// returns no partial result on error.
// matches ErrInvalidRequest. An absent exact ID matches ErrPromptNotFound and
// not ErrPromptLoad. Invalid definitions and resource failures reject
// construction instead. Cancellation during lookup preserves the context
// error. InspectPrompt returns no partial result on error.
func (e *Engine) InspectPrompt(
ctx context.Context,
promptID string,
promptVersion string,
) (*PromptInspection, error) {
if e == nil || e.internal == nil {
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
}
inspection, err := e.internal.InspectPrompt(ctx, promptID, promptVersion)
inspection, err := e.internal.InspectPrompt(ctx, promptID)
if err != nil {
return nil, mapPublicError(err)
}
@@ -346,6 +347,32 @@ func (e *Engine) InspectProfile(ctx context.Context, profileID string) (*Profile
// relationship to ctx.Err is promised. Prepare returns no partial result on
// error. It validates RunRequest.TransientRetries but makes no model attempts.
func (e *Engine) Prepare(ctx context.Context, req RunRequest) (*PreparedRun, error) {
prepared, err := e.prepareRequest(ctx, req)
if err != nil {
return nil, err
}
return fromDomainPreparedRun(prepared), nil
}
// ValidateRequest reports whether req passes the same preparation path used by
// [Engine.Prepare] at the time of this call. A nil error means this concrete
// request passed preparation. It checks only this invocation and returns errors
// as encountered rather than aggregating them. All supplied inputs follow the
// same loading behavior as [Engine.Prepare], including inputs unused by the
// selected template branch. The method does not call an LLM or retain a
// prepared snapshot or executable handle, but preparation may still perform
// resource I/O and incur its associated time and memory costs.
//
// Its errors and context handling follow [Engine.Prepare], including an error
// matching ErrInvalidConfig when the Engine is nil. Success does not guarantee
// provider acceptance or later execution success. Use [Engine.PrepareExecution]
// when execution must use the snapshot produced by preparation.
func (e *Engine) ValidateRequest(ctx context.Context, req RunRequest) error {
_, err := e.prepareRequest(ctx, req)
return err
}
func (e *Engine) prepareRequest(ctx context.Context, req RunRequest) (*domain.PreparedRun, error) {
ctx = observation.WithoutExecutionState(ctx)
if e == nil || e.internal == nil {
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
@@ -360,15 +387,16 @@ func (e *Engine) Prepare(ctx context.Context, req RunRequest) (*PreparedRun, err
if err != nil {
return nil, mapPublicError(err)
}
return fromDomainPreparedRun(prepared), nil
return prepared, nil
}
// PrepareExecution completely prepares a prompt request without calling the
// configured LLMClient or reserving backend admission capacity. Validated
// RunRequest.AppendedMessages are included in the frozen effective messages.
//
// The returned opaque handle is bound to this Engine and permits one
// [Engine.RunPrepared] invocation. Preparation freezes the selected sources,
// The returned opaque handle is bound to this Engine's underlying execution
// owner, shared with its namespace views, and permits one [Engine.RunPrepared]
// invocation in total. Preparation freezes the selected sources,
// rendered messages, effective settings, inputs, provider structured-output
// metadata, and validation resources needed by that invocation. The handle
// retains a direct RunRequest.APIKey only in private execution state;
@@ -480,6 +508,11 @@ func (e *Engine) run(ctx context.Context, req RunRequest) (*RunResult, error) {
// RunPrepared atomically claims and executes a handle created by
// [Engine.PrepareExecution].
//
// The originating Engine and namespace views sharing its underlying owner may
// claim the handle. Execution uses its captured selections without applying the
// invoking view's namespaces. An Engine constructed separately by [NewEngine]
// is a foreign owner.
//
// A valid owning-Engine invocation consumes the handle's one attempt before
// credential revalidation, backend admission, generation, or validation.
// Cancellation, capacity rejection, generation failure, operational

View File

@@ -1,18 +1,18 @@
prompts:
example.assessment:
"1.0.0":
default_profile: assessment-profile
description: Assess a structured proposal independently of schema validity.
inputs:
- name: proposal
required: true
content_type: application/json
messages:
- role: system
content: Return a concise assessment as JSON.
- role: user
content: '{{input "proposal"}}'
output:
format: json
validation_mode: json_schema
schema_path: assessment.schema.json
version: "1.0.0"
default_profile: assessment-profile
description: Assess a structured proposal independently of schema validity.
inputs:
- name: proposal
required: true
content_type: application/json
messages:
- role: system
content: Return a concise assessment as JSON.
- role: user
content: '{{input "proposal"}}'
output:
format: json
validation_mode: json_schema
schema_path: assessment.schema.json

View File

@@ -1,18 +1,18 @@
prompts:
example.proposal:
"1.0.0":
default_profile: proposal-profile
description: Propose a bounded improvement from a synthetic idea.
inputs:
- name: idea
required: true
content_type: text/plain
messages:
- role: system
content: Return a concise proposal as JSON.
- role: user
content: '{{input "idea"}}'
output:
format: json
validation_mode: json_schema
schema_path: proposal.schema.json
version: "1.0.0"
default_profile: proposal-profile
description: Propose a bounded improvement from a synthetic idea.
inputs:
- name: idea
required: true
content_type: text/plain
messages:
- role: system
content: Return a concise proposal as JSON.
- role: user
content: '{{input "idea"}}'
output:
format: json
validation_mode: json_schema
schema_path: proposal.schema.json

View File

@@ -0,0 +1,17 @@
backends:
offline:
endpoint: https://review.example.invalid/v1
profiles:
audit:
backend: review/offline
model: review-model
prompts:
check:
default_profile: review/audit
description: Inspect a review item without contacting a model provider.
messages:
- role: user
content: Review the supplied synthetic example.
output:
format: text
validation_mode: none

View File

@@ -0,0 +1,5 @@
version: "1"
configkit:
imports:
- path: fragments/resources.yaml
release_version: "1.4.0"

View File

@@ -0,0 +1,28 @@
backends:
offline:
endpoint: https://example.invalid/v1
profiles:
base:
backend: support/offline
model: offline-model
timeout_seconds: 90
fast:
extends: base
timeout_seconds: 60
prompts:
prepare:
version: "1.0.0"
default_profile: support/fast
description: Prepare a prompt without contacting a model provider.
inputs:
- name: note
required: true
content_type: text/plain
messages:
- role: system
content: Summarize the note in one sentence.
- role: user
content_file: ../messages/note.md
output:
format: text
validation_mode: basic

View File

@@ -0,0 +1 @@
{{input "note"}}

View File

@@ -0,0 +1,4 @@
# A later input intentionally replaces only the inherited profile timeout.
profiles:
fast:
timeout_seconds: 30

View File

@@ -11,41 +11,138 @@ import (
)
type summary struct {
PromptID string `json:"prompt_id"`
PromptVersion string `json:"prompt_version"`
SelectedProfile string `json:"selected_profile"`
Model string `json:"model"`
MessageCount int `json:"message_count"`
PromptID string `json:"prompt_id"`
PromptVersion *string `json:"prompt_version,omitempty"`
SelectedProfile string `json:"selected_profile"`
Model string `json:"model"`
MessageCount int `json:"message_count"`
ProfileTimeout int `json:"profile_timeout_seconds"`
Namespaces []string `json:"namespaces"`
Collections []collectionSummary `json:"collections"`
ScopedPrompts []string `json:"scoped_prompts"`
ReviewPrompts []string `json:"review_prompts"`
}
type collectionSummary struct {
Namespace string `json:"namespace"`
ReleaseVersion *string `json:"release_version"`
}
func main() {
endpoint, model := "https://example.invalid/v1", "offline-model"
support, err := promptkit.CollectionInput(
"support",
configkit.File("examples/go-library/prepare/collections/support/collection.yaml"),
)
if err != nil {
exit(err)
}
review, err := promptkit.CollectionInput(
"review",
configkit.File("examples/go-library/prepare/collections/review/collection.yaml"),
)
if err != nil {
exit(err)
}
override, err := promptkit.CollectionInput(
"support",
configkit.File("examples/go-library/prepare/collections/support/operator-overrides.yaml"),
)
if err != nil {
exit(err)
}
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: promptkit.ProfileSource("example profile", map[string]promptkit.ProfileConfig{
"offline-example": {Endpoint: &endpoint, Model: &model},
})}}}},
Main: &configkit.Input{Source: configkit.File("examples/go-library/prepare/prompt.yaml")},
Underlays: []configkit.Underlay{
{Inputs: []configkit.Input{support, review}},
{Inputs: []configkit.Input{override}},
},
}})
if err != nil {
exit(err)
}
ctx := context.Background()
scoped, err := engine.WithNamespaces(promptkit.Namespaces{
Prompt: "support", Profile: "support", Backend: "support",
})
if err != nil {
exit(err)
}
view, err := engine.InspectConfiguration()
if err != nil {
exit(err)
}
if _, found := view.Explain(configkit.Keys("profiles", "offline-example", "model")); !found {
exit(fmt.Errorf("configured profile model was not found"))
if _, found := view.Explain(configkit.Keys(
"collections", "support", "profiles", "fast", "timeout_seconds",
)); !found {
exit(fmt.Errorf("collection profile override was not found"))
}
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
PromptID: "example.prepare",
Inputs: map[string]promptkit.ArtifactRef{
"note": promptkit.Inline("Ada finished the migration review."),
},
requirements, err := scoped.InspectRequirements(ctx, promptkit.RequirementInspectionRequest{
PromptID: "prepare",
})
if err != nil {
exit(err)
}
if !requirements.Complete || len(requirements.Inputs) != 1 ||
requirements.Inputs[0].Name != "note" ||
requirements.Inputs[0].Kind != promptkit.RequirementRequired {
exit(fmt.Errorf("required note input was not discovered before preparation"))
}
request := promptkit.RunRequest{
PromptID: "prepare",
ProfileID: "fast",
Inputs: map[string]promptkit.ArtifactRef{
"note": promptkit.Inline("Ada finished the migration review."),
},
}
if err := scoped.ValidateRequest(ctx, request); err != nil {
exit(err)
}
prepared, err := scoped.Prepare(ctx, request)
if err != nil {
exit(err)
}
profile, err := scoped.InspectProfile(ctx, "fast")
if err != nil {
exit(err)
}
if profile.EffectiveModelParams.TimeoutSeconds != 30 {
exit(fmt.Errorf(
"later profile override was not applied: timeout is %d seconds",
profile.EffectiveModelParams.TimeoutSeconds,
))
}
namespaces, err := scoped.ListNamespaces(ctx)
if err != nil {
exit(err)
}
collections, err := scoped.ListCollections(ctx)
if err != nil {
exit(err)
}
scopedPrompts, err := scoped.ListPrompts(ctx)
if err != nil {
exit(err)
}
reviewPrompts, err := scoped.ListPrompts(ctx, promptkit.NamespaceFilter{
Mode: promptkit.NamespaceNamed, Name: "review",
})
if err != nil {
exit(err)
}
scopedPromptIDs := make([]string, len(scopedPrompts))
for index := range scopedPrompts {
scopedPromptIDs[index] = scopedPrompts[index].ID
}
reviewPromptIDs := make([]string, len(reviewPrompts))
for index := range reviewPrompts {
reviewPromptIDs[index] = reviewPrompts[index].ID
}
collectionSummaries := make([]collectionSummary, len(collections))
for index := range collections {
collectionSummaries[index] = collectionSummary{
Namespace: collections[index].Namespace, ReleaseVersion: collections[index].ReleaseVersion,
}
}
encoder := json.NewEncoder(os.Stdout)
encoder.SetIndent("", " ")
@@ -55,6 +152,11 @@ func main() {
SelectedProfile: prepared.SelectedProfileID,
Model: prepared.EffectiveModelParams.Model,
MessageCount: len(prepared.Messages),
ProfileTimeout: profile.EffectiveModelParams.TimeoutSeconds,
Namespaces: namespaces,
Collections: collectionSummaries,
ScopedPrompts: scopedPromptIDs,
ReviewPrompts: reviewPromptIDs,
}); err != nil {
exit(err)
}

View File

@@ -1,17 +0,0 @@
prompts:
example.prepare:
"1.0.0":
default_profile: offline-example
description: Prepare a prompt without contacting a model provider.
inputs:
- name: note
required: true
content_type: text/plain
messages:
- role: system
content: Summarize the note in one sentence.
- role: user
content: '{{input "note"}}'
output:
format: text
validation_mode: basic

View File

@@ -1,17 +1,17 @@
prompts:
example.run:
"1.0.0":
default_profile: offline-example
description: Run a prompt with a deterministic injected model client.
inputs:
- name: note
required: true
content_type: text/plain
messages:
- role: system
content: Summarize the note in one sentence.
- role: user
content: '{{input "note"}}'
output:
format: text
validation_mode: basic
version: "1.0.0"
default_profile: offline-example
description: Run a prompt with a deterministic injected model client.
inputs:
- name: note
required: true
content_type: text/plain
messages:
- role: system
content: Summarize the note in one sentence.
- role: user
content: '{{input "note"}}'
output:
format: text
validation_mode: basic

View File

@@ -18,9 +18,8 @@ func (r RunRequest) GoString() string {
func (r RunRequest) redactedString() string {
return fmt.Sprintf(
"promptkit.RunRequest{PromptID:%q PromptVersion:%q ProfileID:%q APIKeySet:%t Inputs:%d Vars:%d ExecutionSet:%t ValidationSet:%t AppendedMessages:%d}",
"promptkit.RunRequest{PromptID:%q ProfileID:%q APIKeySet:%t Inputs:%d Vars:%d ExecutionSet:%t ValidationSet:%t AppendedMessages:%d}",
r.PromptID,
r.PromptVersion,
r.ProfileID,
r.APIKey != "",
len(r.Inputs),

View File

@@ -19,15 +19,15 @@ import (
const capturedPromptDocument = `
prompts:
captured:
"1":
default_profile: offline
session_id: '{{.session}}'
messages:
- role: user
content_file: body.tmpl
output:
format: text
validation_mode: none
version: "1"
default_profile: offline
session_id: '{{.session}}'
messages:
- role: user
content_file: body.tmpl
output:
format: text
validation_mode: none
`
func capturedPromptEngine(t *testing.T, source fs.FS) *promptkit.Engine {
@@ -149,12 +149,13 @@ func TestPreparationStillReadsRequestArtifactsAndSchemasAfterConstruction(t *tes
"offline": {Endpoint: &endpoint, Model: &model},
})}}}},
Main: &configkit.Input{Source: configkit.Mapping("structured prompt", map[string]any{
"prompts": map[string]any{"structured": map[string]any{"1": map[string]any{
"prompts": map[string]any{"structured": map[string]any{
"version": "1",
"default_profile": "offline",
"inputs": []any{map[string]any{"name": "note", "required": true}},
"messages": []any{map[string]any{"role": "user", "content": `{{input "note"}}`}},
"output": map[string]any{"format": "json", "validation_mode": "json_schema", "schema_path": "schema.json"},
}}},
}},
})},
}}, promptkit.WithSchemaFS(schemas, "."))
if err != nil {

11
go.mod
View File

@@ -1,15 +1,18 @@
module gitea.maximumdirect.net/eric/promptkit
go 1.25.5
go 1.26.0
require (
gitea.maximumdirect.net/eric/ConfigKit v0.5.0
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v3 v3.20260930.0
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v3 v3.20260930.0
gitea.maximumdirect.net/eric/ConfigKit v0.8.0
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2
)
require (
gitea.maximumdirect.net/eric/ReleaseKit v0.4.1 // indirect
github.com/yuin/goldmark/v2 v2.1.6 // indirect
go.yaml.in/yaml/v4 v4.0.0-rc.6 // indirect
golang.org/x/mod v0.40.0 // indirect
golang.org/x/text v0.14.0 // indirect
)
tool gitea.maximumdirect.net/eric/ReleaseKit/cmd/releasekit

16
go.sum
View File

@@ -1,16 +1,16 @@
gitea.maximumdirect.net/eric/ConfigKit v0.5.0 h1:2KwG10+kApWM3avOoYA+p768xcbHUp7tQ2rqizBNpyg=
gitea.maximumdirect.net/eric/ConfigKit v0.5.0/go.mod h1:tYWpsrbLyu05r8+1MapWp5fN6ZrX9I8hU2xEPedYmYk=
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v3 v3.20260930.0 h1:lzfCDbzJQRGzIlRPRprnAV9hVVL5zUQy80VMQ0kbYa0=
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v3 v3.20260930.0/go.mod h1:HjS0f+3q0IdHCTEz+VVAEgIwIWJFAZUjuKdAtvtqKdg=
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v3 v3.20260930.0 h1:h12ZTRldAmpLAomj0oQRGteFxHLN1LPE6XDl9zJNbm8=
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v3 v3.20260930.0/go.mod h1:OEFBeKdjTsLzqJwLmXI2O5gG3r64FktcVJVb3mBrRfg=
gitea.maximumdirect.net/eric/ConfigKit v0.8.0 h1:xCIhYnOHjw7rFvB+vIremPKGeROOxKTomnUEUipuQno=
gitea.maximumdirect.net/eric/ConfigKit v0.8.0/go.mod h1:C82sbyKHYMV1OwaD2GJanFrwTfcZfjlW2JsGze5b0kA=
gitea.maximumdirect.net/eric/ReleaseKit v0.4.1 h1:Qn+mdebYOzCVCEXgCbuGSO/WHmJtItwaQ8e3XwY3qtI=
gitea.maximumdirect.net/eric/ReleaseKit v0.4.1/go.mod h1:LVodqNQ3OIKv9NBq+fFeyMByd1lHY25YOmmpah/NaNI=
github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI=
github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ=
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU=
github.com/yuin/goldmark/v2 v2.1.6 h1:igy0q9JcqovoFXFx/XjtU8sHVhcUHavPj2sA3EoKJUE=
github.com/yuin/goldmark/v2 v2.1.6/go.mod h1:G6M4/qOFtfn01/o14BU1UR2Lo5N3S9Qo7xCuz/sHjGQ=
go.yaml.in/yaml/v4 v4.0.0-rc.6 h1:1h7H1ohdUh93/FyE4YaDa1Zh64K6VVbjF4K6WUxMtH4=
go.yaml.in/yaml/v4 v4.0.0-rc.6/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0=
golang.org/x/mod v0.40.0 h1:hUv+3cXcdRHz08UmSiOob7sadHig73uo5bkXxQ/tvUs=
golang.org/x/mod v0.40.0/go.mod h1:0/weTWkPWGBikyTWAX3dkjVztMmBA5hM0DH6BElSupE=
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

View File

@@ -21,6 +21,7 @@ var (
)
const fileReadChunkSize = 64 * 1024
const fileReadInitialCapacityLimit = 1 << 20
// Reader resolves artifact references into actual artifacts.
type Reader interface {
@@ -133,7 +134,11 @@ func readFileArtifact(ctx context.Context, path string, open func(string) (artif
return nil, fmt.Errorf("%w: %s", ErrUnsupportedFile, path)
}
data := make([]byte, 0)
capacity := 0
if size := openedInfo.Size(); size > 0 {
capacity = int(min(size, int64(fileReadInitialCapacityLimit)))
}
data := make([]byte, 0, capacity)
chunk := make([]byte, fileReadChunkSize)
for {
if err := ctx.Err(); err != nil {
@@ -153,6 +158,13 @@ func readFileArtifact(ctx context.Context, path string, open func(string) (artif
return nil, fmt.Errorf("failed to read file %s: %w", path, readErr)
}
}
if cap(data)-len(data) > len(data) {
// A stale size hint should not make a small completed artifact retain a
// disproportionately large backing array.
compact := make([]byte, len(data))
copy(compact, data)
data = compact
}
contentType := mime.TypeByExtension(filepath.Ext(path))
if contentType == "" {

View File

@@ -0,0 +1,180 @@
package artifact
import (
"bytes"
"context"
"crypto/sha256"
"fmt"
"mime"
"os"
"path/filepath"
"runtime"
"testing"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
)
var artifactFileReadBenchmarkSink *domain.Artifact
func BenchmarkArtifactFileRead(b *testing.B) {
for _, fixture := range makeArtifactReadBenchmarkFiles(b) {
b.Run(fixture.name, func(b *testing.B) {
reader := &fileReader{open: openArtifactFile}
ref := domain.ArtifactRef{Type: domain.ArtifactRefFile, URI: fixture.path}
checkArtifactFileRead(b, reader, ref, fixture.body)
b.ReportAllocs()
b.SetBytes(int64(len(fixture.body)))
b.ResetTimer()
for range b.N {
artifact, err := reader.Read(context.Background(), ref)
if err != nil {
b.Fatalf("read artifact: %v", err)
}
artifactFileReadBenchmarkSink = artifact
}
b.StopTimer()
runtime.KeepAlive(artifactFileReadBenchmarkSink)
})
}
}
// BenchmarkRetainedArtifactFileRead measures live artifact bodies when an
// opened file reports a size larger than the content that reaches EOF.
func BenchmarkRetainedArtifactFileRead(b *testing.B) {
for _, fixture := range []struct {
name string
body []byte
}{
{name: "small", body: bytes.Repeat([]byte("x"), 256<<10)},
{name: "empty", body: []byte{}},
} {
b.Run(fixture.name, func(b *testing.B) {
benchmarkRetainedArtifactFileRead(b, fixture.body)
})
}
}
func benchmarkRetainedArtifactFileRead(b *testing.B, body []byte) {
path := filepath.Join(b.TempDir(), "overreported.txt")
if err := os.WriteFile(path, body, 0o600); err != nil {
b.Fatal(err)
}
mime.TypeByExtension(filepath.Ext(path))
reader := &fileReader{open: func(path string) (artifactFile, error) {
file, err := os.Open(path)
if err != nil {
return nil, err
}
return &sizeReportingArtifactFile{artifactFile: file, reportedSize: 16 << 20}, nil
}}
ref := domain.ArtifactRef{Type: domain.ArtifactRefFile, URI: path}
checkArtifactFileRead(b, reader, ref, body)
const retainedBatchSize = 4
retained := make([]*domain.Artifact, retainedBatchSize)
artifactFileReadBenchmarkSink = nil
runtime.GC()
var setupMemory runtime.MemStats
runtime.ReadMemStats(&setupMemory)
runtime.KeepAlive(reader)
runtime.KeepAlive(ref)
b.ReportAllocs()
b.SetBytes(int64(len(body)))
b.ResetTimer()
for iteration := 0; iteration < b.N; iteration++ {
artifact, err := reader.Read(context.Background(), ref)
if err != nil {
b.Fatalf("read retained artifact: %v", err)
}
retained[iteration%len(retained)] = artifact
}
b.StopTimer()
for i, artifact := range retained {
if artifact != nil {
continue
}
artifact, err := reader.Read(context.Background(), ref)
if err != nil {
b.Fatalf("fill retained artifact batch: %v", err)
}
retained[i] = artifact
}
runtime.GC()
var retainedMemory runtime.MemStats
runtime.ReadMemStats(&retainedMemory)
b.ReportMetric(float64(int64(retainedMemory.HeapAlloc)-int64(setupMemory.HeapAlloc)), "retained-B")
runtime.KeepAlive(retained)
runtime.KeepAlive(reader)
runtime.KeepAlive(ref)
}
type artifactReadBenchmarkFile struct {
name string
path string
body []byte
}
func makeArtifactReadBenchmarkFiles(b *testing.B) []artifactReadBenchmarkFile {
b.Helper()
fixtures := []struct {
name string
size int
}{
{name: "small_4KiB", size: 4 << 10},
{name: "1MiB", size: 1 << 20},
{name: "large_4MiB", size: 4 << 20},
}
results := make([]artifactReadBenchmarkFile, 0, len(fixtures))
for _, fixture := range fixtures {
body := bytes.Repeat([]byte("x"), fixture.size)
path := filepath.Join(b.TempDir(), fixture.name+".txt")
if err := os.WriteFile(path, body, 0o600); err != nil {
b.Fatal(err)
}
mime.TypeByExtension(filepath.Ext(path))
results = append(results, artifactReadBenchmarkFile{
name: fixture.name,
path: path,
body: body,
})
}
return results
}
func checkArtifactFileRead(b *testing.B, reader Reader, ref domain.ArtifactRef, want []byte) {
b.Helper()
artifact, err := reader.Read(context.Background(), ref)
if err != nil {
b.Fatalf("check artifact read: %v", err)
}
if !bytes.Equal(artifact.Body, want) || artifact.Size != int64(len(want)) {
b.Fatalf("artifact size=%d body length=%d, want %d", artifact.Size, len(artifact.Body), len(want))
}
wantHash := fmt.Sprintf("%x", sha256.Sum256(want))
if artifact.Hash != wantHash {
b.Fatalf("artifact hash=%q, want %q", artifact.Hash, wantHash)
}
}
type sizeReportingArtifactFile struct {
artifactFile
reportedSize int64
}
func (file *sizeReportingArtifactFile) Stat() (os.FileInfo, error) {
info, err := file.artifactFile.Stat()
if err != nil {
return nil, err
}
return reportedArtifactFileInfo{FileInfo: info, size: file.reportedSize}, nil
}
type reportedArtifactFileInfo struct {
os.FileInfo
size int64
}
func (info reportedArtifactFileInfo) Size() int64 { return info.size }

View File

@@ -3,7 +3,9 @@ package artifact
import (
"bytes"
"context"
"crypto/sha256"
"errors"
"fmt"
"os"
"path/filepath"
"strings"
@@ -232,6 +234,64 @@ func TestFileReaderFailuresAndMetadata(t *testing.T) {
})
}
func TestFileReaderUsesOpenedSizeOnlyAsACapacityHint(t *testing.T) {
tests := []struct {
name string
body []byte
reportedSize int64
}{
{name: "overreported size", body: []byte("actual file content"), reportedSize: 128},
{name: "overreported size with empty content", body: []byte{}, reportedSize: 16 << 20},
{name: "underreported size", body: bytes.Repeat([]byte("x"), fileReadChunkSize+17), reportedSize: 1},
{name: "very large hint with small content", body: []byte("small actual content"), reportedSize: int64(^uint64(0) >> 1)},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
filePath := filepath.Join(t.TempDir(), "artifact.txt")
if err := os.WriteFile(filePath, tc.body, 0o600); err != nil {
t.Fatal(err)
}
reader := &fileReader{open: func(path string) (artifactFile, error) {
file, err := os.Open(path)
if err != nil {
return nil, err
}
info, err := file.Stat()
if err != nil {
file.Close()
return nil, err
}
return &reportedInfoFile{
artifactFile: file,
info: reportedFileSizeInfo{FileInfo: info, size: tc.reportedSize},
}, nil
}}
artifact, err := reader.Read(context.Background(), domain.ArtifactRef{
Type: domain.ArtifactRefFile,
URI: filePath,
})
if err != nil {
t.Fatalf("read artifact: %v", err)
}
if !bytes.Equal(artifact.Body, tc.body) {
t.Fatalf("artifact body length=%d, want %d", len(artifact.Body), len(tc.body))
}
if artifact.Size != int64(len(tc.body)) {
t.Fatalf("artifact size=%d, want %d", artifact.Size, len(tc.body))
}
if artifact.Name != filepath.Base(filePath) || artifact.URI != filePath || artifact.ContentType == "" {
t.Fatalf("unexpected artifact metadata: name=%q URI=%q content type=%q", artifact.Name, artifact.URI, artifact.ContentType)
}
wantHash := fmt.Sprintf("%x", sha256.Sum256(tc.body))
if artifact.Hash != wantHash {
t.Fatalf("artifact hash=%q, want %q", artifact.Hash, wantHash)
}
})
}
}
func TestFileReaderCancelsAfterReadProgress(t *testing.T) {
filePath := filepath.Join(t.TempDir(), "artifact.bin")
content := bytes.Repeat([]byte("x"), fileReadChunkSize*2)
@@ -268,6 +328,13 @@ func (f *reportedInfoFile) Stat() (os.FileInfo, error) {
return f.info, nil
}
type reportedFileSizeInfo struct {
os.FileInfo
size int64
}
func (info reportedFileSizeInfo) Size() int64 { return info.size }
type cancelAfterProgressFile struct {
artifactFile
cancel context.CancelFunc

View File

@@ -7,13 +7,12 @@ import (
"fmt"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
"gitea.maximumdirect.net/eric/promptkit/internal/jsonvalue"
)
const (
// OpenRouterID identifies the maintained OpenRouter backend.
// OpenRouterID is the conventional OpenRouter backend identifier.
OpenRouterID = "openrouter"
// RakestrawHomeID identifies the maintained Rakestrawhome backend.
// RakestrawHomeID is the conventional Rakestrawhome backend identifier.
RakestrawHomeID = "rakestrawhome"
)
@@ -31,13 +30,12 @@ type Registry struct {
func NewEffectiveRegistry(definitions map[string]domain.Backend) (*Registry, error) {
registry := &Registry{backends: make(map[string]domain.Backend, len(definitions))}
for id, definition := range definitions {
params, err := jsonvalue.CopyMap(definition.ExtraParams)
if err != nil {
return nil, fmt.Errorf("backend %q extra parameters: %w", id, err)
}
definition.ID = id
definition.ExtraParams = params
registry.backends[id] = definition
cloned, err := domain.CloneBackend(definition)
if err != nil {
return nil, fmt.Errorf("backend %q: %w", id, err)
}
registry.backends[id] = cloned
}
return registry, nil
}
@@ -51,12 +49,11 @@ func (r *Registry) GetBackend(id string) (domain.Backend, error) {
if !ok {
return domain.Backend{}, fmt.Errorf("%w: %q", ErrBackendNotFound, id)
}
extraParams, err := jsonvalue.CopyMap(definition.ExtraParams)
cloned, err := domain.CloneBackend(definition)
if err != nil {
return domain.Backend{}, fmt.Errorf("copy backend %q: %w", id, err)
}
definition.ExtraParams = extraParams
return definition, nil
return cloned, nil
}
// CapacityPolicies returns a copy of the normalized policies for limited

View File

@@ -11,25 +11,61 @@ import (
func TestEffectiveRegistryOwnsDefinitionsAndCapacityPolicies(t *testing.T) {
params := map[string]any{"nested": map[string]any{"enabled": true}}
defaultProfile := "catalog-profile"
reachabilityEndpoint := "https://probe.example/ready"
authenticationKind := "openrouter_key"
authenticationEndpoint := "https://probe.example/key"
smokeTests := &domain.BackendSmokeTestConfig{
DefaultProfile: &defaultProfile,
Reachability: &domain.ReachabilityProbeConfig{Endpoint: &reachabilityEndpoint},
Authentication: &domain.AuthenticationProbeConfig{
Kind: &authenticationKind, Endpoint: &authenticationEndpoint,
},
}
registry, err := backend.NewEffectiveRegistry(map[string]domain.Backend{
" local.v1 ": {
Endpoint: "https://backend.example/v1", ExtraParams: params,
ConcurrencyLimit: 2, QueueCapacity: 3, QueueCapacitySet: true,
SmokeTests: smokeTests,
},
})
if err != nil {
t.Fatal(err)
}
params["nested"].(map[string]any)["enabled"] = false
defaultProfile = "mutated-profile"
reachabilityEndpoint = "https://mutated.example/ready"
authenticationKind = "none"
authenticationEndpoint = "https://mutated.example/key"
selected, err := registry.GetBackend(" local.v1 ")
if err != nil || selected.ID != " local.v1 " || selected.ExtraParams["nested"].(map[string]any)["enabled"] != true {
t.Fatalf("effective backend was not captured under its literal ID: %+v, %v", selected, err)
}
if selected.SmokeTests == nil || selected.SmokeTests.DefaultProfile == nil ||
*selected.SmokeTests.DefaultProfile != "catalog-profile" || selected.SmokeTests.Reachability == nil ||
selected.SmokeTests.Reachability.Endpoint == nil || *selected.SmokeTests.Reachability.Endpoint != "https://probe.example/ready" ||
selected.SmokeTests.Authentication == nil || selected.SmokeTests.Authentication.Kind == nil ||
*selected.SmokeTests.Authentication.Kind != "openrouter_key" || selected.SmokeTests.Authentication.Endpoint == nil ||
*selected.SmokeTests.Authentication.Endpoint != "https://probe.example/key" {
t.Fatalf("registry did not capture nested diagnostic metadata: %+v", selected.SmokeTests)
}
selected.ExtraParams["nested"].(map[string]any)["enabled"] = false
*selected.SmokeTests.DefaultProfile = "mutated-return"
*selected.SmokeTests.Reachability.Endpoint = "https://mutated-return.example/ready"
*selected.SmokeTests.Authentication.Kind = "none"
*selected.SmokeTests.Authentication.Endpoint = "https://mutated-return.example/key"
again, err := registry.GetBackend(" local.v1 ")
if err != nil || again.ExtraParams["nested"].(map[string]any)["enabled"] != true {
t.Fatalf("lookup exposed registry values: %+v, %v", again, err)
}
if again.SmokeTests == nil || again.SmokeTests.DefaultProfile == nil ||
*again.SmokeTests.DefaultProfile != "catalog-profile" || again.SmokeTests.Reachability == nil ||
again.SmokeTests.Reachability.Endpoint == nil || *again.SmokeTests.Reachability.Endpoint != "https://probe.example/ready" ||
again.SmokeTests.Authentication == nil || again.SmokeTests.Authentication.Kind == nil ||
*again.SmokeTests.Authentication.Kind != "openrouter_key" || again.SmokeTests.Authentication.Endpoint == nil ||
*again.SmokeTests.Authentication.Endpoint != "https://probe.example/key" {
t.Fatalf("lookup exposed nested diagnostic metadata: %+v", again.SmokeTests)
}
policies := registry.CapacityPolicies()
if !reflect.DeepEqual(policies, map[string]domain.BackendCapacityPolicy{
" local.v1 ": {ConcurrencyLimit: 2, QueueCapacity: 3},

View File

@@ -1,52 +0,0 @@
// Package catalog selects the maintained configuration assets supplied with
// PromptKit. ConfigKit owns their composition and effective validation.
package catalog
import (
"context"
"fmt"
"io/fs"
"gitea.maximumdirect.net/eric/ConfigKit"
openrouter "gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v3"
rakestrawhome "gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v3"
"gitea.maximumdirect.net/eric/promptkit/internal/backend"
"gitea.maximumdirect.net/eric/promptkit/internal/filecatalog"
)
// Underlays selects external catalogs followed by PromptKit-owned profiles.
// Neither group is interpreted before the final effective configuration loads.
func Underlays() ([]configkit.Underlay, error) {
openrouterInputs, err := assetInputs("OpenRouter catalog", openrouter.FS(), openrouter.Root)
if err != nil {
return nil, err
}
rakestrawhomeInputs, err := assetInputs("Rakestrawhome catalog", rakestrawhome.FS(), rakestrawhome.Root)
if err != nil {
return nil, err
}
catalogInputs := append(openrouterInputs, rakestrawhomeInputs...)
owned := configkit.Input{Source: configkit.Mapping("PromptKit-owned profiles", map[string]any{
"profiles": map[string]any{
"promptkit-openrouter-standard": map[string]any{"extends": "gemini-flash-latest"},
"promptkit-openrouter-auto": map[string]any{"backend": backend.OpenRouterID, "model": "openrouter/auto"},
},
})}
return []configkit.Underlay{{Inputs: catalogInputs}, {Inputs: []configkit.Input{owned}}}, nil
}
func assetInputs(label string, filesystem fs.FS, root string) ([]configkit.Input, error) {
files, err := filecatalog.FindFSYAMLFiles(context.Background(), filesystem, root)
if err != nil {
return nil, fmt.Errorf("discover %s configuration: %w", label, err)
}
namespace := configkit.NewFilesystem(label, filesystem)
inputs := make([]configkit.Input, 0, len(files))
for _, name := range files {
inputs = append(inputs, configkit.Input{
Source: namespace.File(name),
Filesystem: &configkit.FilesystemContext{FS: &namespace, Root: root},
})
}
return inputs, nil
}

View File

@@ -1,68 +0,0 @@
package catalog
import (
"context"
"errors"
"testing"
"gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/promptkit/internal/configuration"
)
func TestPublishedCatalogsResolveThroughEffectiveConfiguration(t *testing.T) {
underlays, err := Underlays()
if err != nil {
t.Fatal(err)
}
state, err := configuration.Load(configkit.Input{Source: configkit.NoSource()}, underlays)
if err != nil {
t.Fatal(err)
}
openrouter, err := state.GetBackend("openrouter")
if err != nil || openrouter.Endpoint != "https://openrouter.ai/api/v1" || openrouter.ConcurrencyLimit != 16 || openrouter.QueueCapacity != 1024 {
t.Fatalf("OpenRouter backend: %+v, %v", openrouter, err)
}
rakestrawhome, err := state.GetBackend("rakestrawhome")
if err != nil || rakestrawhome.Endpoint != "https://inference.ai.rakestrawhome.com/v1" || rakestrawhome.ConcurrencyLimit != 4 {
t.Fatalf("Rakestrawhome backend: %+v, %v", rakestrawhome, err)
}
for id, want := range map[string]string{
"promptkit-openrouter-standard": "~google/gemini-flash-latest",
"promptkit-openrouter-auto": "openrouter/auto",
"rakestrawhome-gemma-4-31b": "google/gemma-4-31b-it",
} {
profile, err := state.GetProfile(context.Background(), id)
if err != nil || profile.ResolvedTarget.Model != want {
t.Fatalf("profile %s: %+v, %v", id, profile, err)
}
}
}
func TestConsumerOverlayReplacesCatalogDefaultsBeforeValidation(t *testing.T) {
underlays, err := Underlays()
if err != nil {
t.Fatal(err)
}
underlays = append(underlays, configkit.Underlay{Inputs: []configkit.Input{{Source: configkit.Mapping("operator", map[string]any{
"backends": map[string]any{"openrouter": map[string]any{"endpoint": "invalid lower value"}},
})}}})
main := configkit.Input{Source: configkit.Mapping("application", map[string]any{
"backends": map[string]any{"openrouter": map[string]any{"endpoint": "https://alternate.example/v1"}},
"profiles": map[string]any{"gemini-flash-latest": map[string]any{"model": "consumer-model"}},
})}
state, err := configuration.Load(main, underlays)
if err != nil {
t.Fatalf("replaced lower-layer value failed final validation: %v", err)
}
profile, err := state.GetProfile(context.Background(), "promptkit-openrouter-standard")
if err != nil || profile.ResolvedTarget.Model != "consumer-model" || profile.ResolvedTarget.Endpoint != "https://alternate.example/v1" {
t.Fatalf("maintained profile did not observe final configuration: %+v, %v", profile, err)
}
_, err = configuration.Load(configkit.Input{Source: configkit.Mapping("invalid final", map[string]any{
"backends": map[string]any{"openrouter": map[string]any{"endpoint": "invalid final value"}},
})}, underlays[:2])
var loadError *configkit.LoadError
if !errors.As(err, &loadError) {
t.Fatalf("invalid final backend did not produce a ConfigKit load error: %v", err)
}
}

View File

@@ -13,9 +13,12 @@ import (
const benchmarkDefinitionCount = 128
var (
constructionResult *loaded
selectedPrompt *domain.PromptDefinition
selectedProfile *domain.ExecutionProfile
constructionResult *loaded
selectedPrompt *domain.PromptDefinition
selectedPromptHash string
selectedPromptInspection *domain.PromptInspection
selectedProfile *domain.ExecutionProfile
selectedProfileIdentity string
)
type configurationShape struct {
@@ -28,6 +31,7 @@ type configurationShape struct {
func benchmarkPrompt(profileID string) map[string]any {
return map[string]any{
"version": "1",
"default_profile": profileID,
"messages": []any{map[string]any{"role": "user", "content": "Hello {{.name}}"}},
"output": map[string]any{"format": "text", "validation_mode": "none"},
@@ -41,7 +45,7 @@ func independentDefinitions(count int) configurationShape {
profileID := fmt.Sprintf("profile-%03d", index)
promptID := fmt.Sprintf("prompt-%03d", index)
profiles[profileID] = map[string]any{"endpoint": "https://example.test/v1", "model": profileID}
prompts[promptID] = map[string]any{"1": benchmarkPrompt(profileID)}
prompts[promptID] = benchmarkPrompt(profileID)
}
return configurationShape{
main: configkit.Input{Source: configkit.Mapping("independent definitions", map[string]any{"profiles": profiles, "prompts": prompts})},
@@ -60,7 +64,7 @@ func fileBackedDefinitions(count int) configurationShape {
profiles[profileID] = map[string]any{"endpoint": "https://example.test/v1", "model": profileID}
prompt := benchmarkPrompt(profileID)
prompt["messages"] = []any{map[string]any{"role": "user", "content_file": "body.md"}}
prompts[promptID] = map[string]any{"1": prompt}
prompts[promptID] = prompt
}
namespace := configkit.NewFilesystem("benchmark content", fstest.MapFS{
"assets/body.md": &fstest.MapFile{Data: []byte("Hello {{.name}}")},
@@ -86,7 +90,7 @@ func sharedBaseDefinitions(count int) configurationShape {
return configurationShape{
main: configkit.Input{Source: configkit.Mapping("inherited definitions", map[string]any{
"profiles": profiles,
"prompts": map[string]any{"shared": map[string]any{"1": benchmarkPrompt(fmt.Sprintf("child-%03d", count-1))}},
"prompts": map[string]any{"shared": benchmarkPrompt(fmt.Sprintf("child-%03d", count-1))},
})},
inputEntries: count + 2,
effectiveEntries: count + 2,
@@ -114,7 +118,7 @@ func layeredDefinitions(count int) configurationShape {
},
main: configkit.Input{Source: configkit.Mapping("main", map[string]any{
"profiles": final,
"prompts": map[string]any{"layered": map[string]any{"1": benchmarkPrompt(fmt.Sprintf("profile-%03d", count-1))}},
"prompts": map[string]any{"layered": benchmarkPrompt(fmt.Sprintf("profile-%03d", count-1))},
})},
inputEntries: count*4 + 1,
effectiveEntries: count + 1,
@@ -122,6 +126,40 @@ func layeredDefinitions(count int) configurationShape {
}
}
func BenchmarkCollectionFileBackedConstruction(b *testing.B) {
for _, count := range []int{4, 16, 64} {
b.Run(fmt.Sprintf("collections_%d", count), func(b *testing.B) {
collections := make(map[string]any, count)
for index := range count {
collections[fmt.Sprintf("collection-%03d", index)] = map[string]any{
"prompts": map[string]any{
"example": map[string]any{
"messages": []any{map[string]any{"role": "user", "content_file": "body.md"}},
"output": map[string]any{"format": "text", "validation_mode": "none"},
},
},
}
}
namespace := configkit.NewFilesystem("collection benchmark", fstest.MapFS{
"body.md": {Data: []byte("Hello")},
})
main := configkit.Input{
Source: configkit.Mapping("collections", map[string]any{"collections": collections}),
Filesystem: &configkit.FilesystemContext{FS: &namespace, Root: ".", BaseDir: "."},
}
b.ReportAllocs()
for b.Loop() {
result, err := load(main, nil)
if err != nil {
b.Fatal(err)
}
constructionResult = result
}
b.ReportMetric(float64(count), "collections/op")
})
}
}
func BenchmarkConfigurationConstruction(b *testing.B) {
for _, scenario := range []struct {
name string
@@ -157,7 +195,7 @@ func BenchmarkConfigurationSelection(b *testing.B) {
b.Run("prompt", func(b *testing.B) {
b.ReportAllocs()
for b.Loop() {
definition, _, err := result.prompt("prompt-127", "1")
definition, _, err := result.prompt("prompt-127")
if err != nil {
b.Fatal(err)
}
@@ -177,3 +215,116 @@ func BenchmarkConfigurationSelection(b *testing.B) {
b.ReportMetric(float64(shape.effectiveEntries), "effective-entries/op")
})
}
func inspectionExtraParams(entries int) map[string]any {
providers := make(map[string]any, entries)
for index := range entries {
providers[fmt.Sprintf("provider-%03d", index)] = map[string]any{
"enabled": true,
"labels": []any{fmt.Sprintf("tier-%d", index%4), fmt.Sprintf("region-%d", index%3)},
"limits": map[string]any{"burst": index + 1, "period": "minute"},
"routes": []any{map[string]any{"weight": index + 1, "fallback": true}},
}
}
return map[string]any{"provider_options": providers}
}
func inspectionDefinitions(extraParams map[string]any) configurationShape {
profiles := map[string]any{
"small": map[string]any{"model": "inspection-model", "extra_params": extraParams},
}
prompt := benchmarkPrompt("small")
prompt["inputs"] = []any{map[string]any{"name": "transcript", "required": true}}
prompts := map[string]any{
"inspection": prompt,
}
return configurationShape{
main: configkit.Input{Source: configkit.Mapping("inspection definitions", map[string]any{
"profiles": profiles,
"prompts": prompts,
})},
inputEntries: 2, effectiveEntries: 2, tiers: 1,
}
}
func BenchmarkInspectionSelection(b *testing.B) {
for _, scenario := range []struct {
name string
entries int
}{
{name: "small_extra", entries: 2},
{name: "nested_extra", entries: 64},
} {
shape := inspectionDefinitions(inspectionExtraParams(scenario.entries))
result, err := load(shape.main, nil)
if err != nil {
b.Fatal(err)
}
ctx := context.Background()
definition, hash, err := result.GetPromptSnapshot(ctx, "inspection")
if err != nil || definition.ID != "inspection" || definition.Version == nil || *definition.Version != "1" || hash == "" ||
len(definition.Inputs) != 1 || definition.Inputs[0].Name != "transcript" || !definition.Inputs[0].Required {
b.Fatalf("prompt lookup fixture is invalid: definition=%+v hash=%q err=%v", definition, hash, err)
}
profile, err := result.GetProfile(ctx, "small")
if err != nil || profile.ID != "small" || profile.ResolvedTarget.Model != "inspection-model" ||
len(profile.ResolvedTarget.ExtraParams["provider_options"].(map[string]any)) != scenario.entries {
b.Fatalf("profile lookup fixture is invalid: profile=%+v err=%v", profile, err)
}
inspection, err := result.GetPromptInspection(ctx, "inspection")
if err != nil || inspection.PromptID != "inspection" || inspection.PromptVersion == nil || *inspection.PromptVersion != "1" ||
len(inspection.Inputs) != 1 || inspection.Inputs[0].Name != "transcript" {
b.Fatalf("prompt inspection fixture is invalid: inspection=%+v err=%v", inspection, err)
}
identity, err := result.GetProfileIdentity(ctx, "small")
if err != nil || identity != "small" {
b.Fatalf("profile identity fixture is invalid: identity=%q err=%v", identity, err)
}
b.Run("prompt/"+scenario.name, func(b *testing.B) {
b.ReportAllocs()
for b.Loop() {
definition, hash, err := result.GetPromptSnapshot(ctx, "inspection")
if err != nil {
b.Fatal(err)
}
selectedPrompt = definition
selectedPromptHash = hash
}
b.ReportMetric(float64(scenario.entries), "extra-entries/op")
})
b.Run("prompt_inspection/"+scenario.name, func(b *testing.B) {
b.ReportAllocs()
for b.Loop() {
inspection, err := result.GetPromptInspection(ctx, "inspection")
if err != nil {
b.Fatal(err)
}
selectedPromptInspection = inspection
}
b.ReportMetric(float64(scenario.entries), "extra-entries/op")
})
b.Run("profile/"+scenario.name, func(b *testing.B) {
b.ReportAllocs()
for b.Loop() {
profile, err := result.GetProfile(ctx, "small")
if err != nil {
b.Fatal(err)
}
selectedProfile = profile
}
b.ReportMetric(float64(scenario.entries), "extra-entries/op")
})
b.Run("profile_identity/"+scenario.name, func(b *testing.B) {
b.ReportAllocs()
for b.Loop() {
identity, err := result.GetProfileIdentity(ctx, "small")
if err != nil {
b.Fatal(err)
}
selectedProfileIdentity = identity
}
b.ReportMetric(float64(scenario.entries), "extra-entries/op")
})
}
}

View File

@@ -16,6 +16,15 @@ type Inspection interface {
Explain(configkit.Path) (configkit.Explanation, bool)
WriteEffective(io.Writer) error
WriteExplanation(io.Writer, configkit.Path) error
capturedResult() *configkit.Result[document]
}
type inspectionResult struct {
*configkit.Result[document]
}
func (inspection inspectionResult) capturedResult() *configkit.Result[document] {
return inspection.Result
}
// Inspection returns only the retained result's inspection operations.
@@ -23,7 +32,19 @@ func (state *loaded) Inspection() Inspection {
if state == nil {
return nil
}
return state.result
return inspectionResult{Result: state.result}
}
// Diff compares two captured ConfigKit results without reconstructing them.
func Diff(left, right Inspection) ([]configkit.Difference, error) {
var leftResult, rightResult *configkit.Result[document]
if left != nil {
leftResult = left.capturedResult()
}
if right != nil {
rightResult = right.capturedResult()
}
return configkit.Diff(leftResult, rightResult)
}
func secretProviderPath(path configkit.Path) bool {
@@ -32,7 +53,7 @@ func secretProviderPath(path configkit.Path) bool {
return false
}
parts := strings.Split(strings.TrimPrefix(pointer, "/"), "/")
if len(parts) < 4 || (parts[0] != "backends" && parts[0] != "profiles") || parts[2] != "extra_params" {
if !providerExtraParamsPath(parts) {
return false
}
key := strings.ToLower(strings.ReplaceAll(strings.ReplaceAll(parts[len(parts)-1], "~1", "/"), "~0", "~"))
@@ -43,3 +64,13 @@ func secretProviderPath(path configkit.Path) bool {
return false
}
}
func providerExtraParamsPath(parts []string) bool {
if len(parts) >= 4 && parts[1] != "" &&
(parts[0] == "backends" || parts[0] == "profiles") && parts[2] == "extra_params" {
return true
}
return len(parts) >= 6 && parts[0] == "collections" && parts[1] != "" &&
(parts[2] == "backends" || parts[2] == "profiles") &&
parts[3] != "" && parts[4] == "extra_params"
}

View File

@@ -0,0 +1,32 @@
package configuration
import (
"testing"
"gitea.maximumdirect.net/eric/ConfigKit"
)
func TestSecretProviderPathRecognizesResourceExtraParams(t *testing.T) {
tests := []struct {
name string
path configkit.Path
want bool
}{
{name: "backend nested secret", path: configkit.Keys("backends", "provider", "extra_params", "nested", "Token"), want: true},
{name: "profile nested secret", path: configkit.Keys("profiles", "assistant", "extra_params", "authorization"), want: true},
{name: "collection backend nested secret", path: configkit.Keys("collections", "provider", "backends", "default", "extra_params", "nested", "Token"), want: true},
{name: "collection profile secret", path: configkit.Keys("collections", "assistant", "profiles", "model", "extra_params", "authorization"), want: true},
{name: "unrelated nested mapping", path: configkit.Keys("application", "backends", "provider", "extra_params", "token"), want: false},
{name: "extra params outside resource map", path: configkit.Keys("backends", "provider", "options", "extra_params", "token"), want: false},
{name: "collection extra params outside resource map", path: configkit.Keys("collections", "provider", "settings", "backends", "default", "extra_params", "token"), want: false},
{name: "missing resource key", path: configkit.Keys("backends", "", "extra_params", "token"), want: false},
{name: "ordinary provider value", path: configkit.Keys("profiles", "assistant", "extra_params", "provider", "mode"), want: false},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
if got := secretProviderPath(test.path); got != test.want {
t.Fatalf("secretProviderPath(%q) = %t, want %t", test.path, got, test.want)
}
})
}
}

View File

@@ -0,0 +1,87 @@
package configuration
import (
"context"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
)
// CollectionIdentity is the captured metadata for one final collection entry.
type CollectionIdentity struct {
Namespace string
ReleaseVersion *string
}
func (state *loaded) ListProfiles(ctx context.Context) ([]string, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
profiles := append([]string(nil), state.profileIDs...)
if err := ctx.Err(); err != nil {
return nil, err
}
return profiles, nil
}
func (state *loaded) ListBackends(ctx context.Context) ([]string, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
backends := append([]string(nil), state.backendIDs...)
if err := ctx.Err(); err != nil {
return nil, err
}
return backends, nil
}
func (state *loaded) ListNamespaces(ctx context.Context, kinds []string) ([]string, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
selected := kinds
if len(selected) == 0 {
selected = []string{"backend", "profile", "prompt"}
}
set := make(map[string]struct{})
for _, kind := range selected {
for _, namespace := range state.namespaces[kind] {
set[namespace] = struct{}{}
}
}
namespaces := sortedKeys(set)
if err := ctx.Err(); err != nil {
return nil, err
}
return namespaces, nil
}
func (state *loaded) ListCollections(ctx context.Context) ([]CollectionIdentity, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
collections := make([]CollectionIdentity, len(state.collections))
for index, collection := range state.collections {
collections[index] = CollectionIdentity{
Namespace: collection.Namespace,
ReleaseVersion: stringPointerValue(collection.ReleaseVersion),
}
}
if err := ctx.Err(); err != nil {
return nil, err
}
return collections, nil
}
func (state *loaded) ListPrompts(ctx context.Context) ([]domain.PromptIdentity, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
prompts := append([]domain.PromptIdentity(nil), state.promptIDs...)
for index := range prompts {
prompts[index].Version = stringPointerValue(prompts[index].Version)
}
if err := ctx.Err(); err != nil {
return nil, err
}
return prompts, nil
}

View File

@@ -0,0 +1,160 @@
package configuration
import (
"context"
"errors"
"reflect"
"testing"
"gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
)
func TestLoadedListsSortedEffectiveIdentities(t *testing.T) {
state, err := Load(mapping("identities", map[string]any{
"backends": map[string]any{
"local": map[string]any{"endpoint": "http://localhost:8080/v1"},
},
"collections": map[string]any{
"z": map[string]any{"profiles": map[string]any{"profile": map[string]any{"backend": "local", "model": "z-model"}}, "prompts": map[string]any{"prompt": listingPromptDefinition()}},
"partial": map[string]any{"profiles": map[string]any{"profile": map[string]any{"backend": "local"}}},
"a": map[string]any{"profiles": map[string]any{"profile": map[string]any{"backend": "local", "model": "a-model"}}, "prompts": map[string]any{"prompt": listingPromptDefinition()}},
" z": map[string]any{"profiles": map[string]any{"profile ": map[string]any{"backend": "local", "model": "spaced-model"}}, "prompts": map[string]any{"prompt ": listingPromptDefinition()}},
},
}), nil)
if err != nil {
t.Fatal(err)
}
backends, err := state.ListBackends(context.Background())
if err != nil {
t.Fatal(err)
}
wantBackends := []string{"local"}
if !reflect.DeepEqual(backends, wantBackends) {
t.Fatalf("backend IDs = %#v, want %#v", backends, wantBackends)
}
profiles, err := state.ListProfiles(context.Background())
if err != nil {
t.Fatal(err)
}
wantProfiles := []string{" z/profile ", "a/profile", "partial/profile", "z/profile"}
if !reflect.DeepEqual(profiles, wantProfiles) {
t.Fatalf("profile IDs = %#v, want %#v", profiles, wantProfiles)
}
prompts, err := state.ListPrompts(context.Background())
if err != nil {
t.Fatal(err)
}
wantPrompts := []domain.PromptIdentity{
{ID: " z/prompt ", Version: stringPointer("v1")},
{ID: "a/prompt", Version: stringPointer("v1")},
{ID: "z/prompt", Version: stringPointer("v1")},
}
if !reflect.DeepEqual(prompts, wantPrompts) {
t.Fatalf("prompt identities = %#v, want %#v", prompts, wantPrompts)
}
}
func TestLoadedListsAreIndependent(t *testing.T) {
state, err := Load(mapping("identities", map[string]any{
"backends": map[string]any{"backend": map[string]any{"endpoint": "https://example.test/v1"}},
"profiles": map[string]any{"profile": map[string]any{}},
"prompts": map[string]any{"example": listingPromptDefinition()},
}), nil)
if err != nil {
t.Fatal(err)
}
profiles, err := state.ListProfiles(context.Background())
if err != nil {
t.Fatal(err)
}
profiles = append(profiles, "caller profile")
profiles[0] = "changed profile"
backends, err := state.ListBackends(context.Background())
if err != nil {
t.Fatal(err)
}
backends[0] = "changed backend"
prompts, err := state.ListPrompts(context.Background())
if err != nil {
t.Fatal(err)
}
prompts[0].ID = "changed prompt"
*prompts[0].Version = "changed version"
profiles, err = state.ListProfiles(context.Background())
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(profiles, []string{"profile"}) {
t.Fatalf("caller mutation changed profile inventory: %#v", profiles)
}
backends, err = state.ListBackends(context.Background())
if err != nil || !reflect.DeepEqual(backends, []string{"backend"}) {
t.Fatalf("caller mutation changed backend inventory: %#v, %v", backends, err)
}
prompts, err = state.ListPrompts(context.Background())
if err != nil {
t.Fatal(err)
}
wantPrompts := []domain.PromptIdentity{{ID: "example", Version: stringPointer("v1")}}
if !reflect.DeepEqual(prompts, wantPrompts) {
t.Fatalf("caller mutation changed prompt inventory: %#v", prompts)
}
}
func TestLoadedListsEmptyInventory(t *testing.T) {
state, err := Load(configkit.Input{Source: configkit.NoSource()}, nil)
if err != nil {
t.Fatal(err)
}
profiles, err := state.ListProfiles(context.Background())
if err != nil || len(profiles) != 0 {
t.Fatalf("empty profile listing = %#v, %v", profiles, err)
}
backends, err := state.ListBackends(context.Background())
if err != nil || len(backends) != 0 {
t.Fatalf("empty backend listing = %#v, %v", backends, err)
}
prompts, err := state.ListPrompts(context.Background())
if err != nil || len(prompts) != 0 {
t.Fatalf("empty prompt listing = %#v, %v", prompts, err)
}
}
func TestLoadedListsHonorCanceledContext(t *testing.T) {
state, err := Load(mapping("identities", map[string]any{
"profiles": map[string]any{"partial": map[string]any{}},
"prompts": map[string]any{"example": listingPromptDefinition()},
}), nil)
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithCancel(context.Background())
cancel()
profiles, err := state.ListProfiles(ctx)
if !errors.Is(err, context.Canceled) || len(profiles) != 0 {
t.Fatalf("canceled profile listing = %#v, %v", profiles, err)
}
backends, err := state.ListBackends(ctx)
if !errors.Is(err, context.Canceled) || len(backends) != 0 {
t.Fatalf("canceled backend listing = %#v, %v", backends, err)
}
prompts, err := state.ListPrompts(ctx)
if !errors.Is(err, context.Canceled) || len(prompts) != 0 {
t.Fatalf("canceled prompt listing = %#v, %v", prompts, err)
}
}
func listingPromptDefinition() map[string]any {
return map[string]any{
"version": "v1",
"messages": []any{map[string]any{"role": "user", "content": "Hello."}},
"output": map[string]any{"format": "text", "validation_mode": "basic"},
}
}

View File

@@ -4,6 +4,7 @@ package configuration
import (
"fmt"
"net/http"
"regexp"
"sort"
"strings"
@@ -16,17 +17,42 @@ import (
)
type document struct {
Backends map[string]backendConfig `yaml:"backends"`
Profiles map[string]profileConfig `yaml:"profiles"`
Prompts map[string]map[string]promptConfig `yaml:"prompts"`
Version *string `yaml:"version"`
Backends map[string]backendConfig `yaml:"backends"`
Profiles map[string]profileConfig `yaml:"profiles"`
Prompts map[string]promptConfig `yaml:"prompts"`
Collections map[string]collectionConfig `yaml:"collections"`
}
type collectionConfig struct {
ReleaseVersion *string `yaml:"release_version"`
Backends map[string]backendConfig `yaml:"backends"`
Profiles map[string]profileConfig `yaml:"profiles"`
Prompts map[string]promptConfig `yaml:"prompts"`
}
type backendConfig struct {
Endpoint *string `yaml:"endpoint"`
APIKeyEnv *string `yaml:"api_key_env"`
ConcurrencyLimit *int `yaml:"concurrency_limit"`
QueueCapacity *int `yaml:"queue_capacity"`
ExtraParams map[string]any `yaml:"extra_params"`
Endpoint *string `yaml:"endpoint"`
APIKeyEnv *string `yaml:"api_key_env"`
ConcurrencyLimit *int `yaml:"concurrency_limit"`
QueueCapacity *int `yaml:"queue_capacity"`
ExtraParams map[string]any `yaml:"extra_params"`
SmokeTests *backendSmokeTestConfig `yaml:"smoke_tests"`
}
type backendSmokeTestConfig struct {
DefaultProfile *string `yaml:"default_profile"`
Reachability *reachabilityProbeConfig `yaml:"reachability"`
Authentication *authenticationProbeConfig `yaml:"authentication"`
}
type reachabilityProbeConfig struct {
Endpoint *string `yaml:"endpoint"`
}
type authenticationProbeConfig struct {
Kind *string `yaml:"kind"`
Endpoint *string `yaml:"endpoint"`
}
type profileConfig struct {
@@ -45,6 +71,7 @@ type profileConfig struct {
}
type promptConfig struct {
Version *string `yaml:"version"`
DefaultProfile *string `yaml:"default_profile"`
Description *string `yaml:"description"`
SessionID *string `yaml:"session_id"`
@@ -54,10 +81,10 @@ type promptConfig struct {
}
type promptInputConfig struct {
Name string `yaml:"name"`
Required bool `yaml:"required"`
ContentType string `yaml:"content_type"`
Description string `yaml:"description"`
Name string `yaml:"name"`
Required bool `yaml:"required"`
ContentType *string `yaml:"content_type"`
Description *string `yaml:"description"`
}
type promptMessageConfig struct {
@@ -93,15 +120,98 @@ type loaded struct {
result *configkit.Result[document]
backends map[string]map[string]fieldState
profiles map[string]map[string]fieldState
prompts map[string]map[string]*promptEntry
ambiguous map[string]bool
promptMetadata map[string][]promptInputPresence
prompts map[string]*promptEntry
registry *backend.Registry
indexedProfiles map[string]domain.ExecutionProfile
backendIDs []string
profileIDs []string
promptIDs []domain.PromptIdentity
namespaces map[string][]string
collections []CollectionIdentity
}
type resourceBases struct {
backends configkit.Path
profiles configkit.Path
prompts configkit.Path
}
type resourceSet struct {
isCollection bool
namespace string
bases resourceBases
backends map[string]backendConfig
profiles map[string]profileConfig
prompts map[string]promptConfig
}
func rootResourceBases() resourceBases {
return resourceBases{
backends: configkit.Keys("backends"),
profiles: configkit.Keys("profiles"),
prompts: configkit.Keys("prompts"),
}
}
func resourceSets(config *document) []resourceSet {
sets := make([]resourceSet, 0, len(config.Collections)+1)
sets = append(sets, resourceSet{
bases: rootResourceBases(), backends: config.Backends, profiles: config.Profiles, prompts: config.Prompts,
})
for _, namespace := range sortedKeys(config.Collections) {
collection := config.Collections[namespace]
base := configkit.Keys("collections", namespace)
sets = append(sets, resourceSet{
isCollection: true,
namespace: namespace,
bases: resourceBases{
backends: base.Key("backends"),
profiles: base.Key("profiles"),
prompts: base.Key("prompts"),
},
backends: collection.Backends,
profiles: collection.Profiles,
prompts: collection.Prompts,
})
}
return sets
}
func indexedResourceID(namespace, localKey string) string {
qualified, valid := domain.QualifyLocalResourceID(namespace, localKey)
if valid {
return qualified
}
if namespace == "" {
return localKey
}
return namespace + "/" + localKey
}
func indexedBackends(config *document) map[string]backendConfig {
backends := make(map[string]backendConfig)
for _, resources := range resourceSets(config) {
for localKey, definition := range resources.backends {
backends[indexedResourceID(resources.namespace, localKey)] = definition
}
}
return backends
}
func indexedProfiles(config *document) map[string]profileConfig {
profiles := make(map[string]profileConfig)
for _, resources := range resourceSets(config) {
for localKey, definition := range resources.profiles {
profiles[indexedResourceID(resources.namespace, localKey)] = definition
}
}
return profiles
}
var envName = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`)
var backendFields = []string{"endpoint", "api_key_env", "concurrency_limit", "queue_capacity", "extra_params"}
var backendFields = []string{"endpoint", "api_key_env", "concurrency_limit", "queue_capacity", "extra_params", "smoke_tests"}
var profileFields = []string{"backend", "endpoint", "model", "temperature", "top_p", "max_tokens", "timeout_seconds", "service_tier", "reasoning_effort", "api_key_env", "api_key_required", "extra_params"}
// Load captures one immutable effective configuration for internal consumers.
@@ -112,22 +222,33 @@ func Load(main configkit.Input, underlays []configkit.Underlay) (*loaded, error)
// load performs one ConfigKit lifecycle over the selected source tiers.
func load(main configkit.Input, underlays []configkit.Underlay) (*loaded, error) {
state := &loaded{
backends: make(map[string]map[string]fieldState),
profiles: make(map[string]map[string]fieldState),
backends: make(map[string]map[string]fieldState),
profiles: make(map[string]map[string]fieldState),
promptMetadata: make(map[string][]promptInputPresence),
}
result, err := configkit.LoadLayered(main, configkit.LayeredOptions[document]{
Underlays: underlays,
Underlays: underlays,
SourceVersionMode: configkit.SourceVersionModeIfPresent,
Definition: configkit.Definition[document]{
Version: &configkit.VersionRule{
Path: configkit.Keys("version"),
Check: checkConfigurationVersion,
},
Secret: secretProviderPath,
Resolve: func(config *document, resolution *configkit.Resolution) error {
captureFields(config.Backends, "backends", backendFields, resolution, state.backends)
captureFields(config.Profiles, "profiles", profileFields, resolution, state.profiles)
resolveBackends(config.Backends, resolution)
resolveProfiles(config.Profiles, resolution)
resolvePrompts(config.Prompts, resolution)
for _, resources := range resourceSets(config) {
captureFields(resources.backends, resources.bases.backends, resources.namespace, backendFields, resolution, state.backends)
captureFields(resources.profiles, resources.bases.profiles, resources.namespace, profileFields, resolution, state.profiles)
capturePromptMetadata(resources.prompts, resources.bases.prompts, resources.namespace, resolution, state.promptMetadata)
resolveBackends(resources.backends, resources.bases.backends, resolution)
resolveProfiles(resources.profiles, resources.bases.profiles, resolution)
resolvePrompts(resources.prompts, resources.bases.prompts, resolution)
}
return nil
},
Validate: validateEffective,
Validate: func(config *document, validation *configkit.Validation) {
validateEffective(config, validation)
},
},
})
if err != nil {
@@ -140,14 +261,91 @@ func load(main configkit.Input, underlays []configkit.Underlay) (*loaded, error)
if err := state.buildExecutionProfiles(); err != nil {
return nil, fmt.Errorf("load configuration: %w", err)
}
state.backendIDs = sortedKeys(indexedBackends(&result.Config))
state.profileIDs = sortedKeys(state.indexedProfiles)
for _, id := range sortedKeys(state.prompts) {
state.promptIDs = append(state.promptIDs, domain.PromptIdentity{ID: id, Version: stringPointerValue(state.prompts[id].definition.Version)})
}
state.namespaces = map[string][]string{
"backend": resourceNamespaces(state.backendIDs),
"profile": resourceNamespaces(state.profileIDs),
"prompt": promptNamespaces(state.promptIDs),
}
for _, namespace := range sortedKeys(result.Config.Collections) {
collection := result.Config.Collections[namespace]
state.collections = append(state.collections, CollectionIdentity{
Namespace: namespace, ReleaseVersion: stringPointerValue(collection.ReleaseVersion),
})
}
return state, nil
}
func captureFields[T any](entries map[string]T, collection string, fields []string, resolution *configkit.Resolution, destination map[string]map[string]fieldState) {
func resourceNamespaces(ids []string) []string {
namespaces := make(map[string]struct{})
for _, id := range ids {
if namespace, _, qualified := strings.Cut(id, "/"); qualified {
namespaces[namespace] = struct{}{}
}
}
return sortedKeys(namespaces)
}
func promptNamespaces(prompts []domain.PromptIdentity) []string {
ids := make([]string, len(prompts))
for index := range prompts {
ids[index] = prompts[index].ID
}
return resourceNamespaces(ids)
}
func checkConfigurationVersion(version configkit.Version, validation *configkit.Validation) error {
if !version.Present {
return nil
}
if version.Null || version.Text != "1" {
validation.Error(configkit.Keys("version"), "unsupported configuration specification version")
}
return nil
}
type promptInputPresence struct {
contentType fieldState
description fieldState
}
func capturePromptMetadata(prompts map[string]promptConfig, basePath configkit.Path, namespace string, resolution *configkit.Resolution, destination map[string][]promptInputPresence) {
for _, id := range sortedKeys(prompts) {
inputs := prompts[id].Inputs
states := make([]promptInputPresence, len(inputs))
base := basePath.Key(id).Key("inputs")
for index := range inputs {
path := base.Index(index)
states[index] = promptInputPresence{
contentType: captureFieldState(path.Key("content_type"), resolution),
description: captureFieldState(path.Key("description"), resolution),
}
}
destination[indexedResourceID(namespace, id)] = states
}
}
func captureFieldState(path configkit.Path, resolution *configkit.Resolution) fieldState {
info, exists := resolution.InputInfo(path)
switch {
case !exists || !info.Explicit:
return fieldAbsent
case info.Null:
return fieldNull
default:
return fieldValue
}
}
func captureFields[T any](entries map[string]T, basePath configkit.Path, namespace string, fields []string, resolution *configkit.Resolution, destination map[string]map[string]fieldState) {
for id := range entries {
states := make(map[string]fieldState, len(fields))
for _, field := range fields {
info, exists := resolution.InputInfo(configkit.Keys(collection, id, field))
info, exists := resolution.InputInfo(basePath.Key(id).Key(field))
switch {
case !exists || !info.Explicit:
states[field] = fieldAbsent
@@ -157,7 +355,7 @@ func captureFields[T any](entries map[string]T, collection string, fields []stri
states[field] = fieldValue
}
}
destination[id] = states
destination[indexedResourceID(namespace, id)] = states
}
}
@@ -178,10 +376,10 @@ func normalizeString(value **string, path configkit.Path, resolution *configkit.
return false
}
func resolveBackends(backends map[string]backendConfig, resolution *configkit.Resolution) {
func resolveBackends(backends map[string]backendConfig, basePath configkit.Path, resolution *configkit.Resolution) {
for _, id := range sortedKeys(backends) {
backend := backends[id]
base := configkit.Keys("backends", id)
base := basePath.Key(id)
if backend.Endpoint != nil {
endpoint, err := domain.NormalizeOpenAICompatibleBaseEndpoint(*backend.Endpoint)
if err != nil {
@@ -193,6 +391,21 @@ func resolveBackends(backends map[string]backendConfig, resolution *configkit.Re
if backend.APIKeyEnv != nil {
normalizeString(&backend.APIKeyEnv, base.Key("api_key_env"), resolution, strings.TrimSpace(*backend.APIKeyEnv))
}
if smokeTests := backend.SmokeTests; smokeTests != nil {
smokePath := base.Key("smoke_tests")
if smokeTests.Reachability != nil {
normalizeDiagnosticEndpoint(&smokeTests.Reachability.Endpoint, smokePath.Key("reachability").Key("endpoint"), resolution)
}
if smokeTests.Authentication != nil {
if smokeTests.Authentication.Kind != nil {
normalizeString(&smokeTests.Authentication.Kind,
smokePath.Key("authentication").Key("kind"), resolution,
strings.ToLower(strings.TrimSpace(*smokeTests.Authentication.Kind)))
}
normalizeDiagnosticEndpoint(&smokeTests.Authentication.Endpoint,
smokePath.Key("authentication").Key("endpoint"), resolution)
}
}
if backend.ConcurrencyLimit != nil && *backend.ConcurrencyLimit > 0 && backend.QueueCapacity == nil {
capacity := 1024
if resolution.IsExplicit(base.Key("queue_capacity")) {
@@ -205,10 +418,25 @@ func resolveBackends(backends map[string]backendConfig, resolution *configkit.Re
}
}
func resolveProfiles(profiles map[string]profileConfig, resolution *configkit.Resolution) {
func normalizeDiagnosticEndpoint(value **string, path configkit.Path, resolution *configkit.Resolution) {
if value == nil || *value == nil {
return
}
normalized := strings.TrimSpace(**value)
request, err := http.NewRequest(http.MethodGet, normalized, nil)
if normalized == "" || err != nil || request.URL == nil ||
(strings.ToLower(request.URL.Scheme) != "http" && strings.ToLower(request.URL.Scheme) != "https") ||
request.URL.Hostname() == "" {
resolution.Error(path, "diagnostic endpoint must be an absolute HTTP(S) URL with a host")
return
}
normalizeString(value, path, resolution, normalized)
}
func resolveProfiles(profiles map[string]profileConfig, basePath configkit.Path, resolution *configkit.Resolution) {
for _, id := range sortedKeys(profiles) {
profile := profiles[id]
base := configkit.Keys("profiles", id)
base := basePath.Key(id)
if profile.Endpoint != nil && *profile.Endpoint != "" {
endpoint, err := domain.NormalizeOpenAICompatibleBaseEndpoint(*profile.Endpoint)
if err != nil {
@@ -234,68 +462,110 @@ func resolveProfiles(profiles map[string]profileConfig, resolution *configkit.Re
}
func validateEffective(config *document, validation *configkit.Validation) {
for _, id := range sortedKeys(config.Backends) {
backend := config.Backends[id]
base := configkit.Keys("backends", id)
if strings.TrimSpace(id) == "" {
validation.Error(base, "backend ID must not be blank")
sets := resourceSets(config)
backendIDs := make(map[string]struct{})
for _, resources := range sets {
if resources.isCollection && !domain.IsValidResourceNamespace(resources.namespace) {
validation.Error(configkit.Keys("collections", resources.namespace), "collection namespace must be nonblank and contain no slash")
}
if backend.Endpoint == nil || *backend.Endpoint == "" {
validation.Error(base.Key("endpoint"), "endpoint is required")
for localKey := range resources.backends {
backendIDs[indexedResourceID(resources.namespace, localKey)] = struct{}{}
}
if backend.APIKeyEnv != nil && *backend.APIKeyEnv != "" && !envName.MatchString(*backend.APIKeyEnv) {
validation.Error(base.Key("api_key_env"), "environment variable name is invalid")
}
if backend.ConcurrencyLimit != nil && *backend.ConcurrencyLimit < 0 {
validation.Error(base.Key("concurrency_limit"), "concurrency limit must not be negative")
}
if backend.QueueCapacity != nil {
if *backend.QueueCapacity < 0 {
validation.Error(base.Key("queue_capacity"), "queue capacity must not be negative")
}
if backend.ConcurrencyLimit == nil || *backend.ConcurrencyLimit <= 0 {
validation.Error(base.Key("queue_capacity"), "queue capacity requires a positive concurrency limit")
} else if *backend.QueueCapacity > int(^uint(0)>>1)-*backend.ConcurrencyLimit {
validation.Error(base.Key("queue_capacity"), "total capacity overflows int")
}
}
validateExtraParams(backend.ExtraParams, base.Key("extra_params"), true, validation)
}
for _, id := range sortedKeys(config.Profiles) {
profile := config.Profiles[id]
base := configkit.Keys("profiles", id)
if strings.TrimSpace(id) == "" {
validation.Error(base, "profile ID must not be blank")
}
if profile.Backend != nil && *profile.Backend != "" {
if _, exists := config.Backends[*profile.Backend]; !exists {
validation.Error(base.Key("backend"), "backend is not defined")
for _, resources := range sets {
for _, id := range sortedKeys(resources.backends) {
backend := resources.backends[id]
base := resources.bases.backends.Key(id)
if !domain.IsValidLocalResourceKey(id) {
validation.Error(base, "backend key must be a nonblank local key without a slash")
}
if backend.Endpoint == nil || *backend.Endpoint == "" {
validation.Error(base.Key("endpoint"), "endpoint is required")
}
if backend.APIKeyEnv != nil && *backend.APIKeyEnv != "" && !envName.MatchString(*backend.APIKeyEnv) {
validation.Error(base.Key("api_key_env"), "environment variable name is invalid")
}
if backend.ConcurrencyLimit != nil && *backend.ConcurrencyLimit < 0 {
validation.Error(base.Key("concurrency_limit"), "concurrency limit must not be negative")
}
if backend.QueueCapacity != nil {
if *backend.QueueCapacity < 0 {
validation.Error(base.Key("queue_capacity"), "queue capacity must not be negative")
}
if backend.ConcurrencyLimit == nil || *backend.ConcurrencyLimit <= 0 {
validation.Error(base.Key("queue_capacity"), "queue capacity requires a positive concurrency limit")
} else if *backend.QueueCapacity > int(^uint(0)>>1)-*backend.ConcurrencyLimit {
validation.Error(base.Key("queue_capacity"), "total capacity overflows int")
}
}
validateExtraParams(backend.ExtraParams, base.Key("extra_params"), true, validation)
if smokeTests := backend.SmokeTests; smokeTests != nil {
smokePath := base.Key("smoke_tests")
if smokeTests.DefaultProfile != nil {
if strings.TrimSpace(*smokeTests.DefaultProfile) == "" {
validation.Error(smokePath.Key("default_profile"), "default profile must not be blank")
} else if !domain.IsValidResourceID(*smokeTests.DefaultProfile) {
validation.Error(smokePath.Key("default_profile"), "default profile reference is malformed")
}
}
if authentication := smokeTests.Authentication; authentication != nil {
kindPath := smokePath.Key("authentication").Key("kind")
if authentication.Kind == nil {
validation.Error(kindPath, "authentication kind is required")
} else if *authentication.Kind != "none" && *authentication.Kind != "openrouter_key" {
validation.Error(kindPath, "authentication kind must be none or openrouter_key")
}
}
}
}
if profile.Model != nil && *profile.Model != "" && strings.TrimSpace(*profile.Model) == "" {
validation.Error(base.Key("model"), "model must not be blank")
for _, id := range sortedKeys(resources.profiles) {
profile := resources.profiles[id]
base := resources.bases.profiles.Key(id)
if !domain.IsValidLocalResourceKey(id) {
validation.Error(base, "profile key must be a nonblank local key without a slash")
}
if profile.Backend != nil && *profile.Backend != "" {
if !domain.IsValidResourceID(*profile.Backend) {
validation.Error(base.Key("backend"), "backend reference is malformed")
} else if _, exists := backendIDs[*profile.Backend]; !exists {
validation.Error(base.Key("backend"), "backend is not defined")
}
}
if profile.Model != nil && *profile.Model != "" && strings.TrimSpace(*profile.Model) == "" {
validation.Error(base.Key("model"), "model must not be blank")
}
if profile.Temperature != nil {
if err := domain.ValidateExecutionTargetSettings(domain.ExecutionTarget{Temperature: *profile.Temperature}); err != nil {
validation.Error(base.Key("temperature"), err.Error())
}
}
if profile.TopP != nil {
if err := domain.ValidateExecutionTargetSettings(domain.ExecutionTarget{TopP: *profile.TopP}); err != nil {
validation.Error(base.Key("top_p"), err.Error())
}
}
if profile.MaxTokens != nil {
if err := domain.ValidateExecutionTargetSettings(domain.ExecutionTarget{MaxTokens: *profile.MaxTokens}); err != nil {
validation.Error(base.Key("max_tokens"), err.Error())
}
}
if profile.TimeoutSeconds != nil {
if err := domain.ValidateExecutionTargetSettings(domain.ExecutionTarget{TimeoutSeconds: *profile.TimeoutSeconds}); err != nil {
validation.Error(base.Key("timeout_seconds"), err.Error())
}
}
validateExtraParams(profile.ExtraParams, base.Key("extra_params"), false, validation)
}
if profile.Temperature != nil {
if err := domain.ValidateExecutionTargetSettings(domain.ExecutionTarget{Temperature: *profile.Temperature}); err != nil {
validation.Error(base.Key("temperature"), err.Error())
for _, id := range sortedKeys(resources.prompts) {
prompt := resources.prompts[id]
base := resources.bases.prompts.Key(id)
if !domain.IsValidLocalResourceKey(id) {
validation.Error(base, "prompt key must be a nonblank local key without a slash")
}
if prompt.DefaultProfile != nil && *prompt.DefaultProfile != "" && !domain.IsValidResourceID(*prompt.DefaultProfile) {
validation.Error(base.Key("default_profile"), "default profile reference is malformed")
}
}
if profile.TopP != nil {
if err := domain.ValidateExecutionTargetSettings(domain.ExecutionTarget{TopP: *profile.TopP}); err != nil {
validation.Error(base.Key("top_p"), err.Error())
}
}
if profile.MaxTokens != nil {
if err := domain.ValidateExecutionTargetSettings(domain.ExecutionTarget{MaxTokens: *profile.MaxTokens}); err != nil {
validation.Error(base.Key("max_tokens"), err.Error())
}
}
if profile.TimeoutSeconds != nil {
if err := domain.ValidateExecutionTargetSettings(domain.ExecutionTarget{TimeoutSeconds: *profile.TimeoutSeconds}); err != nil {
validation.Error(base.Key("timeout_seconds"), err.Error())
}
}
validateExtraParams(profile.ExtraParams, base.Key("extra_params"), false, validation)
}
}

View File

@@ -2,6 +2,7 @@ package configuration
import (
"errors"
"strings"
"testing"
"gitea.maximumdirect.net/eric/ConfigKit"
@@ -11,40 +12,128 @@ func mapping(label string, values map[string]any) configkit.Input {
return configkit.Input{Source: configkit.Mapping(label, values)}
}
func TestLoadChecksOptionalConfigurationVersion(t *testing.T) {
for _, test := range []struct {
name string
version any
include bool
valid bool
}{
{name: "string one", version: "1", include: true, valid: true},
{name: "integer one", version: int64(1), include: true, valid: true},
{name: "omitted", valid: true},
{name: "explicit null", include: true},
{name: "unsupported string", version: "2", include: true},
} {
t.Run(test.name, func(t *testing.T) {
values := map[string]any{}
if test.include {
values["version"] = test.version
}
loaded, err := load(mapping("main", values), nil)
if test.valid {
if err != nil {
t.Fatalf("load accepted version: %v", err)
}
if test.include && (loaded.result.Config.Version == nil || *loaded.result.Config.Version != "1") {
t.Fatalf("typed version = %v, want 1", loaded.result.Config.Version)
}
if !test.include && loaded.result.Config.Version != nil {
t.Fatalf("omitted version = %q, want nil", *loaded.result.Config.Version)
}
return
}
var loadErr *configkit.LoadError
if err == nil || !errors.As(err, &loadErr) {
t.Fatalf("load error = %v, want ConfigKit load error", err)
}
if strings.Contains(err.Error(), "2") {
t.Fatalf("load error disclosed the unsupported value: %v", err)
}
found := false
for _, diagnostic := range loadErr.Diagnostics() {
if diagnostic.Path.String() == "/version" && diagnostic.Message == "unsupported configuration specification version" {
found = true
}
}
if !found {
t.Fatalf("load diagnostics = %+v, want safe version diagnostic", loadErr.Diagnostics())
}
})
}
}
func TestLoadConsumesAcceptedUnderlayVersionAndValidatesHeaderlessPayload(t *testing.T) {
underlay := mapping("declared underlay", map[string]any{
"version": int64(1),
})
loaded, err := load(mapping("headerless main", map[string]any{}), []configkit.Underlay{{Inputs: []configkit.Input{underlay}}})
if err != nil {
t.Fatalf("load accepted non-main declaration: %v", err)
}
if loaded.result.Config.Version != nil {
t.Fatalf("consumed non-main declaration remained in effective payload: %q", *loaded.result.Config.Version)
}
invalidFragment := mapping("headerless prompt fragment", map[string]any{
"prompts": map[string]any{"incomplete": map[string]any{"description": "still validated"}},
})
if _, err := load(configkit.Input{Source: configkit.NoSource()}, []configkit.Underlay{{Inputs: []configkit.Input{invalidFragment}}}); err == nil || !strings.Contains(err.Error(), "at least one message is required") {
t.Fatalf("headerless fragment validation error = %v, want final prompt validation", err)
}
}
func TestLoadComposesLiteralKeysAndEffectivePresence(t *testing.T) {
base := mapping("defaults", map[string]any{
"version": int64(1),
"backends": map[string]any{
"local.v1": map[string]any{"endpoint": "http://localhost:8080/v1", "concurrency_limit": 2},
},
"profiles": map[string]any{
"base/one": map[string]any{"backend": "local.v1", "temperature": 0.8, "top_p": 0.9, "api_key_required": true,
"extra_params": map[string]any{"provider": map[string]any{"first": 1, "second": 2}}},
"child.v1": map[string]any{"extends": "base/one"},
},
"prompts": map[string]any{"invoice/summary": map[string]any{"v1.0": map[string]any{
"messages": []any{map[string]any{"role": "user", "content": "Summarize."}},
"output": map[string]any{"format": "text", "validation_mode": "basic"},
}}},
"collections": map[string]any{"base": map[string]any{
"release_version": "",
"backends": map[string]any{
"local.v1": map[string]any{"endpoint": "http://localhost:8080/v1", "concurrency_limit": 3},
},
"profiles": map[string]any{
"one": map[string]any{"backend": "local.v1", "temperature": 0.8, "top_p": 0.9, "api_key_required": true,
"extra_params": map[string]any{"provider": map[string]any{"first": 1, "second": 2}}},
"child.v1": map[string]any{"extends": "one"},
"namespaced": map[string]any{"backend": "base/local.v1", "model": "collection-model"},
},
}, "invoice": map[string]any{
"release_version": "r7",
"prompts": map[string]any{
"summary": map[string]any{
"version": "v1.0",
"messages": []any{map[string]any{"role": "user", "content": "Summarize."}},
"output": map[string]any{"format": "text", "validation_mode": "basic"},
},
"version": map[string]any{
"messages": []any{map[string]any{"role": "user", "content": "Nested ID."}},
"output": map[string]any{"format": "text", "validation_mode": "basic"},
},
},
}},
})
overlay := mapping("operator", map[string]any{
"backends": map[string]any{"local.v1": map[string]any{"queue_capacity": 0}},
"profiles": map[string]any{"child.v1": map[string]any{
"collections": map[string]any{"base": map[string]any{"profiles": map[string]any{"child.v1": map[string]any{
"temperature": 0, "top_p": nil, "api_key_required": false,
"extra_params": map[string]any{"provider": map[string]any{"second": 3}},
}},
}}}},
})
loaded, err := load(configkit.Input{Source: configkit.NoSource()}, []configkit.Underlay{
loaded, err := load(mapping("main", map[string]any{"version": "1"}), []configkit.Underlay{
{Inputs: []configkit.Input{base}}, {Inputs: []configkit.Input{overlay}},
})
if err != nil {
t.Fatal(err)
}
profile := loaded.result.Config.Profiles["child.v1"]
profile := loaded.result.Config.Collections["base"].Profiles["child.v1"]
if profile.Temperature == nil || *profile.Temperature != 0 || profile.TopP != nil || profile.APIKeyRequired == nil || *profile.APIKeyRequired {
t.Fatalf("effective values were not retained: %+v", profile)
}
if loaded.profiles["child.v1"]["temperature"] != fieldValue || loaded.profiles["child.v1"]["top_p"] != fieldNull || loaded.profiles["child.v1"]["api_key_required"] != fieldValue {
t.Fatalf("effective presence was lost: %+v", loaded.profiles["child.v1"])
if loaded.profiles["base/child.v1"]["temperature"] != fieldValue || loaded.profiles["base/child.v1"]["top_p"] != fieldNull || loaded.profiles["base/child.v1"]["api_key_required"] != fieldValue {
t.Fatalf("effective presence was lost: %+v", loaded.profiles["base/child.v1"])
}
provider := profile.ExtraParams["provider"].(map[string]any)
if provider["first"] != int64(1) || provider["second"] != int64(3) {
@@ -54,9 +143,69 @@ func TestLoadComposesLiteralKeysAndEffectivePresence(t *testing.T) {
if backend.QueueCapacity == nil || *backend.QueueCapacity != 0 || loaded.backends["local.v1"]["queue_capacity"] != fieldValue {
t.Fatalf("explicit zero queue capacity was lost: %+v", backend)
}
if loaded.result.Config.Prompts["invoice/summary"]["v1.0"].Messages[0].Content == nil {
if _, err := loaded.registry.GetBackend("base/local.v1"); err != nil {
t.Fatalf("collection backend was not indexed by its full ID: %v", err)
}
collectionProfile, exists := loaded.indexedProfiles["base/namespaced"]
if !exists || collectionProfile.ID != "base/namespaced" || collectionProfile.ResolvedTarget == nil || collectionProfile.ResolvedTarget.BackendID != "base/local.v1" {
t.Fatalf("collection profile snapshot = %+v, exists %t", collectionProfile, exists)
}
if loaded.result.Config.Collections["invoice"].Prompts["summary"].Messages[0].Content == nil || loaded.result.Config.Collections["invoice"].Prompts["summary"].Version == nil || *loaded.result.Config.Collections["invoice"].Prompts["summary"].Version != "v1.0" {
t.Fatal("literal prompt ID and version were not decoded")
}
if loaded.result.Config.Version == nil || *loaded.result.Config.Version != "1" ||
loaded.result.Config.Collections["base"].ReleaseVersion == nil || *loaded.result.Config.Collections["base"].ReleaseVersion != "" ||
loaded.result.Config.Collections["invoice"].ReleaseVersion == nil || *loaded.result.Config.Collections["invoice"].ReleaseVersion != "r7" {
t.Fatalf("root or collection version metadata was not retained: %+v", loaded.result.Config)
}
if _, _, err := loaded.prompt("invoice/version"); err != nil {
t.Fatalf("resource named version was not indexed under its qualified identity: %v", err)
}
}
func TestLoadRejectsInvalidCollectionAndResourceKeys(t *testing.T) {
for _, test := range []struct {
name string
values map[string]any
message string
}{
{
name: "namespace with separator",
values: map[string]any{"collections": map[string]any{"bad/name": map[string]any{}}},
message: "collection namespace must be nonblank and contain no slash",
},
{
name: "root resource with separator",
values: map[string]any{"backends": map[string]any{
"bad/name": map[string]any{"endpoint": "https://example.test/v1"},
}},
message: "backend key must be a nonblank local key without a slash",
},
{
name: "collection resource with separator",
values: map[string]any{"collections": map[string]any{"good": map[string]any{
"prompts": map[string]any{"bad/name": map[string]any{
"messages": []any{map[string]any{"role": "user", "content": "Hello."}},
"output": map[string]any{"format": "text", "validation_mode": "basic"},
}},
}}},
message: "prompt key must be a nonblank local key without a slash",
},
} {
t.Run(test.name, func(t *testing.T) {
_, err := load(mapping("invalid names", test.values), nil)
var loadErr *configkit.LoadError
if err == nil || !errors.As(err, &loadErr) {
t.Fatalf("load error = %v, want ConfigKit validation error", err)
}
for _, diagnostic := range loadErr.Diagnostics() {
if diagnostic.Message == test.message {
return
}
}
t.Fatalf("load diagnostics = %+v, want message %q", loadErr.Diagnostics(), test.message)
})
}
}
func TestLoadRecordsBackendNormalizationAndNullDefault(t *testing.T) {
@@ -78,6 +227,159 @@ func TestLoadRecordsBackendNormalizationAndNullDefault(t *testing.T) {
}
}
func TestLoadBackendSmokeTestsOmissionAndNull(t *testing.T) {
for _, test := range []struct {
name string
smokeTests any
include bool
inherited bool
}{
{name: "omitted"},
{name: "null", smokeTests: nil, include: true, inherited: true},
} {
t.Run(test.name, func(t *testing.T) {
backend := map[string]any{"endpoint": "https://backend.example/v1"}
if test.include {
backend["smoke_tests"] = test.smokeTests
}
main := mapping("backend", map[string]any{
"backends": map[string]any{"local": backend},
})
var underlays []configkit.Underlay
if test.inherited {
underlays = []configkit.Underlay{{Inputs: []configkit.Input{mapping("defaults", map[string]any{
"backends": map[string]any{"local": map[string]any{
"endpoint": "https://backend.example/v1",
"smoke_tests": map[string]any{"default_profile": "inherited-profile"},
}},
})}}}
}
loaded, err := load(main, underlays)
if err != nil {
t.Fatal(err)
}
if got := loaded.result.Config.Backends["local"].SmokeTests; got != nil {
t.Fatalf("smoke_tests = %+v, want nil", got)
}
selected, err := loaded.registry.GetBackend("local")
if err != nil {
t.Fatal(err)
}
if selected.SmokeTests != nil {
t.Fatalf("indexed smoke_tests = %+v, want nil", selected.SmokeTests)
}
})
}
}
func TestLoadMergesBackendSmokeTestsAndRetainsFallbacks(t *testing.T) {
base := mapping("defaults", map[string]any{
"backends": map[string]any{"local": map[string]any{
"endpoint": "https://backend.example/v1",
"smoke_tests": map[string]any{
"default_profile": "catalog-profile",
"reachability": map[string]any{"endpoint": " https://probe.example/ready?check=1 "},
"authentication": map[string]any{
"kind": " OPENROUTER_KEY ",
"endpoint": "https://probe.example/key?scope=current",
},
},
}},
})
overlay := mapping("operator", map[string]any{
"backends": map[string]any{"local": map[string]any{
"smoke_tests": map[string]any{
"default_profile": "not-yet-listed",
"authentication": map[string]any{"kind": " NONE ", "endpoint": nil},
},
}},
})
loaded, err := load(configkit.Input{Source: configkit.NoSource()}, []configkit.Underlay{
{Inputs: []configkit.Input{base}},
{Inputs: []configkit.Input{overlay}},
})
if err != nil {
t.Fatalf("load effective diagnostic metadata: %v", err)
}
selected, err := loaded.registry.GetBackend("local")
if err != nil {
t.Fatal(err)
}
metadata := selected.SmokeTests
if selected.Endpoint != "https://backend.example/v1" {
t.Fatalf("effective backend endpoint = %q", selected.Endpoint)
}
if metadata == nil || metadata.DefaultProfile == nil || *metadata.DefaultProfile != "not-yet-listed" {
t.Fatalf("default profile overlay was not retained: %+v", metadata)
}
if metadata.Reachability == nil || metadata.Reachability.Endpoint == nil ||
*metadata.Reachability.Endpoint != "https://probe.example/ready?check=1" {
t.Fatalf("full reachability destination was not normalized: %+v", metadata.Reachability)
}
if metadata.Authentication == nil || metadata.Authentication.Kind == nil || *metadata.Authentication.Kind != "none" ||
metadata.Authentication.Endpoint != nil {
t.Fatalf("authentication overlay or null fallback was not retained: %+v", metadata.Authentication)
}
}
func TestLoadRejectsInvalidFinalBackendSmokeTests(t *testing.T) {
for _, test := range []struct {
name string
data map[string]any
path string
}{
{
name: "blank default profile",
data: map[string]any{"default_profile": " \t "},
path: "/backends/local/smoke_tests/default_profile",
},
{
name: "authentication kind required",
data: map[string]any{"authentication": map[string]any{}},
path: "/backends/local/smoke_tests/authentication/kind",
},
{
name: "unknown authentication kind",
data: map[string]any{"authentication": map[string]any{"kind": "bearer"}},
path: "/backends/local/smoke_tests/authentication/kind",
},
{
name: "non-http reachability URL",
data: map[string]any{"reachability": map[string]any{"endpoint": "ftp://probe.example/ready"}},
path: "/backends/local/smoke_tests/reachability/endpoint",
},
{
name: "blank authentication URL",
data: map[string]any{"authentication": map[string]any{"kind": "none", "endpoint": " "}},
path: "/backends/local/smoke_tests/authentication/endpoint",
},
{
name: "relative authentication URL",
data: map[string]any{"authentication": map[string]any{"kind": "none", "endpoint": "/key"}},
path: "/backends/local/smoke_tests/authentication/endpoint",
},
} {
t.Run(test.name, func(t *testing.T) {
_, err := load(mapping("backend", map[string]any{
"backends": map[string]any{"local": map[string]any{
"endpoint": "https://backend.example/v1",
"smoke_tests": test.data,
}},
}), nil)
var loadError *configkit.LoadError
if !errors.As(err, &loadError) {
t.Fatalf("expected final-value validation error, got %v", err)
}
for _, diagnostic := range loadError.Diagnostics() {
if diagnostic.Path.String() == test.path {
return
}
}
t.Fatalf("expected diagnostic at %s, got %+v", test.path, loadError.Diagnostics())
})
}
}
func TestLoadDefaultsAbsentQueueCapacity(t *testing.T) {
loaded, err := load(mapping("backend", map[string]any{"backends": map[string]any{
"local": map[string]any{"endpoint": "http://localhost:8080/v1", "concurrency_limit": 1},
@@ -128,6 +430,40 @@ func TestLoadValidatesOnlyFinalDomainValues(t *testing.T) {
}
}
func TestLoadValidatesOnlyFinalBackendSmokeTestValues(t *testing.T) {
underlay := mapping("defaults", map[string]any{
"backends": map[string]any{"local": map[string]any{
"endpoint": "https://backend.example/v1",
"smoke_tests": map[string]any{
"default_profile": " ",
"reachability": map[string]any{"endpoint": "ftp://invalid.example/"},
"authentication": map[string]any{"kind": "unsupported"},
},
}},
})
main := mapping("operator", map[string]any{
"backends": map[string]any{"local": map[string]any{
"smoke_tests": map[string]any{
"default_profile": "dangling-profile",
"reachability": map[string]any{"endpoint": nil},
"authentication": map[string]any{"kind": "none"},
},
}},
})
loaded, err := load(main, []configkit.Underlay{{Inputs: []configkit.Input{underlay}}})
if err != nil {
t.Fatalf("replaced lower metadata should not fail: %v", err)
}
selected, err := loaded.registry.GetBackend("local")
if err != nil {
t.Fatal(err)
}
if selected.SmokeTests == nil || selected.SmokeTests.DefaultProfile == nil ||
*selected.SmokeTests.DefaultProfile != "dangling-profile" {
t.Fatalf("final dangling default profile was not retained: %+v", selected.SmokeTests)
}
}
func TestLoadKeepsSyntaxErrorsFromLowerTiers(t *testing.T) {
_, err := load(mapping("operator", map[string]any{"backends": map[string]any{"local": map[string]any{"endpoint": "http://localhost:8080"}}}),
[]configkit.Underlay{{Inputs: []configkit.Input{{Source: configkit.Bytes("broken", []byte("backends: ["))}}}})

View File

@@ -16,70 +16,73 @@ import (
)
var (
errPromptNotFound = errors.New("prompt definition not found")
errPromptAmbiguous = errors.New("prompt version is ambiguous")
errPromptNotFound = errors.New("prompt definition not found")
)
type promptEntry struct {
definition domain.PromptDefinition
inspection domain.PromptInspection
hash string
session *template.Template
messages []*template.Template
definition domain.PromptDefinition
inspection domain.PromptInspection
requirementInputs []prompt.InputMetadata
hash string
session *template.Template
messages []*template.Template
}
func resolvePrompts(prompts map[string]map[string]promptConfig, resolution *configkit.Resolution) {
func resolvePrompts(prompts map[string]promptConfig, basePath configkit.Path, resolution *configkit.Resolution) {
for _, id := range sortedKeys(prompts) {
for _, version := range sortedKeys(prompts[id]) {
definition := prompts[id][version]
base := configkit.Keys("prompts", id, version)
definition := prompts[id]
base := basePath.Key(id)
for _, field := range []struct {
name string
value **string
}{
{"description", &definition.Description},
{"session_id", &definition.SessionID},
} {
if *field.value != nil {
normalizeString(field.value, base.Key(field.name), resolution, strings.TrimSpace(**field.value))
}
}
inputsChanged := false
for index := range definition.Inputs {
input := &definition.Inputs[index]
inputPath := base.Key("inputs").Index(index)
normalizedName := strings.TrimSpace(input.Name)
if normalizedName != input.Name {
configkit.Normalize(resolution, inputPath.Key("name"), &input.Name, normalizedName, "normalize input metadata")
inputsChanged = true
}
for _, field := range []struct {
name string
value **string
}{
{"description", &definition.Description},
{"session_id", &definition.SessionID},
{"content_type", &input.ContentType},
{"description", &input.Description},
} {
if *field.value != nil {
normalizeString(field.value, base.Key(field.name), resolution, strings.TrimSpace(**field.value))
}
}
inputsChanged := false
for index := range definition.Inputs {
input := &definition.Inputs[index]
inputPath := base.Key("inputs").Index(index)
for _, field := range []struct {
name string
value *string
}{
{"name", &input.Name},
{"content_type", &input.ContentType},
{"description", &input.Description},
} {
normalized := strings.TrimSpace(*field.value)
if normalized != *field.value {
configkit.Normalize(resolution, inputPath.Key(field.name), field.value, normalized, "normalize input metadata")
normalized := strings.TrimSpace(**field.value)
if normalized != **field.value {
configkit.Normalize(resolution, inputPath.Key(field.name), field.value, &normalized, "normalize input metadata")
inputsChanged = true
}
}
}
if inputsChanged {
resolution.Record(base.Key("inputs"), configkit.ChangeNormalization, "normalize input metadata")
}
// Normalize message metadata on execution-owned copies below. ConfigKit
// records sequence mutations as whole-value replacements, which would
// discard the effective source contexts of unchanged content_file fields.
if definition.Output != nil && definition.Output.SchemaPath != nil {
normalizeString(&definition.Output.SchemaPath, base.Key("output").Key("schema_path"), resolution, strings.TrimSpace(*definition.Output.SchemaPath))
}
prompts[id][version] = definition
}
if inputsChanged {
resolution.Record(base.Key("inputs"), configkit.ChangeNormalization, "normalize input metadata")
}
// Normalize message metadata on execution-owned copies below. ConfigKit
// records sequence mutations as whole-value replacements, which would
// discard the effective source contexts of unchanged content_file fields.
if definition.Output != nil && definition.Output.SchemaPath != nil {
normalizeString(&definition.Output.SchemaPath, base.Key("output").Key("schema_path"), resolution, strings.TrimSpace(*definition.Output.SchemaPath))
}
prompts[id] = definition
}
}
func (state *loaded) materializePrompts() error {
state.prompts = make(map[string]map[string]*promptEntry, len(state.result.Config.Prompts))
state.ambiguous = make(map[string]bool)
state.prompts = make(map[string]*promptEntry)
var metadata *configkit.Metadata
lookupOrigin := func(path configkit.Path) (configkit.FieldInfo, bool) {
if metadata == nil {
@@ -88,36 +91,29 @@ func (state *loaded) materializePrompts() error {
}
return metadata.Lookup(path)
}
for _, id := range sortedKeys(state.result.Config.Prompts) {
if strings.TrimSpace(id) == "" {
return fmt.Errorf("%s: prompt ID must not be blank", configkit.Keys("prompts", id))
}
versions := state.result.Config.Prompts[id]
if versions == nil {
return fmt.Errorf("%s: prompt collection must be a mapping", configkit.Keys("prompts", id))
}
state.prompts[id] = make(map[string]*promptEntry, len(versions))
state.ambiguous[id] = len(versions) > 1
for _, version := range sortedKeys(versions) {
if strings.TrimSpace(version) == "" {
return fmt.Errorf("%s: prompt version must not be blank", configkit.Keys("prompts", id, version))
for _, resources := range resourceSets(&state.result.Config) {
for _, id := range sortedKeys(resources.prompts) {
promptBase := resources.bases.prompts.Key(id)
if strings.TrimSpace(id) == "" {
return fmt.Errorf("%s: prompt ID must not be blank", promptBase)
}
entry, err := state.materializePrompt(id, version, versions[version], lookupOrigin)
fullID := indexedResourceID(resources.namespace, id)
raw := resources.prompts[id]
entry, err := state.materializePrompt(fullID, raw.Version, raw, promptBase, lookupOrigin)
if err != nil {
return err
}
state.prompts[id][version] = entry
state.prompts[fullID] = entry
}
}
return nil
}
func (state *loaded) materializePrompt(id, version string, raw promptConfig, lookupOrigin func(configkit.Path) (configkit.FieldInfo, bool)) (*promptEntry, error) {
base := configkit.Keys("prompts", id, version)
func (state *loaded) materializePrompt(id string, version *string, raw promptConfig, base configkit.Path, lookupOrigin func(configkit.Path) (configkit.FieldInfo, bool)) (*promptEntry, error) {
if len(raw.Messages) == 0 {
return nil, fmt.Errorf("%s: at least one message is required", base.Key("messages"))
}
definition := domain.PromptDefinition{ID: id, Version: version}
definition := domain.PromptDefinition{ID: id, Version: stringPointerValue(version)}
if raw.DefaultProfile != nil {
if strings.TrimSpace(*raw.DefaultProfile) == "" {
return nil, fmt.Errorf("%s: default profile must not be blank", base.Key("default_profile"))
@@ -132,6 +128,10 @@ func (state *loaded) materializePrompt(id, version string, raw promptConfig, loo
}
seenInputs := make(map[string]bool, len(raw.Inputs))
definition.Inputs = make([]domain.PromptInput, 0, len(raw.Inputs))
entry := &promptEntry{
messages: make([]*template.Template, 0, len(raw.Messages)),
requirementInputs: make([]prompt.InputMetadata, 0, len(raw.Inputs)),
}
for index, input := range raw.Inputs {
path := base.Key("inputs").Index(index)
if input.Name == "" {
@@ -141,11 +141,27 @@ func (state *loaded) materializePrompt(id, version string, raw promptConfig, loo
return nil, fmt.Errorf("%s: input name is duplicated", path.Key("name"))
}
seenInputs[input.Name] = true
contentType := ""
if input.ContentType != nil {
contentType = *input.ContentType
}
description := ""
if input.Description != nil {
description = *input.Description
}
definition.Inputs = append(definition.Inputs, domain.PromptInput{
Name: input.Name, Required: input.Required, ContentType: input.ContentType, Description: input.Description,
Name: input.Name, Required: input.Required, ContentType: contentType, Description: description,
})
presence := state.promptMetadata[id][index]
metadata := prompt.InputMetadata{Name: input.Name, Required: input.Required}
if presence.contentType == fieldValue {
metadata.ContentType = stringPointer(contentType)
}
if presence.description == fieldValue {
metadata.Description = stringPointer(description)
}
entry.requirementInputs = append(entry.requirementInputs, metadata)
}
entry := &promptEntry{messages: make([]*template.Template, 0, len(raw.Messages))}
definition.Templates = make([]domain.PromptMessageTemplate, 0, len(raw.Messages))
for index, message := range raw.Messages {
path := base.Key("messages").Index(index)
@@ -226,13 +242,25 @@ func (state *loaded) materializePrompt(id, version string, raw promptConfig, loo
entry.definition = definition
entry.hash = hash
entry.inspection = domain.PromptInspection{
PromptID: id, PromptVersion: version, PromptHash: hash,
PromptID: id, PromptVersion: stringPointerValue(version), PromptHash: hash,
DefaultProfileID: definition.DefaultProfile, Inputs: append([]domain.PromptInput(nil), definition.Inputs...),
OutputContract: contract,
}
return entry, nil
}
func stringPointer(value string) *string {
copy := value
return &copy
}
func stringPointerValue(value *string) *string {
if value == nil {
return nil
}
return stringPointer(*value)
}
func (state *loaded) readContent(fieldPath configkit.Path, name string, lookupOrigin func(configkit.Path) (configkit.FieldInfo, bool)) (string, error) {
if strings.TrimSpace(name) == "" {
return "", fmt.Errorf("%s: content_file path must not be blank", fieldPath)
@@ -265,24 +293,19 @@ func (state *loaded) readContent(fieldPath configkit.Path, name string, lookupOr
return string(content), nil
}
func (state *loaded) prompt(id, version string) (*domain.PromptDefinition, domain.PromptInspection, error) {
if state == nil || strings.TrimSpace(id) == "" || (version != "" && strings.TrimSpace(version) == "") {
return nil, domain.PromptInspection{}, errPromptNotFound
func (state *loaded) lookupPromptEntry(id string) (*promptEntry, error) {
if state == nil || strings.TrimSpace(id) == "" {
return nil, errPromptNotFound
}
versions := state.prompts[id]
if version == "" {
if state.ambiguous[id] {
return nil, domain.PromptInspection{}, errPromptAmbiguous
}
for selected := range versions {
version = selected
}
}
entry := versions[version]
entry := state.prompts[id]
if entry == nil {
return nil, domain.PromptInspection{}, errPromptNotFound
return nil, errPromptNotFound
}
definition := entry.definition
return entry, nil
}
func ownPromptDefinition(definition domain.PromptDefinition) *domain.PromptDefinition {
definition.Version = stringPointerValue(definition.Version)
definition.Inputs = append([]domain.PromptInput(nil), definition.Inputs...)
definition.Templates = append([]domain.PromptMessageTemplate(nil), definition.Templates...)
for index := range definition.Templates {
@@ -291,41 +314,100 @@ func (state *loaded) prompt(id, version string) (*domain.PromptDefinition, domai
definition.Templates[index].CacheControl = &copy
}
}
inspection := entry.inspection
return &definition
}
func ownPromptInspection(inspection domain.PromptInspection) *domain.PromptInspection {
inspection.PromptVersion = stringPointerValue(inspection.PromptVersion)
inspection.Inputs = append([]domain.PromptInput(nil), inspection.Inputs...)
return &definition, inspection, nil
return &inspection
}
func promptLookupError(err error) error {
if errors.Is(err, errPromptNotFound) {
return promptdef.ErrPromptDefinitionNotFound
}
return err
}
func (state *loaded) prompt(id string) (*domain.PromptDefinition, domain.PromptInspection, error) {
entry, err := state.lookupPromptEntry(id)
if err != nil {
return nil, domain.PromptInspection{}, err
}
return ownPromptDefinition(entry.definition), *ownPromptInspection(entry.inspection), nil
}
// GetPromptSnapshot returns an independently owned captured definition and its
// construction-time hash. Lookup retains literal ID and version keys.
func (state *loaded) GetPromptSnapshot(ctx context.Context, id, version string) (*domain.PromptDefinition, string, error) {
// construction-time hash.
func (state *loaded) GetPromptSnapshot(ctx context.Context, id string) (*domain.PromptDefinition, string, error) {
if err := ctx.Err(); err != nil {
return nil, "", err
}
definition, inspection, err := state.prompt(id, version)
if errors.Is(err, errPromptNotFound) {
return nil, "", promptdef.ErrPromptDefinitionNotFound
}
entry, err := state.lookupPromptEntry(id)
if err != nil {
return nil, "", err
return nil, "", promptLookupError(err)
}
return definition, inspection.PromptHash, nil
return ownPromptDefinition(entry.definition), entry.hash, nil
}
// GetPromptInspection returns an independently owned inspection of one
// captured prompt without copying its execution templates.
func (state *loaded) GetPromptInspection(ctx context.Context, id string) (*domain.PromptInspection, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
entry, err := state.lookupPromptEntry(id)
if err != nil {
return nil, promptLookupError(err)
}
return ownPromptInspection(entry.inspection), nil
}
// GetPromptDefinition implements the runner's prompt lookup boundary.
func (state *loaded) GetPromptDefinition(ctx context.Context, id, version string) (*domain.PromptDefinition, error) {
definition, _, err := state.GetPromptSnapshot(ctx, id, version)
func (state *loaded) GetPromptDefinition(ctx context.Context, id string) (*domain.PromptDefinition, error) {
definition, _, err := state.GetPromptSnapshot(ctx, id)
return definition, err
}
// GetCompiledTemplates returns construction-time parses for one exact prompt.
func (state *loaded) GetCompiledTemplates(id, version string) (prompt.CompiledTemplates, bool) {
// GetCompiledTemplates returns construction-time parses for one prompt ID.
func (state *loaded) GetCompiledTemplates(id string) (prompt.CompiledTemplates, bool) {
if state == nil {
return prompt.CompiledTemplates{}, false
}
entry := state.prompts[id][version]
entry := state.prompts[id]
if entry == nil {
return prompt.CompiledTemplates{}, false
}
return prompt.CompiledTemplates{Session: entry.session, Messages: entry.messages}, true
}
// GetRequirementSnapshot returns captured templates and input metadata for
// one prompt ID. Returned metadata and slices are independently
// owned; the compiled templates are shared immutable construction-time parses.
func (state *loaded) GetRequirementSnapshot(id string) (prompt.RequirementSnapshot, bool) {
if state == nil {
return prompt.RequirementSnapshot{}, false
}
entry := state.prompts[id]
if entry == nil {
return prompt.RequirementSnapshot{}, false
}
snapshot := prompt.RequirementSnapshot{
Templates: prompt.CompiledTemplates{
Session: entry.session,
Messages: append([]*template.Template(nil), entry.messages...),
},
Inputs: make([]prompt.InputMetadata, len(entry.requirementInputs)),
}
for index, input := range entry.requirementInputs {
snapshot.Inputs[index] = input
if input.ContentType != nil {
snapshot.Inputs[index].ContentType = stringPointer(*input.ContentType)
}
if input.Description != nil {
snapshot.Inputs[index].Description = stringPointer(*input.Description)
}
}
return snapshot, true
}

View File

@@ -1,6 +1,7 @@
package configuration
import (
"context"
"errors"
"io/fs"
"os"
@@ -10,13 +11,16 @@ import (
"testing/fstest"
"gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/promptkit/internal/promptdef"
)
func TestLoadCapturesInheritedImportedContent(t *testing.T) {
files := fstest.MapFS{
"app/main.yaml": {Data: []byte("configkit:\n imports:\n - path: ../catalog/base.yaml\nprompts:\n child:\n extends: base\n")},
"catalog/base.yaml": {Data: []byte("prompts:\n base:\n v1:\n messages:\n - role: ' USER '\n cache_control:\n type: ' ephemeral '\n ttl: ' 1h '\n content_file: ./body.md\n output:\n format: text\n validation_mode: basic\n")},
"catalog/body.md": {Data: []byte("Captured {{.name}}")},
"app/main.yaml": {Data: []byte("configkit:\n imports:\n - path: ../catalog/base.yaml\n - path: ../other/definition.yaml\ncollections:\n docs:\n prompts:\n child:\n extends: base\n")},
"catalog/base.yaml": {Data: []byte("collections:\n docs:\n release_version: r1\n prompts:\n base:\n version: \"v1\"\n messages:\n - role: ' USER '\n cache_control:\n type: ' ephemeral '\n ttl: ' 1h '\n content_file: ./body.md\n output:\n format: text\n validation_mode: basic\n")},
"catalog/body.md": {Data: []byte("Captured {{.name}}")},
"other/definition.yaml": {Data: []byte("collections:\n other:\n prompts:\n child:\n messages:\n - role: user\n content_file: ./body.md\n output:\n format: text\n validation_mode: none\n")},
"other/body.md": {Data: []byte("Other collection {{.name}}")},
}
namespace := configkit.NewFilesystem("embedded", files)
state, err := load(configkit.Input{
@@ -30,7 +34,7 @@ func TestLoadCapturesInheritedImportedContent(t *testing.T) {
}
t.Fatal(err)
}
definition, inspection, err := state.prompt("child", "")
definition, inspection, err := state.prompt("docs/child")
if err != nil {
t.Fatal(err)
}
@@ -41,7 +45,7 @@ func TestLoadCapturesInheritedImportedContent(t *testing.T) {
if message.Role != "user" || message.CacheControl == nil || message.CacheControl.Type != "ephemeral" || message.CacheControl.TTL != "1h" {
t.Fatalf("message metadata was not normalized: %+v", message)
}
field, ok := state.result.Metadata().Lookup(configkit.Keys("prompts", "child", "v1", "messages").Index(0).Key("content_file"))
field, ok := state.result.Metadata().Lookup(configkit.Keys("collections", "docs", "prompts", "child", "messages").Index(0).Key("content_file"))
if !ok || len(field.Origin.Locations) != 1 {
t.Fatalf("content origin was lost during normalization: %+v", field)
}
@@ -54,16 +58,25 @@ func TestLoadCapturesInheritedImportedContent(t *testing.T) {
if originPath != "catalog/base.yaml" {
t.Fatalf("content origin = %q, want catalog/base.yaml", originPath)
}
other, otherInspection, err := state.prompt("other/child")
if err != nil || other.Templates[0].Content != "Other collection {{.name}}" {
t.Fatalf("other collection did not retain its source context: %+v, %v", other, err)
}
files["catalog/body.md"].Data = []byte("Changed")
again, againInspection, err := state.prompt("child", "v1")
files["other/body.md"].Data = []byte("Changed other collection")
again, againInspection, err := state.prompt("docs/child")
if err != nil || again.Templates[0].Content != "Captured {{.name}}" || againInspection.PromptHash != inspection.PromptHash {
t.Fatalf("captured definition changed after source mutation: %+v, %+v, %v", again, againInspection, err)
}
definition.Templates[0].Content = "caller mutation"
third, _, err := state.prompt("child", "v1")
third, _, err := state.prompt("docs/child")
if err != nil || third.Templates[0].Content != "Captured {{.name}}" {
t.Fatalf("caller mutation escaped into index: %+v, %v", third, err)
}
otherAgain, otherAgainInspection, err := state.prompt("other/child")
if err != nil || otherAgain.Templates[0].Content != "Other collection {{.name}}" || otherAgainInspection.PromptHash != otherInspection.PromptHash {
t.Fatalf("other captured definition changed after source mutation: %+v, %+v, %v", otherAgain, otherAgainInspection, err)
}
}
func TestLoadUsesExplicitMappingContextAndClosesContent(t *testing.T) {
@@ -79,12 +92,12 @@ func TestLoadUsesExplicitMappingContextAndClosesContent(t *testing.T) {
if err != nil {
t.Fatal(err)
}
definition, _, err := state.prompt("example", "v1")
definition, _, err := state.prompt("example")
if err != nil || definition.Templates[0].Content != "From mapping" || filesystem.contentOpened != 1 || filesystem.contentClosed != 1 {
t.Fatalf("mapping context or handle ownership failed: %+v, opens=%d closes=%d, %v", definition, filesystem.contentOpened, filesystem.contentClosed, err)
}
_, err = load(mapping("contextless", promptWithFile("body.md")), nil)
if err == nil || !strings.Contains(err.Error(), "/prompts/example/v1/messages/0/content_file") {
if err == nil || !strings.Contains(err.Error(), "/prompts/example/messages/0/content_file") {
t.Fatalf("expected semantic path for contextless resource, got %v", err)
}
}
@@ -93,7 +106,7 @@ func TestLoadCapturesNativeContentFile(t *testing.T) {
root := t.TempDir()
definitionPath := filepath.Join(root, "definition.yaml")
contentPath := filepath.Join(root, "body.md")
if err := os.WriteFile(definitionPath, []byte("prompts:\n example:\n v1:\n messages:\n - role: user\n content_file: body.md\n output:\n format: text\n validation_mode: basic\n"), 0o600); err != nil {
if err := os.WriteFile(definitionPath, []byte("prompts:\n example:\n version: \"v1\"\n messages:\n - role: user\n content_file: body.md\n output:\n format: text\n validation_mode: basic\n"), 0o600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(contentPath, []byte("Native body"), 0o600); err != nil {
@@ -106,41 +119,216 @@ func TestLoadCapturesNativeContentFile(t *testing.T) {
if err := os.WriteFile(contentPath, []byte("Modified"), 0o600); err != nil {
t.Fatal(err)
}
definition, _, err := state.prompt("example", "v1")
definition, _, err := state.prompt("example")
if err != nil || definition.Templates[0].Content != "Native body" {
t.Fatalf("native content was not captured: %+v, %v", definition, err)
}
}
func TestLoadRejectsMalformedStaticSyntaxAndAmbiguousSelection(t *testing.T) {
func TestLoadRejectsMalformedStaticSyntaxAndCapturesOptionalVersion(t *testing.T) {
_, err := load(mapping("bad-template", map[string]any{"prompts": map[string]any{"example": map[string]any{
"v1": map[string]any{"messages": []any{map[string]any{"role": "user", "content": "{{if}}"}},
"output": map[string]any{"format": "text", "validation_mode": "basic"}},
"messages": []any{map[string]any{"role": "user", "content": "{{if}}"}},
"output": map[string]any{"format": "text", "validation_mode": "basic"},
}}}), nil)
if err == nil || !strings.Contains(err.Error(), "invalid template syntax") {
t.Fatalf("expected construction-time syntax error, got %v", err)
}
state, err := load(mapping("versions", map[string]any{"prompts": map[string]any{"example": map[string]any{
"v1": map[string]any{"messages": []any{map[string]any{"role": "user", "content": "one"}},
"output": map[string]any{"format": "text", "validation_mode": "basic"}},
"v2": map[string]any{"messages": []any{map[string]any{"role": "user", "content": "two"}},
"output": map[string]any{"format": "text", "validation_mode": "basic"}},
state, err := load(mapping("version metadata", map[string]any{"prompts": map[string]any{"example": map[string]any{
"version": "release label",
"messages": []any{map[string]any{"role": "user", "content": "one"}},
"output": map[string]any{"format": "text", "validation_mode": "basic"},
}}}), nil)
if err != nil {
t.Fatal(err)
}
if _, _, err := state.prompt("example", ""); !errors.Is(err, errPromptAmbiguous) {
t.Fatalf("expected ambiguous version selection, got %v", err)
definition, inspection, err := state.prompt("example")
if err != nil || definition.Version == nil || *definition.Version != "release label" ||
inspection.PromptVersion == nil || *inspection.PromptVersion != "release label" {
t.Fatalf("optional version metadata was not retained: definition=%+v inspection=%+v err=%v", definition, inspection, err)
}
}
func TestPromptVersionLabelPresenceAndOwnership(t *testing.T) {
tests := []struct {
name string
version any
include bool
wantPresent bool
wantValue string
}{
{name: "absent"},
{name: "null", include: true},
{name: "empty", version: "", include: true, wantPresent: true},
{name: "exact text", version: " Release/v1 ", include: true, wantPresent: true, wantValue: " Release/v1 "},
}
hashes := make(map[string]string, len(tests))
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
definition := map[string]any{
"messages": []any{map[string]any{"role": "user", "content": "hello"}},
"output": map[string]any{"format": "text", "validation_mode": "basic"},
}
if test.include {
definition["version"] = test.version
}
state, err := load(mapping("prompt version label", map[string]any{
"prompts": map[string]any{"example": definition},
}), nil)
if err != nil {
t.Fatal(err)
}
captured, inspection, err := state.prompt("example")
if err != nil {
t.Fatal(err)
}
identities, err := state.ListPrompts(context.Background())
if err != nil || len(identities) != 1 {
t.Fatalf("list prompt labels: %#v, %v", identities, err)
}
check := func(name string, value *string) {
t.Helper()
if !test.wantPresent {
if value != nil {
t.Errorf("%s label = %q, want absent", name, *value)
}
return
}
if value == nil || *value != test.wantValue {
t.Errorf("%s label = %v, want %q", name, value, test.wantValue)
}
}
check("definition", captured.Version)
check("inspection", inspection.PromptVersion)
check("listing", identities[0].Version)
if captured.Version != nil && (captured.Version == inspection.PromptVersion || captured.Version == identities[0].Version || inspection.PromptVersion == identities[0].Version) {
t.Fatal("returned prompt labels share mutable pointers")
}
hashes[test.name] = inspection.PromptHash
if captured.Version != nil {
*captured.Version = "mutated definition"
}
if inspection.PromptVersion != nil {
*inspection.PromptVersion = "mutated inspection"
}
if identities[0].Version != nil {
*identities[0].Version = "mutated listing"
}
again, againInspection, err := state.prompt("example")
if err != nil {
t.Fatal(err)
}
check("captured definition after mutation", again.Version)
check("captured inspection after mutation", againInspection.PromptVersion)
})
}
if hashes["absent"] == hashes["empty"] {
t.Fatal("absent and declared-empty labels produced the same definition hash")
}
}
func TestPromptVersionNullClearsInheritedLabel(t *testing.T) {
prompt := map[string]any{
"version": "base-label",
"messages": []any{map[string]any{"role": "user", "content": "hello"}},
"output": map[string]any{"format": "text", "validation_mode": "basic"},
}
state, err := load(
mapping("clear inherited label", map[string]any{"prompts": map[string]any{"example": map[string]any{"version": nil}}}),
[]configkit.Underlay{{Inputs: []configkit.Input{mapping("base prompt", map[string]any{"prompts": map[string]any{"example": prompt}})}}},
)
if err != nil {
t.Fatal(err)
}
definition, inspection, err := state.prompt("example")
if err != nil || definition.Version != nil || inspection.PromptVersion != nil {
t.Fatalf("null overlay retained inherited label: definition=%+v inspection=%+v err=%v", definition, inspection, err)
}
}
func TestPromptInspectionLookupMatchesSnapshotSelectionAndOwnsInputs(t *testing.T) {
makePrompt := func(content string) map[string]any {
return map[string]any{
"default_profile": "partial",
"inputs": []any{map[string]any{
"name": "document", "required": true,
"content_type": " text/plain ", "description": " source document ",
}},
"messages": []any{map[string]any{"role": "user", "content": content}},
"output": map[string]any{"format": "text", "validation_mode": "none"},
}
}
selectedPrompt := makePrompt("selected")
selectedPrompt["version"] = "v2"
state, err := load(mapping("prompt inspection lookup", map[string]any{
"profiles": map[string]any{"partial": map[string]any{}},
"prompts": map[string]any{
"selected": selectedPrompt,
},
}), nil)
if err != nil {
t.Fatal(err)
}
{
inspection, err := state.GetPromptInspection(context.Background(), "selected")
if err != nil {
t.Fatalf("inspect prompt ID: %v", err)
}
definition, hash, err := state.GetPromptSnapshot(context.Background(), "selected")
if err != nil {
t.Fatalf("snapshot prompt ID: %v", err)
}
if inspection.PromptID != definition.ID || inspection.PromptVersion == nil || definition.Version == nil ||
*inspection.PromptVersion != *definition.Version ||
inspection.PromptHash != hash || inspection.DefaultProfileID != "partial" ||
len(inspection.Inputs) != 1 || inspection.Inputs[0].Name != "document" ||
inspection.Inputs[0].ContentType != "text/plain" || inspection.Inputs[0].Description != "source document" {
t.Fatalf("inspection did not match the selected snapshot: inspection=%+v definition=%+v hash=%q", inspection, definition, hash)
}
if len(definition.Inputs) != len(inspection.Inputs) || definition.Inputs[0].Name != inspection.Inputs[0].Name {
t.Fatalf("definition and inspection inputs diverged: definition=%+v inspection=%+v", definition.Inputs, inspection.Inputs)
}
}
owned, err := state.GetPromptInspection(context.Background(), "selected")
if err != nil {
t.Fatal(err)
}
owned.Inputs[0].Name = "caller mutation"
owned.Inputs[0].Description = "caller mutation"
again, err := state.GetPromptInspection(context.Background(), "selected")
if err != nil || again.Inputs[0].Name != "document" || again.Inputs[0].Description != "source document" {
t.Fatalf("inspection mutation escaped into captured state: inspection=%+v err=%v", again, err)
}
for _, id := range []string{"missing", " "} {
inspection, err := state.GetPromptInspection(context.Background(), id)
if inspection != nil || !errors.Is(err, promptdef.ErrPromptDefinitionNotFound) {
t.Errorf("inspection lookup %q=(%+v, %v), want not found", id, inspection, err)
}
definition, _, err := state.GetPromptSnapshot(context.Background(), id)
if definition != nil || !errors.Is(err, promptdef.ErrPromptDefinitionNotFound) {
t.Errorf("snapshot lookup %q=(%+v, %v), want not found", id, definition, err)
}
}
ctx, cancel := context.WithCancel(context.Background())
cancel()
if _, err := state.GetPromptInspection(ctx, "selected"); !errors.Is(err, context.Canceled) {
t.Errorf("canceled inspection selection error=%v", err)
}
}
func TestLoadAcceptsEmptyInlineContentAndRejectsInvalidPromptFields(t *testing.T) {
state, err := load(mapping("empty-content", map[string]any{"prompts": map[string]any{" exact.id ": map[string]any{"v1": map[string]any{
"inputs": []any{map[string]any{"name": " note ", "required": true}},
state, err := load(mapping("empty-content", map[string]any{"prompts": map[string]any{" exact.id ": map[string]any{
"version": "v1",
"inputs": []any{map[string]any{"name": " note ", "required": true}},
"messages": []any{map[string]any{"role": " USER ", "content": "", "content_file": nil,
"cache_control": map[string]any{"type": " ephemeral ", "ttl": "1h "}}},
"output": map[string]any{"format": "text", "validation_mode": "basic"},
}}}}), nil)
}}}), nil)
if err != nil {
var loadError *configkit.LoadError
if errors.As(err, &loadError) {
@@ -148,18 +336,19 @@ func TestLoadAcceptsEmptyInlineContentAndRejectsInvalidPromptFields(t *testing.T
}
t.Fatal(err)
}
definition, _, err := state.prompt(" exact.id ", "v1")
definition, _, err := state.prompt(" exact.id ")
if err != nil || definition.ID != " exact.id " || definition.Templates[0].Content != "" || definition.Templates[0].Role != "user" ||
definition.Inputs[0].Name != "note" || definition.Templates[0].CacheControl.TTL != "1h" {
t.Fatalf("explicit empty content or literal ID was changed: %+v, %v", definition, err)
}
if _, _, err := state.prompt("exact.id", "v1"); !errors.Is(err, errPromptNotFound) {
if _, _, err := state.prompt("exact.id"); !errors.Is(err, errPromptNotFound) {
t.Fatalf("trimmed prompt identity unexpectedly matched: %v", err)
}
_, err = load(mapping("obsolete-field", map[string]any{"prompts": map[string]any{"example": map[string]any{"v1": map[string]any{
"id": "example", "messages": []any{map[string]any{"role": "user", "content": "ok"}},
_, err = load(mapping("obsolete-field", map[string]any{"prompts": map[string]any{"example": map[string]any{
"version": "v1",
"id": "example", "messages": []any{map[string]any{"role": "user", "content": "ok"}},
"output": map[string]any{"format": "text", "validation_mode": "basic"},
}}}}), nil)
}}}), nil)
var loadError *configkit.LoadError
if !errors.As(err, &loadError) {
t.Fatalf("obsolete identity field should fail strict decoding: %v", err)
@@ -170,11 +359,117 @@ func TestLoadAcceptsEmptyInlineContentAndRejectsInvalidPromptFields(t *testing.T
}
}
func TestRequirementSnapshotCapturesMetadataPresenceAndNormalizedValues(t *testing.T) {
state, err := load(mapping("requirement-metadata", map[string]any{
"prompts": map[string]any{"example": map[string]any{
"version": "v1",
"session_id": "{{.session}}",
"inputs": []any{
map[string]any{"name": "absent"},
map[string]any{"name": "null", "content_type": nil, "description": nil},
map[string]any{"name": "empty", "content_type": "", "description": ""},
map[string]any{"name": "normalized", "content_type": " text/plain ", "description": " final text "},
},
"messages": []any{map[string]any{"role": "user", "content": "{{input \"absent\"}}"}},
"output": map[string]any{"format": "text", "validation_mode": "basic"},
}},
}), nil)
if err != nil {
var loadError *configkit.LoadError
if errors.As(err, &loadError) {
t.Fatalf("%v: %+v", err, loadError.Diagnostics())
}
t.Fatal(err)
}
snapshot, ok := state.GetRequirementSnapshot("example")
if !ok || len(snapshot.Inputs) != 4 || len(snapshot.Templates.Messages) != 1 || snapshot.Templates.Session == nil {
t.Fatalf("requirement snapshot was not captured: %+v, %t", snapshot, ok)
}
for _, index := range []int{0, 1} {
if snapshot.Inputs[index].ContentType != nil || snapshot.Inputs[index].Description != nil {
t.Errorf("input %q should not have explicit string metadata: %+v", snapshot.Inputs[index].Name, snapshot.Inputs[index])
}
}
if snapshot.Inputs[2].ContentType == nil || *snapshot.Inputs[2].ContentType != "" ||
snapshot.Inputs[2].Description == nil || *snapshot.Inputs[2].Description != "" {
t.Errorf("explicit empty metadata presence was lost: %+v", snapshot.Inputs[2])
}
if snapshot.Inputs[3].ContentType == nil || *snapshot.Inputs[3].ContentType != "text/plain" ||
snapshot.Inputs[3].Description == nil || *snapshot.Inputs[3].Description != "final text" {
t.Errorf("snapshot did not use normalized final metadata: %+v", snapshot.Inputs[3])
}
definition, inspection, err := state.prompt("example")
if err != nil {
t.Fatal(err)
}
if len(definition.Inputs) != 4 || len(inspection.Inputs) != 4 || inspection.Inputs[2].ContentType != "" || inspection.Inputs[2].Description != "" ||
inspection.Inputs[3].ContentType != "text/plain" || inspection.Inputs[3].Description != "final text" {
t.Fatalf("existing prompt inspection shape or normalized values changed: %+v", inspection)
}
*snapshot.Inputs[2].ContentType = "caller mutation"
snapshot.Inputs[3].Name = "caller mutation"
snapshot.Templates.Messages[0] = nil
again, ok := state.GetRequirementSnapshot("example")
if !ok || *again.Inputs[2].ContentType != "" || again.Inputs[3].Name != "normalized" || again.Templates.Messages[0] == nil {
t.Fatalf("snapshot mutation escaped into captured state: %+v, %t", again, ok)
}
}
func TestInputMetadataPresenceDoesNotChangePromptHashOrInspection(t *testing.T) {
variants := []struct {
name string
fields map[string]any
}{
{name: "absent"},
{name: "null", fields: map[string]any{"content_type": nil, "description": nil}},
{name: "empty", fields: map[string]any{"content_type": "", "description": ""}},
}
var expectedHash string
for _, variant := range variants {
t.Run(variant.name, func(t *testing.T) {
input := map[string]any{"name": "document"}
for name, value := range variant.fields {
input[name] = value
}
state, err := load(mapping("hash-"+variant.name, map[string]any{
"prompts": map[string]any{"same": map[string]any{
"version": "v1",
"inputs": []any{input},
"messages": []any{map[string]any{"role": "user", "content": "hello"}},
"output": map[string]any{"format": "text", "validation_mode": "basic"},
}},
}), nil)
if err != nil {
var loadError *configkit.LoadError
if errors.As(err, &loadError) {
t.Fatalf("%v: %+v", err, loadError.Diagnostics())
}
t.Fatal(err)
}
_, inspection, err := state.prompt("same")
if err != nil {
t.Fatal(err)
}
if len(inspection.Inputs) != 1 || inspection.Inputs[0].ContentType != "" || inspection.Inputs[0].Description != "" {
t.Fatalf("old inspection metadata changed: %+v", inspection.Inputs)
}
if expectedHash == "" {
expectedHash = inspection.PromptHash
} else if inspection.PromptHash != expectedHash {
t.Fatalf("metadata presence changed the definition hash: got %q, want %q", inspection.PromptHash, expectedHash)
}
})
}
}
func promptWithFile(name string) map[string]any {
return map[string]any{"prompts": map[string]any{"example": map[string]any{"v1": map[string]any{
return map[string]any{"prompts": map[string]any{"example": map[string]any{
"messages": []any{map[string]any{"role": "user", "content_file": name}},
"output": map[string]any{"format": "text", "validation_mode": "basic"},
}}}}
}}}
}
type countingFS struct {

View File

@@ -12,14 +12,25 @@ import (
)
func (state *loaded) buildExecutionProfiles() error {
definitions := make(map[string]domain.Backend, len(state.result.Config.Backends))
for _, id := range sortedKeys(state.result.Config.Backends) {
config := state.result.Config.Backends[id]
params, err := jsonvalue.CopyMap(config.ExtraParams)
if err != nil {
return fmt.Errorf("backend %q extra parameters: %w", id, err)
backendConfigs := indexedBackends(&state.result.Config)
profileConfigs := indexedProfiles(&state.result.Config)
definitions := make(map[string]domain.Backend, len(backendConfigs))
for _, id := range sortedKeys(backendConfigs) {
config := backendConfigs[id]
definition := domain.Backend{ID: id, ExtraParams: config.ExtraParams}
if config.SmokeTests != nil {
smokeTests := &domain.BackendSmokeTestConfig{DefaultProfile: config.SmokeTests.DefaultProfile}
if config.SmokeTests.Reachability != nil {
smokeTests.Reachability = &domain.ReachabilityProbeConfig{Endpoint: config.SmokeTests.Reachability.Endpoint}
}
if config.SmokeTests.Authentication != nil {
smokeTests.Authentication = &domain.AuthenticationProbeConfig{
Kind: config.SmokeTests.Authentication.Kind,
Endpoint: config.SmokeTests.Authentication.Endpoint,
}
}
definition.SmokeTests = smokeTests
}
definition := domain.Backend{ID: id, ExtraParams: params}
if config.Endpoint != nil {
definition.Endpoint = *config.Endpoint
}
@@ -40,9 +51,9 @@ func (state *loaded) buildExecutionProfiles() error {
return err
}
state.registry = registry
state.indexedProfiles = make(map[string]domain.ExecutionProfile, len(state.result.Config.Profiles))
for _, id := range sortedKeys(state.result.Config.Profiles) {
config := state.result.Config.Profiles[id]
state.indexedProfiles = make(map[string]domain.ExecutionProfile, len(profileConfigs))
for _, id := range sortedKeys(profileConfigs) {
config := profileConfigs[id]
fields := state.profiles[id]
target := defaults.ExecutionTargetDefault()
if config.Backend != nil && *config.Backend != "" {
@@ -147,18 +158,26 @@ func overlayProviderMap(lower, upper map[string]any) (map[string]any, error) {
return merged, nil
}
// GetProfile returns an independently owned indexed profile for the internal
// runner's lookup boundary.
func (state *loaded) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) {
func (state *loaded) lookupProfile(ctx context.Context, id string) (domain.ExecutionProfile, error) {
if err := ctx.Err(); err != nil {
return nil, err
return domain.ExecutionProfile{}, err
}
if state == nil {
return nil, fmt.Errorf("%w: %q", profilepkg.ErrProfileNotFound, id)
return domain.ExecutionProfile{}, fmt.Errorf("%w: %q", profilepkg.ErrProfileNotFound, id)
}
profile, found := state.indexedProfiles[id]
if !found {
return nil, fmt.Errorf("%w: %q", profilepkg.ErrProfileNotFound, id)
return domain.ExecutionProfile{}, fmt.Errorf("%w: %q", profilepkg.ErrProfileNotFound, id)
}
return profile, nil
}
// GetProfile returns an independently owned indexed profile for the internal
// runner's lookup boundary.
func (state *loaded) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) {
profile, err := state.lookupProfile(ctx, id)
if err != nil {
return nil, err
}
target := *profile.ResolvedTarget
params, err := jsonvalue.CopyMap(target.ExtraParams)
@@ -170,6 +189,15 @@ func (state *loaded) GetProfile(ctx context.Context, id string) (*domain.Executi
return &profile, nil
}
// GetProfileIdentity returns the exact lookup ID for a captured profile
// without copying its execution target.
func (state *loaded) GetProfileIdentity(ctx context.Context, id string) (string, error) {
if _, err := state.lookupProfile(ctx, id); err != nil {
return "", err
}
return id, nil
}
// GetBackend returns an independently owned effective backend definition.
func (state *loaded) GetBackend(id string) (domain.Backend, error) {
if state == nil {

View File

@@ -3,6 +3,7 @@ package configuration
import (
"context"
"encoding/json"
"errors"
"net/http"
"net/http/httptest"
"reflect"
@@ -11,6 +12,7 @@ import (
"gitea.maximumdirect.net/eric/promptkit/internal/defaults"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
"gitea.maximumdirect.net/eric/promptkit/internal/llm"
profilepkg "gitea.maximumdirect.net/eric/promptkit/internal/profile"
)
func TestLoadBuildsEffectiveTargetsAfterInheritance(t *testing.T) {
@@ -89,6 +91,18 @@ func TestLoadBuildsEffectiveTargetsAfterInheritance(t *testing.T) {
if partial.ResolvedTarget.Endpoint != "" || partial.ResolvedTarget.Model != "" {
t.Fatalf("partial profile gained a runnable target: %+v", partial.ResolvedTarget)
}
identity, err := state.GetProfileIdentity(context.Background(), "partial")
if err != nil || identity != "partial" {
t.Fatalf("partial profile identity=(%q, %v)", identity, err)
}
if _, err := state.GetProfileIdentity(context.Background(), "missing"); !errors.Is(err, profilepkg.ErrProfileNotFound) {
t.Fatalf("missing profile identity error=%v", err)
}
ctx, cancel := context.WithCancel(context.Background())
cancel()
if _, err := state.GetProfileIdentity(ctx, "partial"); !errors.Is(err, context.Canceled) {
t.Fatalf("canceled profile identity lookup error=%v", err)
}
endpointOnly, _ := state.GetProfile(context.Background(), "endpoint-only")
if endpointOnly.ResolvedTarget.Endpoint != "https://standalone.example/v1" ||
endpointOnly.ResolvedTarget.BackendID != "" || endpointOnly.ResolvedTarget.Model != "" {

View File

@@ -61,7 +61,6 @@ type CacheControl struct {
// RunRequest represents a request to generate a single artifact.
type RunRequest struct {
PromptID string
PromptVersion string
ProfileID string
SessionID string
APIKey string `json:"-" yaml:"-"`
@@ -81,7 +80,7 @@ type RunResult struct {
RawOutput string
Validation ValidationResult
PromptID string
PromptVersion string
PromptVersion *string
PromptHash string
SessionID string
RenderedPromptHash string
@@ -101,7 +100,7 @@ type RunResult struct {
// It must never include resolved API key values, model output, or validation data.
type PreparedRun struct {
PromptID string
PromptVersion string
PromptVersion *string
PromptHash string
SelectedProfileID string
SelectedBackendID string
@@ -135,10 +134,16 @@ type Artifact struct {
Hash string
}
// PromptIdentity identifies one configured prompt and its descriptive version label.
type PromptIdentity struct {
ID string
Version *string
}
// PromptDefinition represents a configured prompt execution definition.
type PromptDefinition struct {
ID string `yaml:"id"`
Version string `yaml:"version"`
Version *string `yaml:"version"`
DefaultProfile string `yaml:"default_profile"`
Description string `yaml:"description"`
SessionID string `yaml:"session_id" json:"session_id,omitempty"`
@@ -151,7 +156,7 @@ type PromptDefinition struct {
// PromptInspection is the resolved result of exact prompt inspection.
type PromptInspection struct {
PromptID string
PromptVersion string
PromptVersion *string
PromptHash string
DefaultProfileID string
Inputs []PromptInput
@@ -174,12 +179,14 @@ type PromptMessageTemplate struct {
CacheControl *CacheControl `yaml:"cache_control,omitempty" json:"cache_control,omitempty"`
}
// Backend describes reusable OpenAI-compatible connection defaults.
// Backend describes reusable OpenAI-compatible connection defaults and optional
// diagnostic metadata.
type Backend struct {
ID string
Endpoint string
APIKeyEnv string
ExtraParams map[string]any
SmokeTests *BackendSmokeTestConfig
ConcurrencyLimit int
QueueCapacity int
QueueCapacitySet bool

View File

@@ -68,7 +68,7 @@ type ExecutionEvent struct {
RunID string
Time time.Time
PromptID string
PromptVersion string
PromptVersion *string
ProfileID string
BackendID string
Model string

View File

@@ -0,0 +1,58 @@
package domain
// RequirementKind classifies how a prompt may depend on an input or variable.
type RequirementKind string
const (
RequirementRequired RequirementKind = "required"
RequirementConditional RequirementKind = "conditional"
RequirementOptional RequirementKind = "optional"
)
// InputRequirement describes one declared or template-referenced input.
type InputRequirement struct {
Name string
Kind RequirementKind
Declared bool
ContentType *string
Description *string
Explanation string
}
// VariableRequirement describes one caller variable referenced by a prompt.
type VariableRequirement struct {
Name string
Kind RequirementKind
Supplied bool
Explanation string
}
// RequirementIssueCode identifies a limit that may affect requirement
// discovery.
type RequirementIssueCode string
const (
RequirementIssueMissingVariable RequirementIssueCode = "missing_variable"
RequirementIssueInputContent RequirementIssueCode = "input_content"
RequirementIssueDynamicInputName RequirementIssueCode = "dynamic_input_name"
RequirementIssueUnsupportedConstruct RequirementIssueCode = "unsupported_construct"
)
// RequirementIssue describes one limitation on requirement discovery.
type RequirementIssue struct {
Code RequirementIssueCode
InputName string
VariableName string
Description string
}
// RequirementInspection contains the inventories discovered for one prompt.
type RequirementInspection struct {
PromptID string
PromptVersion *string
ProfileID string
Inputs []InputRequirement
Variables []VariableRequirement
Complete bool
Issues []RequirementIssue
}

View File

@@ -0,0 +1,70 @@
package domain
import "strings"
func validResourceID(id string) bool {
if !strings.Contains(id, "/") {
return strings.TrimSpace(id) != ""
}
if strings.Count(id, "/") != 1 {
return false
}
namespace, localID, _ := strings.Cut(id, "/")
return namespace != "" && localID != ""
}
func validResourceNamespace(namespace string) bool {
return strings.TrimSpace(namespace) != "" && !strings.Contains(namespace, "/")
}
func validLocalResourceKey(key string) bool {
return strings.TrimSpace(key) != "" && !strings.Contains(key, "/")
}
func qualifyResourceID(id, namespace string) (string, bool) {
if !validResourceID(id) || (namespace != "" && !validResourceNamespace(namespace)) {
return "", false
}
if strings.Contains(id, "/") || namespace == "" {
return id, true
}
return namespace + "/" + id, true
}
func resourceNamespaceFromID(id string) (string, bool) {
if !validResourceID(id) || !strings.Contains(id, "/") {
return "", false
}
namespace, _, _ := strings.Cut(id, "/")
return namespace, true
}
// IsValidResourceID reports whether id is an unqualified resource ID or one
// valid namespace/local-ID pair.
func IsValidResourceID(id string) bool { return validResourceID(id) }
// QualifyResourceID validates id and applies namespace to an unqualified ID.
// An already-qualified ID is returned unchanged.
func QualifyResourceID(id, namespace string) (string, bool) {
return qualifyResourceID(id, namespace)
}
// IsValidResourceNamespace reports whether namespace is nonblank and contains
// no namespace separator.
func IsValidResourceNamespace(namespace string) bool { return validResourceNamespace(namespace) }
// IsValidLocalResourceKey reports whether key is a nonblank key for a
// namespace-local resource mapping.
func IsValidLocalResourceKey(key string) bool { return validLocalResourceKey(key) }
// QualifyLocalResourceID combines a validated namespace and local mapping key.
// An empty namespace leaves a valid unqualified key unchanged.
func QualifyLocalResourceID(namespace, key string) (string, bool) {
if !validLocalResourceKey(key) || (namespace != "" && !validResourceNamespace(namespace)) {
return "", false
}
if namespace == "" {
return key, true
}
return namespace + "/" + key, true
}

View File

@@ -0,0 +1,110 @@
package domain
import "testing"
func TestValidResourceID(t *testing.T) {
tests := []struct {
name string
id string
want bool
}{
{name: "ordinary", id: "draft", want: true},
{name: "dotted", id: "group.prompt.v2", want: true},
{name: "preserved surrounding whitespace", id: " draft ", want: true},
{name: "qualified", id: "group/draft", want: true},
{name: "qualified components preserve spelling", id: " Group / draft ", want: true},
{name: "empty", id: "", want: false},
{name: "blank", id: " \t", want: false},
{name: "missing namespace", id: "/draft", want: false},
{name: "missing local id", id: "group/", want: false},
{name: "multiple separators", id: "group/subgroup/draft", want: false},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
if got := validResourceID(test.id); got != test.want {
t.Fatalf("validResourceID(%q) = %t, want %t", test.id, got, test.want)
}
})
}
}
func TestValidResourceNamespace(t *testing.T) {
tests := []struct {
name string
namespace string
want bool
}{
{name: "ordinary", namespace: "group", want: true},
{name: "dotted", namespace: "group.v2", want: true},
{name: "preserved whitespace", namespace: " group ", want: true},
{name: "empty", namespace: "", want: false},
{name: "blank", namespace: " \t", want: false},
{name: "slash", namespace: "group/subgroup", want: false},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
if got := validResourceNamespace(test.namespace); got != test.want {
t.Fatalf("validResourceNamespace(%q) = %t, want %t", test.namespace, got, test.want)
}
})
}
}
func TestValidLocalResourceKey(t *testing.T) {
if !validLocalResourceKey("draft.v2") || !validLocalResourceKey(" draft ") {
t.Fatal("expected nonblank slash-free keys to be valid")
}
for _, key := range []string{"", " \t", "group/draft"} {
if validLocalResourceKey(key) {
t.Errorf("validLocalResourceKey(%q) = true, want false", key)
}
}
}
func TestQualifyResourceID(t *testing.T) {
tests := []struct {
name string
id string
namespace string
want string
wantOK bool
}{
{name: "without scope", id: "draft", want: "draft", wantOK: true},
{name: "with scope", id: "draft", namespace: "group", want: "group/draft", wantOK: true},
{name: "qualified bypass", id: "other/draft", namespace: "group", want: "other/draft", wantOK: true},
{name: "preserve spelling", id: " draft ", namespace: " Group ", want: " Group / draft ", wantOK: true},
{name: "omission remains caller policy", id: "", wantOK: false},
{name: "malformed ID", id: "group/draft/extra", namespace: "group", wantOK: false},
{name: "malformed scope", id: "draft", namespace: "group/subgroup", wantOK: false},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
got, ok := qualifyResourceID(test.id, test.namespace)
if got != test.want || ok != test.wantOK {
t.Fatalf("qualifyResourceID(%q, %q) = (%q, %t), want (%q, %t)", test.id, test.namespace, got, ok, test.want, test.wantOK)
}
})
}
}
func TestResourceNamespaceFromID(t *testing.T) {
tests := []struct {
name string
id string
namespace string
wantOK bool
}{
{name: "qualified", id: "team/item", namespace: "team", wantOK: true},
{name: "same-prefix namespace stays distinct", id: "team-extra/item", namespace: "team-extra", wantOK: true},
{name: "unqualified", id: "item", wantOK: false},
{name: "malformed", id: "team/sub/item", wantOK: false},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
got, ok := resourceNamespaceFromID(test.id)
if got != test.namespace || ok != test.wantOK {
t.Fatalf("resourceNamespaceFromID(%q) = (%q, %t), want (%q, %t)", test.id, got, ok, test.namespace, test.wantOK)
}
})
}
}

View File

@@ -0,0 +1,263 @@
package domain
import (
"time"
"gitea.maximumdirect.net/eric/promptkit/internal/jsonvalue"
)
// BackendSmokeTestConfig describes optional backend-owned diagnostic defaults.
type BackendSmokeTestConfig struct {
DefaultProfile *string
Reachability *ReachabilityProbeConfig
Authentication *AuthenticationProbeConfig
}
// ReachabilityProbeConfig describes an optional endpoint for reachability tests.
type ReachabilityProbeConfig struct {
Endpoint *string
}
// AuthenticationProbeConfig describes an optional authentication diagnostic.
type AuthenticationProbeConfig struct {
Kind *string
Endpoint *string
}
// SmokeTestKind identifies one built-in diagnostic operation.
type SmokeTestKind string
const (
SmokeTestReachability SmokeTestKind = "reachability"
SmokeTestAuthentication SmokeTestKind = "authentication"
SmokeTestTextInference SmokeTestKind = "text_inference"
SmokeTestStructuredOutput SmokeTestKind = "structured_output"
)
// SmokeTestRequest selects one diagnostic and its execution target.
type SmokeTestRequest struct {
Kind SmokeTestKind
BackendID string
PromptID string
ProfileID string
Execution *ExecutionTargetOverride
APIKey string `json:"-" yaml:"-"`
TransientRetries *int
}
// SmokeTestReport contains factual diagnostic results, not generated content.
type SmokeTestReport struct {
Kind SmokeTestKind
Supported bool
Applicable *bool
Completed bool
Reason SmokeTestReason
Target SmokeTestTarget
Facts SmokeTestFacts
Timing SmokeTestTiming
Attempts []SmokeTestAttempt
Usage ObservedTokenUsage
Failure ExecutionFailureKind
}
// SmokeTestTarget identifies the selected diagnostic target and its settings.
type SmokeTestTarget struct {
BackendID string
ProfileID string
PromptID string
PromptVersion *string
DefaultSourceBackendID string
SelectionSource string
Endpoint string
EffectiveSettings ExecutionTarget
SettingsPresence ExecutionTargetPresence
}
// SmokeTestFacts contains optional factual observations from a diagnostic.
type SmokeTestFacts struct {
CredentialsSupplied *bool
EndpointReachable *bool
CredentialsAccepted *bool
ResponseReceived *bool
GenerationCompleted *bool
TextReturned *bool
ExpectedTextMatched *bool
StructuredOutputRequested *bool
ValidJSON *bool
SchemaMatched *bool
}
// SmokeTestTiming records elapsed diagnostic and phase durations.
type SmokeTestTiming struct {
StartedAt time.Time
Duration time.Duration
CapacityWait *time.Duration
RequestDuration *time.Duration
RetryDelay *time.Duration
EvaluationDuration *time.Duration
}
// SmokeTestAttempt records one dispatched attempt or an interrupted acquisition.
type SmokeTestAttempt struct {
Dispatched bool
InitialEndpoint string
Endpoint string
HTTPStatus *int
Facts SmokeTestFacts
Timing SmokeTestTiming
Failure ExecutionFailureKind
Usage ObservedTokenUsage
}
// SmokeTestReason identifies additional factual context for a diagnostic.
type SmokeTestReason string
const (
SmokeTestReasonNone SmokeTestReason = ""
SmokeTestReasonAuthenticationNotApplicable SmokeTestReason = "authentication_not_applicable"
SmokeTestReasonProbeNotConfigured SmokeTestReason = "probe_not_configured"
SmokeTestReasonHTTPDiagnosticsUnavailable SmokeTestReason = "http_diagnostics_unavailable"
SmokeTestReasonRequiredCredentialMissing SmokeTestReason = "required_credential_missing"
SmokeTestReasonAuthenticationResponseUnrecognized SmokeTestReason = "authentication_response_unrecognized"
)
// CloneBackend returns a copy of a backend, including its mutable settings.
func CloneBackend(backend Backend) (Backend, error) {
cloned := backend
var err error
cloned.ExtraParams, err = jsonvalue.CopyMap(backend.ExtraParams)
if err != nil {
return Backend{}, err
}
cloned.SmokeTests = CloneBackendSmokeTestConfig(backend.SmokeTests)
return cloned, nil
}
// CloneBackendSmokeTestConfig returns an independent copy of optional metadata.
func CloneBackendSmokeTestConfig(config *BackendSmokeTestConfig) *BackendSmokeTestConfig {
if config == nil {
return nil
}
cloned := *config
cloned.DefaultProfile = cloneString(config.DefaultProfile)
if config.Reachability != nil {
reachability := *config.Reachability
reachability.Endpoint = cloneString(config.Reachability.Endpoint)
cloned.Reachability = &reachability
}
if config.Authentication != nil {
authentication := *config.Authentication
authentication.Kind = cloneString(config.Authentication.Kind)
authentication.Endpoint = cloneString(config.Authentication.Endpoint)
cloned.Authentication = &authentication
}
return &cloned
}
// CloneSmokeTestRequest returns a copy that owns its optional settings.
func CloneSmokeTestRequest(request SmokeTestRequest) (SmokeTestRequest, error) {
cloned := request
cloned.TransientRetries = cloneInt(request.TransientRetries)
if request.Execution == nil {
return cloned, nil
}
execution := *request.Execution
execution.Temperature = cloneFloat64(request.Execution.Temperature)
execution.MaxTokens = cloneInt(request.Execution.MaxTokens)
execution.TopP = cloneFloat64(request.Execution.TopP)
execution.TimeoutSeconds = cloneInt(request.Execution.TimeoutSeconds)
execution.ReasoningEffort = cloneString(request.Execution.ReasoningEffort)
var err error
execution.ExtraParams, err = jsonvalue.CopyMap(request.Execution.ExtraParams)
if err != nil {
return SmokeTestRequest{}, err
}
cloned.Execution = &execution
return cloned, nil
}
// CloneSmokeTestReport returns a copy with independent optional facts and maps.
func CloneSmokeTestReport(report SmokeTestReport) (SmokeTestReport, error) {
cloned := report
cloned.Applicable = cloneBool(report.Applicable)
cloned.Target.PromptVersion = cloneString(report.Target.PromptVersion)
cloned.Facts = cloneSmokeTestFacts(report.Facts)
cloned.Timing = cloneSmokeTestTiming(report.Timing)
var err error
cloned.Target.EffectiveSettings.ExtraParams, err = jsonvalue.CopyMap(report.Target.EffectiveSettings.ExtraParams)
if err != nil {
return SmokeTestReport{}, err
}
if report.Attempts != nil {
cloned.Attempts = make([]SmokeTestAttempt, len(report.Attempts))
for index, attempt := range report.Attempts {
cloned.Attempts[index] = attempt
cloned.Attempts[index].HTTPStatus = cloneInt(attempt.HTTPStatus)
cloned.Attempts[index].Facts = cloneSmokeTestFacts(attempt.Facts)
cloned.Attempts[index].Timing = cloneSmokeTestTiming(attempt.Timing)
}
}
return cloned, nil
}
func cloneSmokeTestFacts(facts SmokeTestFacts) SmokeTestFacts {
facts.CredentialsSupplied = cloneBool(facts.CredentialsSupplied)
facts.EndpointReachable = cloneBool(facts.EndpointReachable)
facts.CredentialsAccepted = cloneBool(facts.CredentialsAccepted)
facts.ResponseReceived = cloneBool(facts.ResponseReceived)
facts.GenerationCompleted = cloneBool(facts.GenerationCompleted)
facts.TextReturned = cloneBool(facts.TextReturned)
facts.ExpectedTextMatched = cloneBool(facts.ExpectedTextMatched)
facts.StructuredOutputRequested = cloneBool(facts.StructuredOutputRequested)
facts.ValidJSON = cloneBool(facts.ValidJSON)
facts.SchemaMatched = cloneBool(facts.SchemaMatched)
return facts
}
func cloneSmokeTestTiming(timing SmokeTestTiming) SmokeTestTiming {
timing.CapacityWait = cloneDuration(timing.CapacityWait)
timing.RequestDuration = cloneDuration(timing.RequestDuration)
timing.RetryDelay = cloneDuration(timing.RetryDelay)
timing.EvaluationDuration = cloneDuration(timing.EvaluationDuration)
return timing
}
func cloneBool(value *bool) *bool {
if value == nil {
return nil
}
cloned := *value
return &cloned
}
func cloneDuration(value *time.Duration) *time.Duration {
if value == nil {
return nil
}
cloned := *value
return &cloned
}
func cloneFloat64(value *float64) *float64 {
if value == nil {
return nil
}
cloned := *value
return &cloned
}
func cloneInt(value *int) *int {
if value == nil {
return nil
}
cloned := *value
return &cloned
}
func cloneString(value *string) *string {
if value == nil {
return nil
}
cloned := *value
return &cloned
}

View File

@@ -0,0 +1,115 @@
package domain
import (
"testing"
"time"
)
func TestCloneBackendCopiesDiagnosticMetadataAndProviderValues(t *testing.T) {
profile := "profile"
reachabilityEndpoint := "https://probe.example/health"
authKind := "openrouter_key"
authEndpoint := "https://probe.example/key"
backend := Backend{
ExtraParams: map[string]any{"nested": map[string]any{"value": "original"}},
SmokeTests: &BackendSmokeTestConfig{
DefaultProfile: &profile,
Reachability: &ReachabilityProbeConfig{Endpoint: &reachabilityEndpoint},
Authentication: &AuthenticationProbeConfig{
Kind: &authKind,
Endpoint: &authEndpoint,
},
},
}
cloned, err := CloneBackend(backend)
if err != nil {
t.Fatal(err)
}
*cloned.SmokeTests.DefaultProfile = "changed"
*cloned.SmokeTests.Reachability.Endpoint = "changed"
*cloned.SmokeTests.Authentication.Kind = "changed"
*cloned.SmokeTests.Authentication.Endpoint = "changed"
cloned.ExtraParams["nested"].(map[string]any)["value"] = "changed"
if profile != "profile" || reachabilityEndpoint != "https://probe.example/health" ||
authKind != "openrouter_key" || authEndpoint != "https://probe.example/key" {
t.Fatal("cloned diagnostic metadata shared source pointers")
}
if got := backend.ExtraParams["nested"].(map[string]any)["value"]; got != "original" {
t.Fatalf("source provider parameter changed through clone: %v", got)
}
}
func TestCloneSmokeTestRequestOwnsOptionalSettings(t *testing.T) {
temperature := 0.25
retries := 2
reasoning := "low"
request := SmokeTestRequest{
APIKey: "secret",
TransientRetries: &retries,
Execution: &ExecutionTargetOverride{
Temperature: &temperature,
ReasoningEffort: &reasoning,
ExtraParams: map[string]any{"nested": map[string]any{"value": "original"}},
},
}
cloned, err := CloneSmokeTestRequest(request)
if err != nil {
t.Fatal(err)
}
*cloned.TransientRetries = 3
*cloned.Execution.Temperature = 0.5
*cloned.Execution.ReasoningEffort = "high"
cloned.Execution.ExtraParams["nested"].(map[string]any)["value"] = "changed"
if retries != 2 || temperature != 0.25 || reasoning != "low" {
t.Fatal("cloned request shared optional scalar pointers")
}
if got := request.Execution.ExtraParams["nested"].(map[string]any)["value"]; got != "original" {
t.Fatalf("source provider parameter changed through clone: %v", got)
}
}
func TestCloneSmokeTestReportOwnsMutableValues(t *testing.T) {
applicable := true
reachable := true
status := 200
wait := 3 * time.Millisecond
promptVersion := "v1"
report := SmokeTestReport{
Applicable: &applicable,
Target: SmokeTestTarget{
PromptVersion: &promptVersion,
EffectiveSettings: ExecutionTarget{
ExtraParams: map[string]any{"nested": map[string]any{"value": "original"}},
},
},
Facts: SmokeTestFacts{EndpointReachable: &reachable},
Attempts: []SmokeTestAttempt{{
HTTPStatus: &status,
Facts: SmokeTestFacts{EndpointReachable: &reachable},
Timing: SmokeTestTiming{CapacityWait: &wait},
}},
}
cloned, err := CloneSmokeTestReport(report)
if err != nil {
t.Fatal(err)
}
*cloned.Applicable = false
*cloned.Facts.EndpointReachable = false
*cloned.Attempts[0].HTTPStatus = 503
*cloned.Attempts[0].Facts.EndpointReachable = false
*cloned.Attempts[0].Timing.CapacityWait = time.Second
*cloned.Target.PromptVersion = "changed"
cloned.Target.EffectiveSettings.ExtraParams["nested"].(map[string]any)["value"] = "changed"
if !applicable || !reachable || status != 200 || wait != 3*time.Millisecond || promptVersion != "v1" {
t.Fatal("cloned report shared optional scalar pointers")
}
if got := report.Target.EffectiveSettings.ExtraParams["nested"].(map[string]any)["value"]; got != "original" {
t.Fatalf("source provider parameter changed through clone: %v", got)
}
}

View File

@@ -14,7 +14,6 @@ import (
"gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/promptkit/internal/artifact"
"gitea.maximumdirect.net/eric/promptkit/internal/capacity"
"gitea.maximumdirect.net/eric/promptkit/internal/catalog"
"gitea.maximumdirect.net/eric/promptkit/internal/configuration"
"gitea.maximumdirect.net/eric/promptkit/internal/defaults"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
@@ -81,8 +80,41 @@ func (o *Options) SetSchemaFile(path string) error {
// Engine forwards domain operations to the single assembled use-case runner.
type Engine struct {
runner *usecase.Runner
inspection configuration.Inspection
runner *usecase.Runner
smokeTester *usecase.SmokeTester
inspection configuration.Inspection
listing identityInventory
namespaces resourceNamespaces
}
type resourceNamespaces struct {
prompt string
profile string
backend string
}
type identityInventory interface {
ListBackends(context.Context) ([]string, error)
ListProfiles(context.Context) ([]string, error)
ListPrompts(context.Context) ([]domain.PromptIdentity, error)
ListNamespaces(context.Context, []string) ([]string, error)
ListCollections(context.Context) ([]configuration.CollectionIdentity, error)
}
// ListingFilterMode selects namespace membership for one identity listing.
type ListingFilterMode uint8
const (
ListingDefault ListingFilterMode = iota
ListingNamed
ListingUnqualified
ListingAll
)
// ListingFilter optionally overrides the namespace selected by an engine view.
type ListingFilter struct {
Mode ListingFilterMode
Name string
}
// New assembles an engine from facade-selected configuration and collaborators.
@@ -91,11 +123,7 @@ func New(cfg Config, options Options) (*Engine, error) {
if err != nil {
return nil, err
}
underlays, err := catalog.Underlays()
if err != nil {
return nil, fmt.Errorf("select maintained configuration: %w", err)
}
underlays = append(underlays, cfg.Sources.Underlays...)
underlays := cfg.Sources.Underlays
main := configkit.Input{Source: configkit.NoSource()}
if cfg.Sources.Main != nil {
main = *cfg.Sources.Main
@@ -118,11 +146,13 @@ func New(cfg Config, options Options) (*Engine, error) {
validator = validate.NewStandardValidator(schemaDir)
}
var client llm.Client
var diagnosticHTTPClient *llm.OpenAICompatibleClient
if options.llmClient == nil {
builtIn, newErr := llm.NewOpenAICompatibleClient(llm.OpenAICompatibleConfig{Timeout: cfg.Timeout, HTTPClient: cfg.HTTPClient})
if newErr != nil {
return nil, newErr
}
diagnosticHTTPClient = builtIn
client = generation.NewBuiltIn(manager, builtIn)
} else {
client = generation.NewInjected(manager, options.llmClient)
@@ -132,12 +162,25 @@ func New(cfg Config, options Options) (*Engine, error) {
reader = artifact.NewCompositeReader()
}
runner := usecase.NewRunnerWithRepairer(snapshot, snapshot, reader, prompt.NewCompiledRenderer(snapshot), client, validator, usecase.NewDefaultOutputRepairer(client), manager, defaultRetries)
return &Engine{
runner: usecase.NewRunnerWithRepairer(snapshot, snapshot, reader, prompt.NewCompiledRenderer(snapshot), client, validator, usecase.NewDefaultOutputRepairer(client), manager, defaultRetries),
inspection: snapshot.Inspection(),
runner: runner,
smokeTester: usecase.NewSmokeTester(runner, diagnosticHTTPClient),
inspection: snapshot.Inspection(),
listing: snapshot,
}, nil
}
// WithNamespaces returns an immutable view sharing this engine's assembled
// collaborators and configuration snapshot. Caller-supplied selection IDs are
// qualified at each operation boundary; captured configuration references are
// not rewritten.
func (e *Engine) WithNamespaces(prompt, profile, backend string) *Engine {
view := *e
view.namespaces = resourceNamespaces{prompt: prompt, profile: profile, backend: backend}
return &view
}
// InspectConfiguration returns read-only operations over the captured result.
func (e *Engine) InspectConfiguration() configuration.Inspection {
if e == nil {
@@ -175,21 +218,213 @@ func fileSource(name string) (fs.FS, string, string, error) {
return os.DirFS(directory), filepath.ToSlash(base), directory, nil
}
func (e *Engine) InspectPrompt(ctx context.Context, id, version string) (*domain.PromptInspection, error) {
return e.runner.InspectPrompt(ctx, id, version)
func (e *Engine) InspectPrompt(ctx context.Context, id string) (*domain.PromptInspection, error) {
var err error
id, err = e.qualifyResourceID(id, e.namespaces.prompt, "prompt")
if err != nil {
return nil, err
}
return e.runner.InspectPrompt(ctx, id)
}
func (e *Engine) InspectRequirements(
ctx context.Context,
request usecase.RequirementInspectionRequest,
) (*domain.RequirementInspection, error) {
var err error
request.PromptID, err = e.qualifyResourceID(request.PromptID, e.namespaces.prompt, "prompt")
if err != nil {
return nil, err
}
request.ProfileID, err = e.qualifyResourceID(request.ProfileID, e.namespaces.profile, "profile")
if err != nil {
return nil, err
}
return e.runner.InspectRequirements(ctx, request)
}
func (e *Engine) InspectProfile(ctx context.Context, id string) (*domain.ProfileInspection, error) {
var err error
id, err = e.qualifyResourceID(id, e.namespaces.profile, "profile")
if err != nil {
return nil, err
}
return e.runner.InspectProfile(ctx, id)
}
func (e *Engine) ListBackends(ctx context.Context, filters ...ListingFilter) ([]string, error) {
return filterIdentityList(ctx, filters, e.namespaces.backend, e.listing.ListBackends)
}
func (e *Engine) ListProfiles(ctx context.Context, filters ...ListingFilter) ([]string, error) {
return filterIdentityList(ctx, filters, e.namespaces.profile, e.listing.ListProfiles)
}
func (e *Engine) ListPrompts(ctx context.Context, filters ...ListingFilter) ([]domain.PromptIdentity, error) {
mode, namespace, err := resolveListingFilter(filters, e.namespaces.prompt)
if err != nil {
return nil, err
}
prompts, err := e.listing.ListPrompts(ctx)
if err != nil {
return nil, err
}
var filtered []domain.PromptIdentity
for _, prompt := range prompts {
if matchesListingNamespace(prompt.ID, mode, namespace) {
filtered = append(filtered, prompt)
}
}
if err := ctx.Err(); err != nil {
return nil, err
}
return filtered, nil
}
func (e *Engine) ListNamespaces(ctx context.Context, kinds []string) ([]string, error) {
for _, kind := range kinds {
switch kind {
case "prompt", "profile", "backend":
default:
return nil, fmt.Errorf("%w: unknown resource kind %q", usecase.ErrInvalidRequest, kind)
}
}
return e.listing.ListNamespaces(ctx, kinds)
}
func (e *Engine) ListCollections(ctx context.Context) ([]configuration.CollectionIdentity, error) {
return e.listing.ListCollections(ctx)
}
func filterIdentityList(
ctx context.Context,
filters []ListingFilter,
scope string,
list func(context.Context) ([]string, error),
) ([]string, error) {
mode, namespace, err := resolveListingFilter(filters, scope)
if err != nil {
return nil, err
}
ids, err := list(ctx)
if err != nil {
return nil, err
}
var filtered []string
for _, id := range ids {
if matchesListingNamespace(id, mode, namespace) {
filtered = append(filtered, id)
}
}
if err := ctx.Err(); err != nil {
return nil, err
}
return filtered, nil
}
func resolveListingFilter(filters []ListingFilter, scope string) (ListingFilterMode, string, error) {
if len(filters) > 1 {
return 0, "", fmt.Errorf("%w: at most one listing filter is allowed", usecase.ErrInvalidRequest)
}
filter := ListingFilter{}
if len(filters) == 1 {
filter = filters[0]
}
switch filter.Mode {
case ListingDefault:
if scope == "" {
return ListingAll, "", nil
}
return ListingNamed, scope, nil
case ListingNamed:
if !domain.IsValidResourceNamespace(filter.Name) {
return 0, "", fmt.Errorf("%w: invalid listing namespace", usecase.ErrInvalidRequest)
}
return ListingNamed, filter.Name, nil
case ListingUnqualified, ListingAll:
return filter.Mode, "", nil
default:
return 0, "", fmt.Errorf("%w: unknown listing filter mode", usecase.ErrInvalidRequest)
}
}
func matchesListingNamespace(id string, mode ListingFilterMode, namespace string) bool {
switch mode {
case ListingNamed:
idNamespace, _, qualified := strings.Cut(id, "/")
return qualified && idNamespace == namespace
case ListingUnqualified:
return !strings.Contains(id, "/")
case ListingAll:
return true
default:
return false
}
}
func (e *Engine) Prepare(ctx context.Context, request domain.RunRequest) (*domain.PreparedRun, error) {
request, err := e.qualifyRunRequest(request)
if err != nil {
return nil, err
}
return e.runner.Prepare(ctx, request)
}
func (e *Engine) PrepareExecution(ctx context.Context, request domain.RunRequest) (*usecase.PreparedExecution, error) {
request, err := e.qualifyRunRequest(request)
if err != nil {
return nil, err
}
return e.runner.PrepareExecution(ctx, request)
}
func (e *Engine) Run(ctx context.Context, request domain.RunRequest) (*domain.RunResult, error) {
request, err := e.qualifyRunRequest(request)
if err != nil {
return nil, err
}
return e.runner.Run(ctx, request)
}
func (e *Engine) qualifyRunRequest(request domain.RunRequest) (domain.RunRequest, error) {
var err error
request.PromptID, err = e.qualifyResourceID(request.PromptID, e.namespaces.prompt, "prompt")
if err != nil {
return domain.RunRequest{}, err
}
request.ProfileID, err = e.qualifyResourceID(request.ProfileID, e.namespaces.profile, "profile")
if err != nil {
return domain.RunRequest{}, err
}
return request, nil
}
func (e *Engine) qualifyResourceID(id, namespace, kind string) (string, error) {
if id == "" {
return "", nil
}
qualified, ok := domain.QualifyResourceID(id, namespace)
if !ok {
return "", fmt.Errorf("%w: invalid %s ID", usecase.ErrInvalidRequest, kind)
}
return qualified, nil
}
func (e *Engine) RunPrepared(ctx context.Context, prepared *usecase.PreparedExecution) (*domain.RunResult, error) {
return e.runner.RunPrepared(ctx, prepared)
}
func (e *Engine) SmokeTest(ctx context.Context, request domain.SmokeTestRequest) (*domain.SmokeTestReport, error) {
if e == nil || e.smokeTester == nil {
return nil, fmt.Errorf("%w: smoke tester is not configured", usecase.ErrInvalidRequest)
}
var err error
request.PromptID, err = e.qualifyResourceID(request.PromptID, e.namespaces.prompt, "prompt")
if err != nil {
return nil, err
}
request.ProfileID, err = e.qualifyResourceID(request.ProfileID, e.namespaces.profile, "profile")
if err != nil {
return nil, err
}
request.BackendID, err = e.qualifyResourceID(request.BackendID, e.namespaces.backend, "backend")
if err != nil {
return nil, err
}
return e.smokeTester.Run(ctx, request)
}

View File

@@ -0,0 +1,124 @@
package engine
import (
"context"
"io"
"net/http"
"strings"
"testing"
"gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
)
func engineSmokeBackendInput() configkit.Input {
return configkit.Input{Source: configkit.Mapping("smoke-test backend", map[string]any{
"backends": map[string]any{"test-backend": map[string]any{
"endpoint": "https://test-backend.example/v1",
"smoke_tests": map[string]any{
"default_profile": "test-profile",
"authentication": map[string]any{"kind": "none"},
},
}},
"profiles": map[string]any{"test-profile": map[string]any{
"backend": "test-backend",
"model": "synthetic-model",
}},
})}
}
func TestEngineSmokeTestUsesInjectedGenerationWithoutHTTPSupport(t *testing.T) {
calls := 0
options := Options{}
options.SetLLMClient(engineSmokeClientFunc(func(_ context.Context, request domain.GenerateRequest) (*domain.GenerateResponse, error) {
calls++
if len(request.Prompt.Messages) != 1 || request.Prompt.Messages[0].Role != "user" {
t.Fatalf("injected generation request = %+v", request)
}
return &domain.GenerateResponse{Content: "engine response"}, nil
}))
input := engineSmokeBackendInput()
assembled, err := New(Config{Sources: ConfigurationSources{Main: &input}}, options)
if err != nil {
t.Fatal(err)
}
report, err := assembled.SmokeTest(context.Background(), domain.SmokeTestRequest{
Kind: domain.SmokeTestTextInference, BackendID: "test-backend",
})
if err != nil {
t.Fatal(err)
}
if calls != 1 || !report.Completed || !report.Supported || report.Applicable == nil || !*report.Applicable ||
report.Facts.GenerationCompleted == nil || !*report.Facts.GenerationCompleted ||
report.Facts.TextReturned == nil || !*report.Facts.TextReturned ||
report.Facts.ExpectedTextMatched == nil || *report.Facts.ExpectedTextMatched ||
report.Facts.EndpointReachable != nil || report.Target.BackendID != "test-backend" {
t.Fatalf("injected smoke report calls=%d: %+v", calls, report)
}
if report.Target.Endpoint != "" || report.Target.EffectiveSettings.Endpoint == "" || len(report.Attempts) != 1 ||
report.Attempts[0].InitialEndpoint != "" || report.Attempts[0].Endpoint != "" {
t.Fatalf("injected report claimed HTTP destinations: %+v", report)
}
authentication, err := assembled.SmokeTest(context.Background(), domain.SmokeTestRequest{
Kind: domain.SmokeTestAuthentication, BackendID: "test-backend",
})
if err != nil {
t.Fatal(err)
}
if !authentication.Supported || authentication.Applicable == nil || *authentication.Applicable ||
authentication.Reason != domain.SmokeTestReasonAuthenticationNotApplicable || len(authentication.Attempts) != 0 {
t.Fatalf("injected authentication report = %+v", authentication)
}
}
func TestEngineSmokeTestUsesBuiltInHTTPClientForInferenceEvidence(t *testing.T) {
var observedURL string
var authorization string
options := Options{}
input := engineSmokeBackendInput()
assembled, err := New(Config{Sources: ConfigurationSources{Main: &input}, HTTPClient: &http.Client{Transport: engineRoundTripperFunc(func(request *http.Request) (*http.Response, error) {
observedURL = request.URL.String()
authorization = request.Header.Get("Authorization")
return &http.Response{
StatusCode: http.StatusOK,
Header: make(http.Header),
Body: io.NopCloser(strings.NewReader(`{"choices":[{"message":{"content":"engine response"}}],"usage":{"prompt_tokens":1,"completion_tokens":1,"total_tokens":2}}`)),
}, nil
})}}, options)
if err != nil {
t.Fatal(err)
}
report, err := assembled.SmokeTest(context.Background(), domain.SmokeTestRequest{
Kind: domain.SmokeTestTextInference, BackendID: "test-backend",
})
if err != nil {
t.Fatal(err)
}
if observedURL == "" || authorization != "" {
t.Fatalf("built-in request URL=%q authorization=%q", observedURL, authorization)
}
if !report.Completed || report.Facts.EndpointReachable == nil || !*report.Facts.EndpointReachable ||
report.Facts.ResponseReceived == nil || !*report.Facts.ResponseReceived ||
report.Facts.GenerationCompleted == nil || !*report.Facts.GenerationCompleted ||
report.Facts.CredentialsSupplied == nil || *report.Facts.CredentialsSupplied ||
report.Facts.TextReturned == nil || !*report.Facts.TextReturned ||
report.Facts.ExpectedTextMatched == nil || *report.Facts.ExpectedTextMatched || len(report.Attempts) != 1 ||
report.Attempts[0].Endpoint != observedURL {
t.Fatalf("built-in smoke report = %+v", report)
}
}
type engineSmokeClientFunc func(context.Context, domain.GenerateRequest) (*domain.GenerateResponse, error)
func (f engineSmokeClientFunc) Generate(ctx context.Context, request domain.GenerateRequest) (*domain.GenerateResponse, error) {
return f(ctx, request)
}
type engineRoundTripperFunc func(*http.Request) (*http.Response, error)
func (f engineRoundTripperFunc) RoundTrip(request *http.Request) (*http.Response, error) {
return f(request)
}

View File

@@ -72,14 +72,43 @@ func (e *ExhaustedError) Attempts() uint64 {
// Generate executes either one opaque client call or a frozen built-in HTTP
// sequence with one permit per actual dispatch.
func (c *Coordinator) Generate(ctx context.Context, req domain.GenerateRequest) (*domain.GenerateResponse, error) {
response, err, _ := c.generate(ctx, req, false)
return response, err
}
// GenerateWithEvidence runs the built-in retry path and returns its factual
// attempt evidence alongside any response retained before cancellation.
func (c *Coordinator) GenerateWithEvidence(
ctx context.Context,
req domain.GenerateRequest,
) (*domain.GenerateResponse, error, GenerationEvidence) {
return c.generate(ctx, req, true)
}
func (c *Coordinator) generate(
ctx context.Context,
req domain.GenerateRequest,
captureEvidence bool,
) (*domain.GenerateResponse, error, GenerationEvidence) {
if c.injected != nil {
return c.generateInjected(ctx, req)
return c.generateInjected(ctx, req, captureEvidence)
}
prepared, err := c.builtIn.PrepareGeneration(req)
if err != nil {
return nil, err
return nil, err, GenerationEvidence{}
}
evidence := GenerationEvidence{}
if captureEvidence {
evidence.InitialEndpoint = prepared.InitialEndpoint()
}
var usage observation.UsageAccumulator
finish := func(response *domain.GenerateResponse, err error) (*domain.GenerateResponse, error, GenerationEvidence) {
if captureEvidence {
evidence.Usage = usage.Observed()
}
return response, err, evidence
}
var attempts, retriesUsed uint64
recorder := observation.FromContext(ctx)
generation := req.Generation
@@ -88,56 +117,102 @@ func (c *Coordinator) Generate(ctx context.Context, req domain.GenerateRequest)
}
for {
if err := ctx.Err(); err != nil {
return nil, err
return finish(nil, err)
}
ordinal := attempts + 1
outcome, dispatched, capacityWait, generationDuration := c.execute(ctx, req.Target.BackendID, prepared)
outcome, dispatched, capacityWait, generationDuration, startedAt := c.execute(ctx, req.Target.BackendID, prepared, captureEvidence)
if dispatched {
attempts++
}
if captureEvidence {
attempt := AttemptEvidence{
StartedAt: startedAt.UTC(),
Dispatched: dispatched,
ResponseReceived: outcome.Evidence.ResponseReceived,
HTTP: cloneHTTPEvidence(outcome.Evidence),
CapacityWait: capacityWait,
RequestDuration: generationDuration,
Failure: ClassifyFailure(outcome.Err),
ResponseDecoded: outcome.ResponseDecoded,
GenerationCompleted: outcome.GenerationCompleted,
}
if outcome.StructuredOutputRequested != nil {
requested := *outcome.StructuredOutputRequested
attempt.StructuredOutputRequested = &requested
}
if dispatched {
usage.Add(outcome.Usage, outcome.UsagePresence)
var attemptUsage observation.UsageAccumulator
attemptUsage.Add(outcome.Usage, outcome.UsagePresence)
attempt.Usage = attemptUsage.Observed()
}
evidence.Attempts = append(evidence.Attempts, attempt)
}
if recorder != nil {
recorder.Attempt(observation.AttemptData{Generation: generation, Attempt: ordinal,
Dispatched: dispatched, CapacityWait: capacityWait, GenerationDuration: generationDuration,
Outcome: attemptOutcome(outcome.Err), Failure: attemptFailure(outcome.Err),
Outcome: attemptOutcome(outcome.Err), Failure: ClassifyFailure(outcome.Err),
StatusCode: outcome.StatusCode, Usage: outcome.Usage, UsagePresence: outcome.UsagePresence})
}
if outcome.Err == nil {
if err := ctx.Err(); err != nil {
return nil, err
if captureEvidence {
return finish(outcome.Response, err)
}
return finish(nil, err)
}
return outcome.Response, nil
return finish(outcome.Response, nil)
}
if !dispatched || !eligibleStatus(outcome) {
return nil, outcome.Err
if captureEvidence {
return finish(outcome.Response, outcome.Err)
}
return finish(nil, outcome.Err)
}
if retriesUsed == req.TransientRetries {
if retriesUsed == 0 {
return nil, outcome.Err
return finish(nil, outcome.Err)
}
return nil, &ExhaustedError{attempts: attempts, cause: outcome.Err}
return finish(nil, &ExhaustedError{attempts: attempts, cause: outcome.Err})
}
if err := ctx.Err(); err != nil {
return nil, err
return finish(nil, err)
}
retryOrdinal := retriesUsed + 1
delay := retryDelay(c.backoff(retryOrdinal), outcome.RetryNotBefore, c.now())
waitStart := c.now()
waitErr := c.wait(ctx, delay)
waitDuration := elapsed(waitStart, c.now())
if captureEvidence {
evidence.RetryDelays = append(evidence.RetryDelays, waitDuration)
}
if recorder != nil {
recorder.Delay(observation.DelayData{Generation: generation, Attempt: attempts,
Elapsed: elapsed(waitStart, c.now()), Failure: attemptFailure(waitErr)})
Elapsed: waitDuration, Failure: ClassifyFailure(waitErr)})
}
if waitErr != nil {
return nil, waitErr
return finish(nil, waitErr)
}
if err := ctx.Err(); err != nil {
return nil, err
return finish(nil, err)
}
retriesUsed++
}
}
func (c *Coordinator) generateInjected(ctx context.Context, req domain.GenerateRequest) (*domain.GenerateResponse, error) {
func (c *Coordinator) generateInjected(
ctx context.Context,
req domain.GenerateRequest,
captureEvidence bool,
) (*domain.GenerateResponse, error, GenerationEvidence) {
evidence := GenerationEvidence{}
var usage observation.UsageAccumulator
finish := func(response *domain.GenerateResponse, err error) (*domain.GenerateResponse, error, GenerationEvidence) {
if captureEvidence {
evidence.Usage = usage.Observed()
}
return response, err, evidence
}
recorder := observation.FromContext(ctx)
generation := req.Generation
if generation == 0 {
@@ -147,18 +222,32 @@ func (c *Coordinator) generateInjected(ctx context.Context, req domain.GenerateR
release, err := c.manager.Acquire(ctx, req.Target.BackendID)
capacityWait := c.capacityWait(req.Target.BackendID, start)
if err != nil {
if recorder != nil {
recorder.Attempt(observation.AttemptData{Generation: generation, Attempt: 1, CapacityWait: capacityWait, Outcome: domain.ExecutionOutcomeError, Failure: attemptFailure(err)})
if captureEvidence {
evidence.Attempts = append(evidence.Attempts, AttemptEvidence{
StartedAt: start.UTC(),
CapacityWait: capacityWait,
Failure: ClassifyFailure(err),
})
}
return nil, err
if recorder != nil {
recorder.Attempt(observation.AttemptData{Generation: generation, Attempt: 1, CapacityWait: capacityWait, Outcome: domain.ExecutionOutcomeError, Failure: ClassifyFailure(err)})
}
return finish(nil, err)
}
defer release()
if err := ctx.Err(); err != nil {
release()
if recorder != nil {
recorder.Attempt(observation.AttemptData{Generation: generation, Attempt: 1, CapacityWait: capacityWait, Outcome: domain.ExecutionOutcomeError, Failure: attemptFailure(err)})
if captureEvidence {
evidence.Attempts = append(evidence.Attempts, AttemptEvidence{
StartedAt: start.UTC(),
CapacityWait: capacityWait,
Failure: ClassifyFailure(err),
})
}
return nil, err
if recorder != nil {
recorder.Attempt(observation.AttemptData{Generation: generation, Attempt: 1, CapacityWait: capacityWait, Outcome: domain.ExecutionOutcomeError, Failure: ClassifyFailure(err)})
}
return finish(nil, err)
}
dispatchStart := c.now()
response, err := c.injected.Generate(ctx, req)
@@ -166,10 +255,33 @@ func (c *Coordinator) generateInjected(ctx context.Context, req domain.GenerateR
if err == nil && response == nil {
err = llm.ErrMalformedResponse
}
if captureEvidence {
attempt := AttemptEvidence{
StartedAt: start.UTC(),
Dispatched: true,
ResponseReceived: response != nil,
RequestDuration: duration,
Failure: ClassifyFailure(err),
GenerationCompleted: err == nil && response != nil,
}
requested := req.StructuredOutput != nil
attempt.StructuredOutputRequested = &requested
var reportedUsage domain.TokenUsage
var reportedPresence *domain.TokenUsagePresence
if err == nil && response != nil {
reportedUsage = response.Usage
reportedPresence = response.UsagePresence
}
usage.Add(reportedUsage, reportedPresence)
var attemptUsage observation.UsageAccumulator
attemptUsage.Add(reportedUsage, reportedPresence)
attempt.Usage = attemptUsage.Observed()
evidence.Attempts = append(evidence.Attempts, attempt)
}
release()
if recorder != nil {
data := observation.AttemptData{Generation: generation, Attempt: 1, Dispatched: true, CapacityWait: capacityWait, GenerationDuration: duration,
Outcome: attemptOutcome(err), Failure: attemptFailure(err)}
Outcome: attemptOutcome(err), Failure: ClassifyFailure(err)}
if err == nil {
data.Usage = response.Usage
data.UsagePresence = response.UsagePresence
@@ -177,28 +289,38 @@ func (c *Coordinator) generateInjected(ctx context.Context, req domain.GenerateR
recorder.Attempt(data)
}
if err != nil {
return nil, err
return finish(nil, err)
}
if err := ctx.Err(); err != nil {
return nil, err
if captureEvidence {
return finish(response, err)
}
return finish(nil, err)
}
return response, nil
return finish(response, nil)
}
func (c *Coordinator) execute(ctx context.Context, backendID string, prepared *llm.PreparedGeneration) (outcome llm.AttemptOutcome, dispatched bool, capacityWait, generationDuration time.Duration) {
func (c *Coordinator) execute(ctx context.Context, backendID string, prepared *llm.PreparedGeneration, captureEvidence bool) (outcome llm.AttemptOutcome, dispatched bool, capacityWait, generationDuration time.Duration, startedAt time.Time) {
start := c.now()
startedAt = start
release, err := c.manager.Acquire(ctx, backendID)
capacityWait = c.capacityWait(backendID, start)
if err != nil {
return llm.AttemptOutcome{Err: err}, false, capacityWait, 0
return llm.AttemptOutcome{Err: err}, false, capacityWait, 0, startedAt
}
defer release()
if err := ctx.Err(); err != nil {
return llm.AttemptOutcome{Err: err}, false, capacityWait, 0
return llm.AttemptOutcome{Err: err}, false, capacityWait, 0, startedAt
}
dispatchStart := c.now()
outcome = prepared.Execute(ctx)
return outcome, true, capacityWait, elapsed(dispatchStart, c.now())
if captureEvidence {
outcome = prepared.ExecuteWithEvidence(ctx)
dispatched = outcome.Evidence.DispatchInvoked
} else {
outcome = prepared.Execute(ctx)
dispatched = true
}
return outcome, dispatched, capacityWait, elapsed(dispatchStart, c.now()), startedAt
}
func (c *Coordinator) capacityWait(backendID string, start time.Time) time.Duration {
@@ -222,7 +344,8 @@ func attemptOutcome(err error) domain.ExecutionOutcome {
return domain.ExecutionOutcomeError
}
func attemptFailure(err error) domain.ExecutionFailureKind {
// ClassifyFailure maps an execution error to the shared content-free category.
func ClassifyFailure(err error) domain.ExecutionFailureKind {
if err == nil {
return domain.ExecutionFailureNone
}

View File

@@ -0,0 +1,43 @@
package generation
import (
"time"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
"gitea.maximumdirect.net/eric/promptkit/internal/llm"
)
// GenerationEvidence contains invocation-local request evidence without raw
// provider bodies, headers, credentials, or generated content.
type GenerationEvidence struct {
// InitialEndpoint is the frozen built-in request URL, if prepared.
// Injected generation leaves it unknown.
InitialEndpoint string
Attempts []AttemptEvidence
RetryDelays []time.Duration
Usage domain.ObservedTokenUsage
}
// AttemptEvidence describes one capacity acquisition and any dispatch it
// reached. Generated content remains on the separate response return value.
type AttemptEvidence struct {
StartedAt time.Time
Dispatched bool
ResponseReceived bool
HTTP llm.HTTPExecutionEvidence
CapacityWait time.Duration
RequestDuration time.Duration
Failure domain.ExecutionFailureKind
Usage domain.ObservedTokenUsage
ResponseDecoded bool
GenerationCompleted bool
StructuredOutputRequested *bool
}
func cloneHTTPEvidence(evidence llm.HTTPExecutionEvidence) llm.HTTPExecutionEvidence {
if evidence.HTTPStatus != nil {
status := *evidence.HTTPStatus
evidence.HTTPStatus = &status
}
return evidence
}

View File

@@ -0,0 +1,321 @@
package generation
import (
"context"
"errors"
"net/http"
"sync"
"sync/atomic"
"testing"
"time"
"gitea.maximumdirect.net/eric/promptkit/internal/capacity"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
"gitea.maximumdirect.net/eric/promptkit/internal/llm"
)
func TestGenerateWithEvidenceRetainsAttemptsAndUsage(t *testing.T) {
var now = time.Date(2026, 10, 13, 12, 0, 0, 0, time.UTC)
calls := 0
coordinator := testCoordinator(t, func(*http.Request) (*http.Response, error) {
calls++
if calls == 1 {
now = now.Add(4 * time.Millisecond)
return response(http.StatusServiceUnavailable, `{"error":{}}`), nil
}
now = now.Add(7 * time.Millisecond)
return response(http.StatusOK, `{"choices":[{"message":{"content":"ok"}}],"usage":{"prompt_tokens":3}}`), nil
})
coordinator.now = func() time.Time { return now }
coordinator.wait = func(context.Context, time.Duration) error {
now = now.Add(19 * time.Millisecond)
return nil
}
result, err, evidence := coordinator.GenerateWithEvidence(context.Background(), testRequest(1))
if err != nil || result == nil || result.Content != "ok" || calls != 2 {
t.Fatalf("result=%+v error=%v calls=%d", result, err, calls)
}
if len(evidence.Attempts) != 2 || len(evidence.RetryDelays) != 1 || evidence.RetryDelays[0] != 19*time.Millisecond {
t.Fatalf("attempts/delays = %#v / %#v", evidence.Attempts, evidence.RetryDelays)
}
first, second := evidence.Attempts[0], evidence.Attempts[1]
if !first.Dispatched || !first.HTTP.RequestObserved || !first.HTTP.ResponseReceived || first.HTTP.HTTPStatus == nil || *first.HTTP.HTTPStatus != http.StatusServiceUnavailable {
t.Fatalf("first attempt evidence = %+v", first)
}
if first.Failure != domain.ExecutionFailureProviderStatus || first.RequestDuration != 4*time.Millisecond {
t.Fatalf("first attempt failure/timing = %v / %v", first.Failure, first.RequestDuration)
}
if !second.Dispatched || !second.GenerationCompleted || !second.ResponseDecoded || second.HTTP.HTTPStatus == nil || *second.HTTP.HTTPStatus != http.StatusOK {
t.Fatalf("second attempt evidence = %+v", second)
}
if second.Failure != domain.ExecutionFailureNone || second.RequestDuration != 7*time.Millisecond {
t.Fatalf("second attempt failure/timing = %v / %v", second.Failure, second.RequestDuration)
}
if !second.Usage.PromptTokens.Known || !second.Usage.PromptTokens.Complete || second.Usage.PromptTokens.Value != 3 {
t.Fatalf("per-attempt usage = %+v", second.Usage.PromptTokens)
}
if !evidence.Usage.PromptTokens.Known || evidence.Usage.PromptTokens.Complete || evidence.Usage.PromptTokens.Value != 3 {
t.Fatalf("aggregate usage = %+v", evidence.Usage.PromptTokens)
}
}
func TestGenerateWithEvidenceRetainsExhaustedAttempts(t *testing.T) {
calls := 0
coordinator := testCoordinator(t, func(*http.Request) (*http.Response, error) {
calls++
return response(http.StatusServiceUnavailable, `{"error":{}}`), nil
})
coordinator.wait = func(context.Context, time.Duration) error { return nil }
result, err, evidence := coordinator.GenerateWithEvidence(context.Background(), testRequest(1))
var exhausted *ExhaustedError
if result != nil || !errors.As(err, &exhausted) || calls != 2 || exhausted.Attempts() != 2 {
t.Fatalf("result=%+v error=%v calls=%d", result, err, calls)
}
if len(evidence.Attempts) != 2 || len(evidence.RetryDelays) != 1 {
t.Fatalf("attempt/delay evidence = %#v / %#v", evidence.Attempts, evidence.RetryDelays)
}
for _, attempt := range evidence.Attempts {
if !attempt.Dispatched || attempt.Failure != domain.ExecutionFailureProviderStatus || attempt.HTTP.HTTPStatus == nil || *attempt.HTTP.HTTPStatus != http.StatusServiceUnavailable {
t.Fatalf("exhausted attempt evidence = %+v", attempt)
}
}
if evidence.Usage.PromptTokens.Known || evidence.Usage.PromptTokens.Complete {
t.Fatalf("missing provider usage was claimed: %+v", evidence.Usage.PromptTokens)
}
}
func TestGenerateWithEvidenceRetainsInterruptedBackoffWithoutAttempt(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
var now = time.Date(2026, 10, 13, 12, 0, 0, 0, time.UTC)
calls := 0
coordinator := testCoordinator(t, func(*http.Request) (*http.Response, error) {
calls++
now = now.Add(3 * time.Millisecond)
return response(http.StatusServiceUnavailable, `{"error":{}}`), nil
})
coordinator.now = func() time.Time { return now }
coordinator.wait = func(ctx context.Context, _ time.Duration) error {
now = now.Add(11 * time.Millisecond)
cancel()
return ctx.Err()
}
result, err, evidence := coordinator.GenerateWithEvidence(ctx, testRequest(2))
if result != nil || !errors.Is(err, context.Canceled) || calls != 1 {
t.Fatalf("result=%+v error=%v calls=%d", result, err, calls)
}
if len(evidence.Attempts) != 1 || !evidence.Attempts[0].Dispatched || len(evidence.RetryDelays) != 1 || evidence.RetryDelays[0] != 11*time.Millisecond {
t.Fatalf("interrupted evidence = %#v", evidence)
}
if evidence.Attempts[0].HTTP.HTTPStatus == nil || *evidence.Attempts[0].HTTP.HTTPStatus != http.StatusServiceUnavailable {
t.Fatalf("last dispatched response was lost: %+v", evidence.Attempts[0])
}
}
func TestGenerateWithEvidenceRecordsCanceledReacquisition(t *testing.T) {
var now = time.Date(2026, 10, 13, 12, 0, 0, 0, time.UTC)
calls := 0
coordinator := testCoordinator(t, func(*http.Request) (*http.Response, error) {
calls++
now = now.Add(2 * time.Millisecond)
return response(http.StatusServiceUnavailable, `{"error":{}}`), nil
})
coordinator.now = func() time.Time { return now }
baseContext, cancel := context.WithCancel(context.Background())
defer cancel()
contextWithCancelOnWait := &cancelOnDoneContext{Context: baseContext, cancel: cancel}
var releaseHeld func()
coordinator.wait = func(context.Context, time.Duration) error {
release, err := coordinator.manager.Acquire(context.Background(), "limited")
if err != nil {
t.Fatalf("hold permit: %v", err)
}
releaseHeld = release
contextWithCancelOnWait.armed.Store(true)
return nil
}
// Signal cancellation only when the second acquisition has queued behind
// the held permit, avoiding scheduler- or sleep-based synchronization.
result, err, evidence := coordinator.GenerateWithEvidence(contextWithCancelOnWait, testRequest(2))
if releaseHeld != nil {
releaseHeld()
}
if result != nil || !errors.Is(err, context.Canceled) || calls != 1 {
t.Fatalf("result=%+v error=%v calls=%d", result, err, calls)
}
if len(evidence.Attempts) != 2 || !evidence.Attempts[0].Dispatched || evidence.Attempts[1].Dispatched {
t.Fatalf("acquisition evidence = %#v", evidence.Attempts)
}
if evidence.Attempts[1].Failure != domain.ExecutionFailureCanceled {
t.Fatalf("canceled acquisition failure = %v", evidence.Attempts[1].Failure)
}
if evidence.Attempts[0].HTTP.HTTPStatus == nil || *evidence.Attempts[0].HTTP.HTTPStatus != http.StatusServiceUnavailable {
t.Fatalf("last dispatched evidence was lost: %+v", evidence.Attempts)
}
}
func TestGenerateWithEvidenceNoDispatchHasCompleteZeroUsage(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
cancel()
coordinator := testCoordinator(t, func(*http.Request) (*http.Response, error) {
t.Fatal("canceled generation dispatched")
return nil, nil
})
result, err, evidence := coordinator.GenerateWithEvidence(ctx, testRequest(0))
if result != nil || !errors.Is(err, context.Canceled) || len(evidence.Attempts) != 0 {
t.Fatalf("result=%+v error=%v evidence=%+v", result, err, evidence)
}
if !evidence.Usage.PromptTokens.Known || !evidence.Usage.PromptTokens.Complete || evidence.Usage.PromptTokens.Value != 0 {
t.Fatalf("no-dispatch usage = %+v", evidence.Usage.PromptTokens)
}
}
func TestGenerateWithEvidenceForInjectedClient(t *testing.T) {
opaqueFailure := errors.New("injected client failed")
presence := &domain.TokenUsagePresence{PromptTokens: true}
for _, tc := range []struct {
name string
response *domain.GenerateResponse
err error
wantResponse bool
wantGeneration bool
wantFailure domain.ExecutionFailureKind
wantUsage bool
wantStructured bool
}{
{
name: "success",
response: &domain.GenerateResponse{Content: "private output", Usage: domain.TokenUsage{PromptTokens: 4}, UsagePresence: presence},
wantResponse: true,
wantGeneration: true,
wantUsage: true,
wantStructured: true,
},
{
name: "nil response",
wantFailure: domain.ExecutionFailureMalformedResponse,
},
{
name: "opaque failure",
err: opaqueFailure,
wantFailure: domain.ExecutionFailureGeneration,
},
{
name: "response with error",
response: &domain.GenerateResponse{Content: "must not be evaluated", Usage: domain.TokenUsage{PromptTokens: 9}, UsagePresence: presence},
err: opaqueFailure,
wantResponse: true,
wantFailure: domain.ExecutionFailureGeneration,
wantStructured: true,
},
} {
t.Run(tc.name, func(t *testing.T) {
client := &evidenceInjectedClient{response: tc.response, err: tc.err}
coordinator := newEvidenceInjectedCoordinator(t, client)
req := testRequest(7)
if tc.wantStructured {
req.StructuredOutput = &domain.StructuredOutputSpec{}
}
result, err, evidence := coordinator.GenerateWithEvidence(context.Background(), req)
if client.calls != 1 || len(evidence.Attempts) != 1 || len(evidence.RetryDelays) != 0 {
t.Fatalf("calls=%d attempts=%+v retry delays=%v", client.calls, evidence.Attempts, evidence.RetryDelays)
}
attempt := evidence.Attempts[0]
if !attempt.Dispatched || attempt.ResponseReceived != tc.wantResponse || attempt.GenerationCompleted != tc.wantGeneration || attempt.Failure != tc.wantFailure {
t.Fatalf("attempt evidence = %+v", attempt)
}
if attempt.HTTP != (llm.HTTPExecutionEvidence{}) {
t.Fatalf("injected call claimed HTTP evidence: %+v", attempt.HTTP)
}
if attempt.StructuredOutputRequested == nil || *attempt.StructuredOutputRequested != tc.wantStructured {
t.Fatalf("structured-output request fact = %v", attempt.StructuredOutputRequested)
}
if tc.wantFailure == domain.ExecutionFailureNone {
if err != nil || result == nil || result.Content != "private output" {
t.Fatalf("result=%+v error=%v", result, err)
}
} else if result != nil || err == nil {
t.Fatalf("result=%+v error=%v; expected failed call", result, err)
}
if attempt.Usage.PromptTokens.Known != tc.wantUsage || evidence.Usage.PromptTokens.Known != tc.wantUsage {
t.Fatalf("attempt/aggregate usage = %+v / %+v", attempt.Usage, evidence.Usage)
}
if tc.wantUsage && (attempt.Usage.PromptTokens.Value != 4 || evidence.Usage.PromptTokens.Value != 4) {
t.Fatalf("reported usage was not retained: %+v / %+v", attempt.Usage, evidence.Usage)
}
})
}
}
func TestGenerateWithEvidenceRetainsInjectedSuccessAfterCancellation(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
client := &evidenceInjectedClient{
response: &domain.GenerateResponse{Content: "private output"},
afterGenerate: func() { cancel() },
}
coordinator := newEvidenceInjectedCoordinator(t, client)
req := testRequest(5)
req.StructuredOutput = &domain.StructuredOutputSpec{}
result, err, evidence := coordinator.GenerateWithEvidence(ctx, req)
if !errors.Is(err, context.Canceled) || result == nil || result.Content != "private output" {
t.Fatalf("result=%+v error=%v", result, err)
}
if client.calls != 1 || len(evidence.Attempts) != 1 {
t.Fatalf("calls=%d attempts=%+v", client.calls, evidence.Attempts)
}
attempt := evidence.Attempts[0]
if !attempt.ResponseReceived || !attempt.GenerationCompleted || attempt.Failure != domain.ExecutionFailureNone {
t.Fatalf("successful generation evidence was lost: %+v", attempt)
}
if attempt.HTTP != (llm.HTTPExecutionEvidence{}) {
t.Fatalf("injected call claimed HTTP evidence: %+v", attempt.HTTP)
}
}
type evidenceInjectedClient struct {
calls int
response *domain.GenerateResponse
err error
afterGenerate func()
}
func (c *evidenceInjectedClient) Generate(context.Context, domain.GenerateRequest) (*domain.GenerateResponse, error) {
c.calls++
if c.afterGenerate != nil {
c.afterGenerate()
}
return c.response, c.err
}
func newEvidenceInjectedCoordinator(t *testing.T, client llm.Client) *Coordinator {
t.Helper()
manager, err := capacity.NewManager(map[string]domain.BackendCapacityPolicy{
"limited": {ConcurrencyLimit: 1},
})
if err != nil {
t.Fatal(err)
}
return NewInjected(manager, client)
}
type cancelOnDoneContext struct {
context.Context
cancel context.CancelFunc
once sync.Once
armed atomic.Bool
}
func (c *cancelOnDoneContext) Done() <-chan struct{} {
if c.armed.Load() {
c.once.Do(c.cancel)
}
return c.Context.Done()
}

View File

@@ -0,0 +1,191 @@
package llm
import (
"context"
"io"
"net/http"
"sync"
"time"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
)
// HTTPExecutionEvidence contains only facts observable at the configured
// RoundTripper boundary. It intentionally excludes request headers and bodies.
type HTTPExecutionEvidence struct {
DispatchInvoked bool
RequestObserved bool
Endpoint string
CredentialsSupplied bool
ResponseReceived bool
HTTPStatus *int
}
type httpEvidenceCollector struct {
mu sync.Mutex
evidence HTTPExecutionEvidence
}
func (c *httpEvidenceCollector) observeRequest(request *http.Request) {
c.mu.Lock()
defer c.mu.Unlock()
c.evidence.RequestObserved = true
c.evidence.Endpoint = endpointWithoutUserInfo(request)
c.evidence.CredentialsSupplied = requestHasCredentials(request)
c.evidence.ResponseReceived = false
c.evidence.HTTPStatus = nil
}
func (c *httpEvidenceCollector) observeResponse(response *http.Response) {
if response == nil {
return
}
c.mu.Lock()
defer c.mu.Unlock()
status := response.StatusCode
c.evidence.ResponseReceived = true
c.evidence.HTTPStatus = &status
}
func (c *httpEvidenceCollector) snapshot() HTTPExecutionEvidence {
c.mu.Lock()
defer c.mu.Unlock()
evidence := c.evidence
if c.evidence.HTTPStatus != nil {
status := *c.evidence.HTTPStatus
evidence.HTTPStatus = &status
}
return evidence
}
type observingRoundTripper struct {
next http.RoundTripper
evidence *httpEvidenceCollector
}
func (t observingRoundTripper) RoundTrip(request *http.Request) (*http.Response, error) {
t.evidence.observeRequest(request)
response, err := t.next.RoundTrip(request)
t.evidence.observeResponse(response)
return response, err
}
// doWithEvidence executes one request with a shallow copy of client. The copy
// preserves timeout, redirect, and cookie-jar behavior without changing client.
func doWithEvidence(client *http.Client, request *http.Request) (*http.Response, error, HTTPExecutionEvidence) {
if client == nil {
client = &http.Client{}
}
collector := &httpEvidenceCollector{}
clientCopy := *client
transport := clientCopy.Transport
if transport == nil {
transport = http.DefaultTransport
}
clientCopy.Transport = observingRoundTripper{next: transport, evidence: collector}
response, err := clientCopy.Do(request)
observed := collector.snapshot()
observed.DispatchInvoked = true
return response, err, observed
}
// GetWithEvidence performs a plain GET with the configured built-in HTTP
// client. A positive timeoutSeconds applies to the request through response
// body closure, matching the whole-request behavior of http.Client.Timeout.
func (c *OpenAICompatibleClient) GetWithEvidence(
ctx context.Context,
endpoint string,
timeoutSeconds int,
) (*http.Response, HTTPExecutionEvidence, error) {
return c.getWithEvidence(ctx, endpoint, timeoutSeconds, "")
}
// GetWithAuthenticationEvidence performs a plain GET with credentials resolved
// from the selected target. Missing credentials are allowed for diagnostic
// authentication probes even when generation requires a key.
func (c *OpenAICompatibleClient) GetWithAuthenticationEvidence(
ctx context.Context,
endpoint string,
timeoutSeconds int,
target domain.ExecutionTarget,
) (*http.Response, HTTPExecutionEvidence, error) {
return c.getWithEvidence(ctx, endpoint, timeoutSeconds, resolveAPIKey(target))
}
func (c *OpenAICompatibleClient) getWithEvidence(
ctx context.Context,
endpoint string,
timeoutSeconds int,
apiKey string,
) (*http.Response, HTTPExecutionEvidence, error) {
requestContext := ctx
var cancel context.CancelFunc
if timeoutSeconds > 0 {
requestContext, cancel = context.WithTimeout(ctx, secondsDuration(timeoutSeconds))
}
request, err := http.NewRequestWithContext(requestContext, http.MethodGet, endpoint, nil)
if err != nil {
if cancel != nil {
cancel()
}
return nil, HTTPExecutionEvidence{}, err
}
if apiKey != "" {
request.Header.Set("Authorization", "Bearer "+apiKey)
}
response, err, evidence := doWithEvidence(c.httpClient, request)
if response == nil || response.Body == nil {
if cancel != nil {
cancel()
}
return response, evidence, err
}
if cancel != nil {
response.Body = &cancelOnCloseReadCloser{ReadCloser: response.Body, cancel: cancel}
}
return response, evidence, err
}
func secondsDuration(seconds int) time.Duration {
return time.Duration(seconds) * time.Second
}
type cancelOnCloseReadCloser struct {
io.ReadCloser
cancel context.CancelFunc
once sync.Once
}
func (r *cancelOnCloseReadCloser) Close() error {
r.once.Do(r.cancel)
return r.ReadCloser.Close()
}
func endpointWithoutUserInfo(request *http.Request) string {
if request == nil || request.URL == nil {
return ""
}
endpoint := *request.URL
endpoint.User = nil
return endpoint.String()
}
func requestHasCredentials(request *http.Request) bool {
if request == nil {
return false
}
if request.Header.Get("Authorization") != "" {
return true
}
return request.URL != nil && request.URL.User != nil
}
var _ http.RoundTripper = observingRoundTripper{}
var _ io.ReadCloser = (*cancelOnCloseReadCloser)(nil)

View File

@@ -0,0 +1,340 @@
package llm
import (
"context"
"errors"
"io"
"net/http"
"net/http/cookiejar"
"net/http/httptest"
"strconv"
"strings"
"sync"
"testing"
"time"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
)
func TestGetWithEvidenceCapturesResponseAndOmitsURLUserInfo(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) {
if request.Header.Get("Authorization") == "" {
t.Error("request did not carry URL user credentials")
}
w.WriteHeader(http.StatusNoContent)
}))
defer server.Close()
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{Timeout: time.Second})
if err != nil {
t.Fatalf("construct client: %v", err)
}
endpoint := strings.Replace(server.URL, "http://", "http://user:secret@", 1) + "/health"
response, evidence, err := client.GetWithEvidence(context.Background(), endpoint, 0)
if err != nil {
t.Fatalf("GET endpoint: %v", err)
}
if response == nil {
t.Fatal("GET returned no response")
}
response.Body.Close()
if !evidence.DispatchInvoked || !evidence.RequestObserved || !evidence.ResponseReceived {
t.Fatalf("incomplete response evidence: %+v", evidence)
}
if evidence.HTTPStatus == nil || *evidence.HTTPStatus != http.StatusNoContent {
t.Fatalf("status evidence = %v, want %d", evidence.HTTPStatus, http.StatusNoContent)
}
if !evidence.CredentialsSupplied {
t.Fatal("evidence did not record URL credentials")
}
if strings.Contains(evidence.Endpoint, "user") || strings.Contains(evidence.Endpoint, "secret") {
t.Fatalf("exposed endpoint contains URL user info: %q", evidence.Endpoint)
}
if want := server.URL + "/health"; evidence.Endpoint != want {
t.Fatalf("endpoint = %q, want %q", evidence.Endpoint, want)
}
}
func TestGetWithAuthenticationEvidenceResolvesOptionalCredentials(t *testing.T) {
tests := []struct {
name string
target domain.ExecutionTarget
environment string
wantAuth string
}{
{
name: "direct key takes precedence",
target: domain.ExecutionTarget{
APIKey: " direct-key ",
APIKeyEnv: "PROMPTKIT_DIAGNOSTIC_KEY",
},
environment: "environment-key",
wantAuth: "Bearer direct-key",
},
{
name: "environment key is used",
target: domain.ExecutionTarget{APIKeyEnv: "PROMPTKIT_DIAGNOSTIC_KEY"},
environment: " environment-key ",
wantAuth: "Bearer environment-key",
},
{
name: "missing required key is optional here",
target: domain.ExecutionTarget{APIKeyEnv: "PROMPTKIT_DIAGNOSTIC_KEY", APIKeyRequired: true},
environment: " ",
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
t.Setenv("PROMPTKIT_DIAGNOSTIC_KEY", tc.environment)
var authorization string
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
HTTPClient: &http.Client{Transport: roundTripFunc(func(request *http.Request) (*http.Response, error) {
authorization = request.Header.Get("Authorization")
return &http.Response{
StatusCode: http.StatusUnauthorized,
Header: make(http.Header),
Body: io.NopCloser(strings.NewReader("")),
}, nil
})},
})
if err != nil {
t.Fatal(err)
}
response, evidence, err := client.GetWithAuthenticationEvidence(
context.Background(), "https://example.invalid/key", 0, tc.target,
)
if err != nil || response == nil {
t.Fatalf("response=%v error=%v", response, err)
}
response.Body.Close()
if authorization != tc.wantAuth || evidence.CredentialsSupplied != (tc.wantAuth != "") {
t.Fatalf("authorization=%q evidence=%+v", authorization, evidence)
}
if evidence.Endpoint != "https://example.invalid/key" || evidence.HTTPStatus == nil || *evidence.HTTPStatus != http.StatusUnauthorized {
t.Fatalf("HTTP evidence = %+v", evidence)
}
})
}
}
func TestDoWithEvidenceRecordsRequestWithoutResponse(t *testing.T) {
transportErr := errors.New("transport unavailable")
client := &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
return nil, transportErr
})}
request, err := http.NewRequest(http.MethodGet, "https://diagnostic.example/status", nil)
if err != nil {
t.Fatalf("create request: %v", err)
}
response, err, evidence := doWithEvidence(client, request)
if response != nil || !errors.Is(err, transportErr) {
t.Fatalf("response/error = %v/%v, want nil/transport error", response, err)
}
if !evidence.DispatchInvoked || !evidence.RequestObserved || evidence.ResponseReceived {
t.Fatalf("unexpected request evidence: %+v", evidence)
}
if evidence.HTTPStatus != nil || evidence.Endpoint != request.URL.String() || evidence.CredentialsSupplied {
t.Fatalf("unexpected evidence values: %+v", evidence)
}
}
func TestDoWithEvidenceRetainsResponseReturnedAlongsideError(t *testing.T) {
transportErr := errors.New("transport reported an error with a response")
client := &http.Client{Transport: roundTripFunc(func(request *http.Request) (*http.Response, error) {
return responseFor(request, http.StatusBadGateway, ""), transportErr
})}
request, err := http.NewRequest(http.MethodGet, "https://diagnostic.example/status", nil)
if err != nil {
t.Fatalf("create request: %v", err)
}
response, err, evidence := doWithEvidence(client, request)
if !errors.Is(err, transportErr) {
t.Fatalf("error = %v, want transport error", err)
}
if response != nil {
response.Body.Close()
}
if !evidence.RequestObserved || !evidence.ResponseReceived || evidence.HTTPStatus == nil || *evidence.HTTPStatus != http.StatusBadGateway {
t.Fatalf("observed response was lost alongside error: %+v", evidence)
}
}
func TestDoWithEvidenceUsesLastRedirectRequest(t *testing.T) {
var observedAuthorization []string
transport := roundTripFunc(func(request *http.Request) (*http.Response, error) {
observedAuthorization = append(observedAuthorization, request.Header.Get("Authorization"))
if request.URL.Host == "source.example" {
return responseFor(request, http.StatusFound, "https://other.example/final"), nil
}
return responseFor(request, http.StatusUnauthorized, ""), nil
})
client := &http.Client{Transport: transport}
request, err := http.NewRequest(http.MethodGet, "https://source.example/start", nil)
if err != nil {
t.Fatalf("create request: %v", err)
}
request.Header.Set("Authorization", "Bearer private-token")
response, err, evidence := doWithEvidence(client, request)
if err != nil {
t.Fatalf("follow redirect: %v", err)
}
if response == nil {
t.Fatal("redirect returned no response")
}
response.Body.Close()
if len(observedAuthorization) != 2 || observedAuthorization[0] == "" || observedAuthorization[1] != "" {
t.Fatalf("authorization across redirect = %#v", observedAuthorization)
}
if evidence.Endpoint != "https://other.example/final" || evidence.CredentialsSupplied {
t.Fatalf("evidence did not describe credential-stripped last hop: %+v", evidence)
}
if !evidence.ResponseReceived || evidence.HTTPStatus == nil || *evidence.HTTPStatus != http.StatusUnauthorized {
t.Fatalf("last-hop response evidence = %+v", evidence)
}
}
func TestDoWithEvidenceClearsPriorResponseOnFailedRedirect(t *testing.T) {
transportErr := errors.New("redirect target unavailable")
transport := roundTripFunc(func(request *http.Request) (*http.Response, error) {
if request.URL.Host == "source.example" {
return responseFor(request, http.StatusTemporaryRedirect, "https://target.example/unavailable"), nil
}
return nil, transportErr
})
client := &http.Client{Transport: transport}
request, err := http.NewRequest(http.MethodGet, "https://source.example/start", nil)
if err != nil {
t.Fatalf("create request: %v", err)
}
response, err, evidence := doWithEvidence(client, request)
if response != nil || !errors.Is(err, transportErr) {
t.Fatalf("response/error = %v/%v, want nil/transport error", response, err)
}
if !evidence.RequestObserved || evidence.Endpoint != "https://target.example/unavailable" {
t.Fatalf("evidence did not retain failed last request: %+v", evidence)
}
if evidence.ResponseReceived || evidence.HTTPStatus != nil {
t.Fatalf("prior redirect response was not cleared: %+v", evidence)
}
}
func TestDoWithEvidenceRetainsResponseWhenRedirectPolicyStops(t *testing.T) {
redirectErr := errors.New("redirect rejected")
transportCalls := 0
client := &http.Client{
Transport: roundTripFunc(func(request *http.Request) (*http.Response, error) {
transportCalls++
return responseFor(request, http.StatusFound, "https://other.example/never-requested"), nil
}),
CheckRedirect: func(*http.Request, []*http.Request) error { return redirectErr },
}
request, err := http.NewRequest(http.MethodGet, "https://source.example/start", nil)
if err != nil {
t.Fatalf("create request: %v", err)
}
response, err, evidence := doWithEvidence(client, request)
if !errors.Is(err, redirectErr) {
t.Fatalf("redirect error = %v, want policy error", err)
}
if response == nil || response.StatusCode != http.StatusFound {
t.Fatalf("redirect response = %v, want original response", response)
}
response.Body.Close()
if transportCalls != 1 || evidence.Endpoint != "https://source.example/start" {
t.Fatalf("redirect policy inferred or dispatched another request: calls=%d evidence=%+v", transportCalls, evidence)
}
if !evidence.ResponseReceived || evidence.HTTPStatus == nil || *evidence.HTTPStatus != http.StatusFound {
t.Fatalf("original response evidence was lost: %+v", evidence)
}
}
func TestGetWithEvidencePreservesClientAndIsolatesConcurrentCalls(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) {
w.WriteHeader(http.StatusNoContent)
}))
defer server.Close()
redirect := func(*http.Request, []*http.Request) error { return nil }
jar, err := cookiejar.New(nil)
if err != nil {
t.Fatalf("construct cookie jar: %v", err)
}
transport := http.DefaultTransport
supplied := &http.Client{Transport: transport, Timeout: 7 * time.Second, CheckRedirect: redirect, Jar: jar}
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{HTTPClient: supplied})
if err != nil {
t.Fatalf("construct client: %v", err)
}
const calls = 8
evidences := make([]HTTPExecutionEvidence, calls)
errs := make([]error, calls)
var wait sync.WaitGroup
for index := 0; index < calls; index++ {
wait.Add(1)
go func(index int) {
defer wait.Done()
endpoint := server.URL + "/call/" + strconv.Itoa(index)
response, evidence, getErr := client.GetWithEvidence(context.Background(), endpoint, 0)
if response != nil {
response.Body.Close()
}
evidences[index], errs[index] = evidence, getErr
}(index)
}
wait.Wait()
if supplied.Transport != transport || supplied.Timeout != 7*time.Second || supplied.CheckRedirect == nil || supplied.Jar != jar {
t.Fatal("diagnostic GET mutated the supplied HTTP client")
}
for index, evidence := range evidences {
if errs[index] != nil {
t.Fatalf("GET %d: %v", index, errs[index])
}
want := server.URL + "/call/" + strconv.Itoa(index)
if !evidence.RequestObserved || evidence.Endpoint != want {
t.Fatalf("GET %d evidence = %+v, want endpoint %q", index, evidence, want)
}
}
}
func TestGetWithEvidenceAppliesTargetTimeout(t *testing.T) {
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{Timeout: 5 * time.Second, HTTPClient: &http.Client{
Transport: roundTripFunc(func(request *http.Request) (*http.Response, error) {
<-request.Context().Done()
return nil, request.Context().Err()
}),
}})
if err != nil {
t.Fatalf("construct client: %v", err)
}
response, evidence, err := client.GetWithEvidence(context.Background(), "https://diagnostic.example/slow", 1)
if response != nil || !errors.Is(err, context.DeadlineExceeded) {
t.Fatalf("response/error = %v/%v, want nil/deadline exceeded", response, err)
}
if !evidence.DispatchInvoked || !evidence.RequestObserved || evidence.ResponseReceived {
t.Fatalf("timeout evidence = %+v", evidence)
}
}
func responseFor(request *http.Request, status int, location string) *http.Response {
header := make(http.Header)
if location != "" {
header.Set("Location", location)
}
return &http.Response{
StatusCode: status,
Header: header,
Body: io.NopCloser(strings.NewReader("")),
Request: request,
}
}

View File

@@ -101,24 +101,29 @@ func (c *OpenAICompatibleClient) Generate(ctx context.Context, req domain.Genera
}
// PreparedGeneration holds one encoded request and resolved authentication for
// repeated attempts. Each Execute call creates a fresh HTTP request and context.
// repeated attempts. Each execution creates a fresh HTTP request and context.
type PreparedGeneration struct {
endpoint string
payload []byte
apiKey string
timeoutSeconds int
httpClient *http.Client
endpoint string
payload []byte
apiKey string
timeoutSeconds int
httpClient *http.Client
structuredOutputRequested bool
}
// AttemptOutcome describes one HTTP execution without exposing response bodies
// or headers to the orchestration layer.
type AttemptOutcome struct {
Response *domain.GenerateResponse
Err error
StatusCode int
Usage domain.TokenUsage
UsagePresence *domain.TokenUsagePresence
RetryNotBefore time.Time
Response *domain.GenerateResponse
Err error
StatusCode int
Usage domain.TokenUsage
UsagePresence *domain.TokenUsagePresence
RetryNotBefore time.Time
Evidence HTTPExecutionEvidence
StructuredOutputRequested *bool
ResponseDecoded bool
GenerationCompleted bool
}
// PrepareGeneration validates and freezes one built-in request before dispatch.
@@ -152,13 +157,15 @@ func (c *OpenAICompatibleClient) PrepareGeneration(req domain.GenerateRequest) (
payload, err := json.Marshal(wirePayload)
if err != nil {
for key, value := range req.Target.ExtraParams {
if _, extraErr := json.Marshal(value); extraErr != nil {
return nil, fmt.Errorf("%w: extra_params.%s must be JSON-serializable: %v", ErrInvalidRequest, key, extraErr)
}
}
return nil, fmt.Errorf("%w: failed to encode request: %v", ErrRequestFailed, err)
}
apiKey := strings.TrimSpace(req.Target.APIKey)
apiKey := resolveAPIKey(req.Target)
envName := strings.TrimSpace(req.Target.APIKeyEnv)
if apiKey == "" && envName != "" {
apiKey = strings.TrimSpace(os.Getenv(envName))
}
if apiKey == "" && req.Target.APIKeyRequired {
if envName != "" {
return nil, fmt.Errorf("%w: api key environment variable %q is not set", ErrInvalidRequest, envName)
@@ -172,12 +179,36 @@ func (c *OpenAICompatibleClient) PrepareGeneration(req domain.GenerateRequest) (
return &PreparedGeneration{
endpoint: endpoint, payload: payload, apiKey: apiKey,
timeoutSeconds: req.Target.TimeoutSeconds, httpClient: httpClient,
structuredOutputRequested: wireReq.ResponseFormat != nil,
}, nil
}
func resolveAPIKey(target domain.ExecutionTarget) string {
apiKey := strings.TrimSpace(target.APIKey)
if apiKey == "" {
if envName := strings.TrimSpace(target.APIKeyEnv); envName != "" {
apiKey = strings.TrimSpace(os.Getenv(envName))
}
}
return apiKey
}
// InitialEndpoint returns the frozen request URL before redirects.
func (p *PreparedGeneration) InitialEndpoint() string { return p.endpoint }
// Execute dispatches one attempt. The returned response is non-nil only when
// the successful envelope contains a usable first choice.
func (p *PreparedGeneration) Execute(ctx context.Context) AttemptOutcome {
return p.execute(ctx, false)
}
// ExecuteWithEvidence performs one prepared attempt and retains HTTP and
// decoded-generation facts even when a later context check returns an error.
func (p *PreparedGeneration) ExecuteWithEvidence(ctx context.Context) AttemptOutcome {
return p.execute(ctx, true)
}
func (p *PreparedGeneration) execute(ctx context.Context, captureEvidence bool) AttemptOutcome {
requestContext := ctx
if p.timeoutSeconds > 0 {
var cancel context.CancelFunc
@@ -193,13 +224,29 @@ func (p *PreparedGeneration) Execute(ctx context.Context) AttemptOutcome {
httpReq.Header.Set("Authorization", "Bearer "+p.apiKey)
}
httpResp, err := p.httpClient.Do(httpReq)
var (
httpResp *http.Response
outcome AttemptOutcome
)
if captureEvidence {
httpResp, err, outcome.Evidence = doWithEvidence(p.httpClient, httpReq)
if outcome.Evidence.DispatchInvoked {
requested := p.structuredOutputRequested
outcome.StructuredOutputRequested = &requested
}
} else {
httpResp, err = p.httpClient.Do(httpReq)
}
if httpResp != nil && (captureEvidence || err == nil) {
outcome.StatusCode = httpResp.StatusCode
}
if err != nil {
return AttemptOutcome{Err: &requestFailedError{cause: err}}
outcome.Err = &requestFailedError{cause: err}
return outcome
}
receivedAt := time.Now()
defer httpResp.Body.Close()
outcome := AttemptOutcome{StatusCode: httpResp.StatusCode}
outcome.StatusCode = httpResp.StatusCode
if httpResp.StatusCode < 200 || httpResp.StatusCode >= 300 {
outcome.RetryNotBefore = retryNotBefore(httpResp.Header.Values("Retry-After"), receivedAt)
@@ -215,6 +262,11 @@ func (p *PreparedGeneration) Execute(ctx context.Context) AttemptOutcome {
}
if httpResp.ContentLength > maxOpenAIChatResponseBytes {
outcome.Err = openAIChatResponseTooLargeError()
if captureEvidence {
if contextErr := observedContextError(requestContext, ctx, outcome.Err); contextErr != nil {
outcome.Err = contextErr
}
}
return outcome
}
@@ -226,6 +278,7 @@ func (p *PreparedGeneration) Execute(ctx context.Context) AttemptOutcome {
}
return outcome
}
outcome.ResponseDecoded = true
outcome.Usage = domain.TokenUsage{
PromptTokens: tokenCount(wireResp.Usage.PromptTokens), CompletionTokens: tokenCount(wireResp.Usage.CompletionTokens),
TotalTokens: tokenCount(wireResp.Usage.TotalTokens), CachedTokens: tokenCount(wireResp.Usage.PromptTokensDetails.CachedTokens),
@@ -239,14 +292,30 @@ func (p *PreparedGeneration) Execute(ctx context.Context) AttemptOutcome {
if len(wireResp.Choices) == 0 {
outcome.Err = fmt.Errorf("%w: no choices returned", ErrMalformedResponse)
if captureEvidence {
if contextErr := observedContextError(requestContext, ctx, outcome.Err); contextErr != nil {
outcome.Err = contextErr
}
}
return outcome
}
content := wireResp.Choices[0].Message.Content
if content == nil {
outcome.Err = fmt.Errorf("%w: first choice has missing message content", ErrMalformedResponse)
if captureEvidence {
if contextErr := observedContextError(requestContext, ctx, outcome.Err); contextErr != nil {
outcome.Err = contextErr
}
}
return outcome
}
outcome.Response = &domain.GenerateResponse{Content: *content, Usage: outcome.Usage, UsagePresence: outcome.UsagePresence}
outcome.GenerationCompleted = true
if captureEvidence {
if contextErr := observedContextError(requestContext, ctx, nil); contextErr != nil {
outcome.Err = contextErr
}
}
return outcome
}
@@ -446,9 +515,6 @@ func openAIChatRequestPayload(req openAIChatRequest) (map[string]any, error) {
if IsReservedOpenAIChatRequestField(key) {
return nil, fmt.Errorf("extra_params key %q collides with reserved request field", key)
}
if _, err := json.Marshal(value); err != nil {
return nil, fmt.Errorf("extra_params.%s must be JSON-serializable: %w", key, err)
}
out[key] = value
}

View File

@@ -1113,6 +1113,13 @@ func TestOpenAICompatibleClientRejectsInvalidExtraParamsBeforeProviderCall(t *te
}{
{name: "empty key", extraParams: map[string]any{"": "empty"}, want: "key must not be empty"},
{name: "unserializable value", extraParams: map[string]any{"bad": math.Inf(1)}, want: "JSON-serializable"},
{
name: "nested unserializable value",
extraParams: map[string]any{
"nested": map[string]any{"values": []any{"valid", math.NaN()}},
},
want: "extra_params.nested",
},
}
for _, key := range []string{
"model",
@@ -1161,6 +1168,56 @@ func TestOpenAICompatibleClientRejectsInvalidExtraParamsBeforeProviderCall(t *te
}
}
func TestOpenAICompatibleClientClassifiesSchemaEncodingFailures(t *testing.T) {
tests := []struct {
name string
extraParams map[string]any
want error
unwanted error
}{
{
name: "serializable extra keeps full-request failure",
extraParams: map[string]any{"client_option": "accepted"},
want: ErrRequestFailed,
unwanted: ErrInvalidRequest,
},
{
name: "invalid extra keeps invalid-request failure",
extraParams: map[string]any{"nested": map[string]any{"bad": math.Inf(1)}},
want: ErrInvalidRequest,
unwanted: ErrRequestFailed,
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
provider := newRecordingProvider(t)
client := newProviderClient(t, provider, OpenAICompatibleConfig{})
request := ordinaryGenerateRequest()
request.Target.ExtraParams = tc.extraParams
request.StructuredOutput = &domain.StructuredOutputSpec{
Type: domain.StructuredOutputJSONSchema,
JSONSchema: &domain.StructuredOutputJSONSpec{
Name: "invalid-schema",
Strict: true,
Schema: json.RawMessage("{"),
},
}
_, err := client.Generate(context.Background(), request)
if err == nil || !errors.Is(err, tc.want) {
t.Fatalf("error = %v, want identity %v", err, tc.want)
}
if errors.Is(err, tc.unwanted) {
t.Fatalf("error = %v, unexpectedly has identity %v", err, tc.unwanted)
}
if calls := provider.requestCount(); calls != 0 {
t.Fatalf("provider calls = %d, want 0", calls)
}
})
}
}
func checkConfiguredModelFallback(t *testing.T) {
provider := newRecordingProvider(t)
client := newProviderClient(t, provider, OpenAICompatibleConfig{Model: "default-model"})

View File

@@ -0,0 +1,119 @@
package llm
import (
"encoding/json"
"runtime"
"strings"
"testing"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
)
var prepareGenerationBenchmarkSink *PreparedGeneration
func BenchmarkPrepareGeneration(b *testing.B) {
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
BaseURL: "https://api.example.test/v1",
Model: "benchmark-model",
})
if err != nil {
b.Fatalf("construct benchmark client: %v", err)
}
prompt := domain.RenderedPrompt{
Messages: []domain.RenderedMessage{{Role: "user", Content: "Summarize this request."}},
}
tests := []struct {
name string
request domain.GenerateRequest
}{
{
name: "no_extras",
request: domain.GenerateRequest{
Prompt: prompt,
Target: domain.ExecutionTarget{Model: "benchmark-model"},
},
},
{
name: "small_extras",
request: domain.GenerateRequest{
Prompt: prompt,
Target: domain.ExecutionTarget{Model: "benchmark-model", ExtraParams: map[string]any{
"priority": "low",
"attempt": 3,
"enabled": true,
}},
},
},
{
name: "1MiB_string_extra",
request: domain.GenerateRequest{
Prompt: prompt,
Target: domain.ExecutionTarget{Model: "benchmark-model", ExtraParams: map[string]any{
"payload": strings.Repeat("x", 1<<20),
}},
},
},
{
name: "nested_json_extras",
request: domain.GenerateRequest{
Prompt: prompt,
Target: domain.ExecutionTarget{Model: "benchmark-model", ExtraParams: map[string]any{
"options": map[string]any{
"enabled": true,
"limits": []any{2, 4, 8},
"tags": []string{"summary", "offline"},
},
}},
},
},
{
name: "structured_output",
request: domain.GenerateRequest{
Prompt: prompt,
Target: domain.ExecutionTarget{Model: "benchmark-model"},
StructuredOutput: &domain.StructuredOutputSpec{
Type: domain.StructuredOutputJSONSchema,
JSONSchema: &domain.StructuredOutputJSONSpec{
Name: "summary",
Strict: true,
Schema: map[string]any{
"type": "object",
"properties": map[string]any{
"summary": map[string]any{"type": "string"},
"count": map[string]any{"type": "integer"},
},
"required": []string{"summary", "count"},
"additionalProperties": false,
},
},
},
},
},
}
for _, tc := range tests {
b.Run(tc.name, func(b *testing.B) {
prepared, err := client.PrepareGeneration(tc.request)
if err != nil {
b.Fatalf("check prepared request: %v", err)
}
if !json.Valid(prepared.payload) {
b.Fatal("prepared request is not valid JSON")
}
b.ReportAllocs()
b.SetBytes(int64(len(prepared.payload)))
b.ResetTimer()
for range b.N {
prepared, err := client.PrepareGeneration(tc.request)
if err != nil {
b.Fatalf("prepare generation: %v", err)
}
prepareGenerationBenchmarkSink = prepared
}
b.StopTimer()
runtime.KeepAlive(prepareGenerationBenchmarkSink)
})
}
}

View File

@@ -0,0 +1,222 @@
package llm
import (
"context"
"errors"
"io"
"net/http"
"strings"
"testing"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
)
func TestPreparedGenerationEvidenceSeparatesResponseAndGenerationFacts(t *testing.T) {
tests := []struct {
name string
status int
body string
structured bool
wantDecoded bool
wantGeneration bool
wantResponse bool
wantContent string
wantFailure error
wantUsagePresence bool
wantStructuredFact bool
}{
{
name: "valid content",
status: http.StatusOK,
body: `{"choices":[{"message":{"content":"ok"}}],"usage":{"prompt_tokens":0}}`,
structured: true,
wantDecoded: true,
wantGeneration: true,
wantResponse: true,
wantContent: "ok",
wantUsagePresence: true,
wantStructuredFact: true,
},
{
name: "empty content",
status: http.StatusOK,
body: `{"choices":[{"message":{"content":""}}]}`,
wantDecoded: true,
wantGeneration: true,
wantResponse: true,
wantStructuredFact: false,
},
{
name: "decoded envelope without generated content",
status: http.StatusOK,
body: `{"choices":[],"usage":{"prompt_tokens":0}}`,
wantDecoded: true,
wantUsagePresence: true,
wantFailure: ErrMalformedResponse,
wantStructuredFact: false,
},
{
name: "malformed envelope",
status: http.StatusOK,
body: `not json`,
wantFailure: ErrMalformedResponse,
wantStructuredFact: false,
},
{
name: "provider status",
status: http.StatusTooManyRequests,
body: `{"error":{"message":"try again"}}`,
wantFailure: ErrUnexpectedStatus,
wantStructuredFact: false,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
BaseURL: "https://provider.example/v1",
Model: "model",
HTTPClient: &http.Client{Transport: roundTripFunc(func(request *http.Request) (*http.Response, error) {
response := responseFor(request, test.status, "")
response.Body = io.NopCloser(strings.NewReader(test.body))
response.ContentLength = int64(len(test.body))
return response, nil
})},
})
if err != nil {
t.Fatal(err)
}
request := ordinaryGenerateRequest()
if test.structured {
request.StructuredOutput = &domain.StructuredOutputSpec{
Type: domain.StructuredOutputJSONSchema,
JSONSchema: &domain.StructuredOutputJSONSpec{
Name: "smoke",
Schema: map[string]any{"type": "object"},
},
}
}
prepared, err := client.PrepareGeneration(request)
if err != nil {
t.Fatal(err)
}
outcome := prepared.ExecuteWithEvidence(context.Background())
if (outcome.Err != nil) != (test.wantFailure != nil) || (test.wantFailure != nil && !errors.Is(outcome.Err, test.wantFailure)) {
t.Fatalf("error = %v, want failure %v", outcome.Err, test.wantFailure)
}
if !outcome.Evidence.DispatchInvoked || !outcome.Evidence.RequestObserved || !outcome.Evidence.ResponseReceived {
t.Fatalf("response evidence = %+v", outcome.Evidence)
}
if outcome.Evidence.HTTPStatus == nil || *outcome.Evidence.HTTPStatus != test.status {
t.Fatalf("HTTP evidence status = %v, want %d", outcome.Evidence.HTTPStatus, test.status)
}
if outcome.GenerationCompleted != test.wantGeneration {
t.Fatalf("generation completed = %t, want %t", outcome.GenerationCompleted, test.wantGeneration)
}
if outcome.ResponseDecoded != test.wantDecoded {
t.Fatalf("response decoded = %t, want %t", outcome.ResponseDecoded, test.wantDecoded)
}
if (outcome.Response != nil) != test.wantResponse {
t.Fatalf("response presence = %t, want %t", outcome.Response != nil, test.wantResponse)
}
if test.wantResponse && outcome.Response.Content != test.wantContent {
t.Fatalf("generated content = %q, want %q", outcome.Response.Content, test.wantContent)
}
if (outcome.UsagePresence != nil && outcome.UsagePresence.PromptTokens) != test.wantUsagePresence {
t.Fatalf("usage presence = %+v, want prompt token presence %t", outcome.UsagePresence, test.wantUsagePresence)
}
if outcome.StructuredOutputRequested == nil || *outcome.StructuredOutputRequested != test.wantStructuredFact {
t.Fatalf("structured-output evidence = %v, want %t", outcome.StructuredOutputRequested, test.wantStructuredFact)
}
})
}
}
func TestPreparedGenerationEvidenceRetainsDecodedResponseOnCancellation(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
body := `{"choices":[{"message":{"content":"ok"}}],"usage":{"prompt_tokens":0}}`
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
BaseURL: "https://provider.example/v1",
Model: "model",
HTTPClient: &http.Client{Transport: roundTripFunc(func(request *http.Request) (*http.Response, error) {
return &http.Response{
StatusCode: http.StatusOK,
Header: make(http.Header),
Body: &cancelAfterReadBody{Reader: strings.NewReader(body), cancel: cancel},
ContentLength: int64(len(body)),
Request: request,
}, nil
})},
})
if err != nil {
t.Fatal(err)
}
prepared, err := client.PrepareGeneration(ordinaryGenerateRequest())
if err != nil {
t.Fatal(err)
}
outcome := prepared.ExecuteWithEvidence(ctx)
if !errors.Is(outcome.Err, context.Canceled) || !errors.Is(outcome.Err, ErrTransport) {
t.Fatalf("canceled outcome = %+v", outcome)
}
if !outcome.Evidence.ResponseReceived || outcome.Evidence.HTTPStatus == nil || *outcome.Evidence.HTTPStatus != http.StatusOK {
t.Fatalf("response evidence was lost after cancellation: %+v", outcome.Evidence)
}
if !outcome.ResponseDecoded || !outcome.GenerationCompleted || outcome.Response == nil || outcome.Response.Content != "ok" {
t.Fatalf("decoded generation was lost after cancellation: %+v", outcome)
}
if outcome.UsagePresence == nil || !outcome.UsagePresence.PromptTokens || outcome.Usage.PromptTokens != 0 {
t.Fatalf("decoded usage was lost after cancellation: %+v", outcome)
}
}
func TestPreparedGenerationEvidenceRecordsStructuredRequestAtDispatch(t *testing.T) {
transportErr := errors.New("provider unavailable")
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
BaseURL: "https://provider.example/v1",
Model: "model",
HTTPClient: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
return nil, transportErr
})},
})
if err != nil {
t.Fatal(err)
}
request := ordinaryGenerateRequest()
request.StructuredOutput = &domain.StructuredOutputSpec{
Type: domain.StructuredOutputJSONSchema,
JSONSchema: &domain.StructuredOutputJSONSpec{
Name: "smoke",
Schema: map[string]any{"type": "object"},
},
}
prepared, err := client.PrepareGeneration(request)
if err != nil {
t.Fatal(err)
}
outcome := prepared.ExecuteWithEvidence(context.Background())
if !errors.Is(outcome.Err, transportErr) || !outcome.Evidence.DispatchInvoked || !outcome.Evidence.RequestObserved {
t.Fatalf("dispatch outcome = %+v", outcome)
}
if outcome.Evidence.ResponseReceived || outcome.StructuredOutputRequested == nil || !*outcome.StructuredOutputRequested {
t.Fatalf("structured request evidence = %+v, request=%v", outcome.Evidence, outcome.StructuredOutputRequested)
}
}
type cancelAfterReadBody struct {
io.Reader
cancel context.CancelFunc
}
func (b *cancelAfterReadBody) Read(buffer []byte) (int, error) {
n, err := b.Reader.Read(buffer)
if n > 0 {
b.cancel()
}
return n, err
}
func (b *cancelAfterReadBody) Close() error { return nil }

View File

@@ -28,13 +28,13 @@ type Recorder struct {
capacityWait time.Duration
generationDuration time.Duration
retryDelay time.Duration
usage usageTotals
usage UsageAccumulator
}
// Selection contains resolved identities only. Unresolved fields stay empty.
type Selection struct {
PromptID string
PromptVersion string
PromptVersion *string
ProfileID string
BackendID string
Model string
@@ -79,8 +79,7 @@ func New(observer domain.ExecutionObserver) (*Recorder, error) {
}
func newRecorder(runID string, now func() time.Time, observer domain.ExecutionObserver) *Recorder {
return &Recorder{runID: runID, startedAt: now(), now: now, observer: observer,
usage: newUsageTotals()}
return &Recorder{runID: runID, startedAt: now(), now: now, observer: observer}
}
// NewRunID creates the identity shared by observations and successful results.
@@ -104,7 +103,10 @@ func (r *Recorder) Start() {
}
// UpdateSelection replaces the known resolved identities for later events.
func (r *Recorder) UpdateSelection(selection Selection) { r.selection = selection }
func (r *Recorder) UpdateSelection(selection Selection) {
selection.PromptVersion = copyStringPointer(selection.PromptVersion)
r.selection = selection
}
// Attempt records one completed acquisition and optional dispatch.
func (r *Recorder) Attempt(data AttemptData) {
@@ -119,7 +121,7 @@ func (r *Recorder) Attempt(data AttemptData) {
r.lastRepairGeneration = data.Generation
}
r.generationDuration = addDuration(r.generationDuration, data.GenerationDuration)
r.usage.add(data.Usage, data.UsagePresence)
r.usage.Add(data.Usage, data.UsagePresence)
}
event := r.event(domain.ExecutionAttempt)
event.Generation = data.Generation
@@ -171,18 +173,26 @@ func (r *Recorder) Finish(data FinishData) {
if event.Duration < 0 {
event.Duration = 0
}
event.Usage = r.usage.snapshot(r.attempts)
event.Usage = r.usage.Observed()
r.deliver(event)
}
func (r *Recorder) event(kind domain.ExecutionEventKind) domain.ExecutionEvent {
return domain.ExecutionEvent{
Kind: kind, RunID: r.runID, Time: r.now(), PromptID: r.selection.PromptID,
PromptVersion: r.selection.PromptVersion, ProfileID: r.selection.ProfileID,
PromptVersion: copyStringPointer(r.selection.PromptVersion), ProfileID: r.selection.ProfileID,
BackendID: r.selection.BackendID, Model: r.selection.Model,
}
}
func copyStringPointer(value *string) *string {
if value == nil {
return nil
}
copy := *value
return &copy
}
func (r *Recorder) deliver(event domain.ExecutionEvent) {
if r.observer != nil {
r.observer(event)
@@ -243,7 +253,29 @@ type usageTotals struct {
prompt, completion, total, cached, cacheWrite usageField
}
func newUsageTotals() usageTotals {
// UsageAccumulator combines presence-aware usage across dispatched attempts.
// Its zero value represents no dispatched attempts.
type UsageAccumulator struct {
totals usageTotals
attempts uint64
}
// Add includes the usage reported by one dispatched attempt.
func (u *UsageAccumulator) Add(usage domain.TokenUsage, presence *domain.TokenUsagePresence) {
if u.attempts == 0 {
u.totals = completeUsageTotals()
}
u.attempts++
u.totals.add(usage, presence)
}
// Observed returns aggregate counts, marking every field known and complete
// when no attempt has been added.
func (u UsageAccumulator) Observed() domain.ObservedTokenUsage {
return u.totals.snapshot(u.attempts)
}
func completeUsageTotals() usageTotals {
return usageTotals{prompt: usageField{complete: true}, completion: usageField{complete: true},
total: usageField{complete: true}, cached: usageField{complete: true}, cacheWrite: usageField{complete: true}}
}

View File

@@ -50,7 +50,8 @@ func TestRecorderAccountsDispatchedAttemptsAndReportedUsage(t *testing.T) {
var events []domain.ExecutionEvent
recorder := newRecorder("run-1", now, func(event domain.ExecutionEvent) { events = append(events, event) })
recorder.Start()
recorder.UpdateSelection(Selection{PromptID: "prompt", PromptVersion: "v1", ProfileID: "profile", BackendID: "backend", Model: "model"})
version := "v1"
recorder.UpdateSelection(Selection{PromptID: "prompt", PromptVersion: &version, ProfileID: "profile", BackendID: "backend", Model: "model"})
recorder.Attempt(AttemptData{
Generation: 1, Attempt: 1, Dispatched: true,
CapacityWait: 2 * time.Millisecond, GenerationDuration: 10 * time.Millisecond,

View File

@@ -0,0 +1,27 @@
package observation
import (
"testing"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
)
func TestUsageAccumulatorPreservesPresenceAndNoAttemptSemantics(t *testing.T) {
var empty UsageAccumulator
zero := empty.Observed()
if !zero.PromptTokens.Known || !zero.PromptTokens.Complete || zero.PromptTokens.Value != 0 ||
!zero.CacheWriteTokens.Known || !zero.CacheWriteTokens.Complete || zero.CacheWriteTokens.Value != 0 {
t.Fatalf("zero-attempt usage = %+v", zero)
}
var usage UsageAccumulator
usage.Add(domain.TokenUsage{PromptTokens: 0}, &domain.TokenUsagePresence{PromptTokens: true})
usage.Add(domain.TokenUsage{PromptTokens: 3}, &domain.TokenUsagePresence{PromptTokens: true})
observed := usage.Observed()
if observed.PromptTokens.Value != 3 || !observed.PromptTokens.Known || !observed.PromptTokens.Complete {
t.Fatalf("reported prompt usage = %+v", observed.PromptTokens)
}
if observed.CompletionTokens.Known || observed.CompletionTokens.Complete {
t.Fatalf("unreported completion usage = %+v", observed.CompletionTokens)
}
}

View File

@@ -0,0 +1,253 @@
package prompt
import (
"bytes"
"context"
"runtime"
"strconv"
"testing"
"text/template"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
)
const rendererBenchmarkArtifactSize = 256 << 10
var (
renderedPromptBenchmarkSink *domain.RenderedPrompt
renderedPromptsBenchmarkSink []*domain.RenderedPrompt
)
type rendererBenchmarkIdentity struct {
id string
}
type rendererBenchmarkLookup struct {
identity rendererBenchmarkIdentity
templates CompiledTemplates
}
func (lookup rendererBenchmarkLookup) GetCompiledTemplates(id string) (CompiledTemplates, bool) {
if id != lookup.identity.id {
return CompiledTemplates{}, false
}
return lookup.templates, true
}
type compiledRenderingBenchmarkCase struct {
name string
renderer Renderer
definition *domain.PromptDefinition
inputs map[string]*domain.Artifact
vars map[string]string
ctx context.Context
outputBytes int
check func(*testing.B, *domain.RenderedPrompt)
}
func BenchmarkCompiledRendering(b *testing.B) {
body := bytes.Repeat([]byte("x"), rendererBenchmarkArtifactSize)
bodyText := string(body)
small := newCompiledRenderingBenchmarkCase(b,
"small_message", "", []string{"Summary for {{.audience}}."},
&domain.PromptDefinition{
ID: "compiled-render-benchmark", Version: stringPointerForTest("v1"),
Templates: []domain.PromptMessageTemplate{{Role: "user", Content: "Summary for {{.audience}}."}},
}, nil, map[string]string{"audience": "project leads"},
len("Summary for project leads."),
func(b *testing.B, rendered *domain.RenderedPrompt) {
if rendered.SessionID != "" || len(rendered.Messages) != 1 || rendered.Messages[0].Content != "Summary for project leads." {
b.Fatalf("unexpected small compiled render: %+v", rendered)
}
},
)
artifactInputs := map[string]*domain.Artifact{
"document": {Name: "document", Body: body},
}
largeContent := `{{input "document"}}`
largeDefinition := &domain.PromptDefinition{
ID: "compiled-render-benchmark", Version: stringPointerForTest("v1"),
Inputs: []domain.PromptInput{{Name: "document"}},
Templates: []domain.PromptMessageTemplate{{Role: "user", Content: largeContent}},
}
large := newCompiledRenderingBenchmarkCase(b,
"artifact_backed_256KiB", "", []string{largeContent},
largeDefinition, artifactInputs, nil, len(bodyText),
func(b *testing.B, rendered *domain.RenderedPrompt) {
if len(rendered.Messages) != 1 {
b.Fatalf("artifact render message count=%d, want 1", len(rendered.Messages))
}
if rendered.Messages[0].Content != bodyText {
b.Fatalf("artifact render length=%d, want %d", len(rendered.Messages[0].Content), len(bodyText))
}
},
)
repeatedSession := `document-size={{len (input "document")}}`
repeatedMessages := []string{
`{{input "document"}}`,
`{{input "document"}}|{{input "document"}}`,
}
repeatedDefinition := &domain.PromptDefinition{
ID: "compiled-render-benchmark", Version: stringPointerForTest("v1"), SessionID: repeatedSession,
Inputs: []domain.PromptInput{{Name: "document"}},
Templates: []domain.PromptMessageTemplate{
{Role: "system", Content: repeatedMessages[0]},
{Role: "user", Content: repeatedMessages[1]},
},
}
repeatedOutputBytes := len(bodyText)*3 + 1
repeated := newCompiledRenderingBenchmarkCase(b,
"repeated_across_session_and_messages", repeatedSession, repeatedMessages,
repeatedDefinition, artifactInputs, nil, repeatedOutputBytes,
func(b *testing.B, rendered *domain.RenderedPrompt) {
if rendered.SessionID != "document-size="+strconv.Itoa(len(bodyText)) || len(rendered.Messages) != 2 ||
rendered.Messages[0].Content != bodyText || rendered.Messages[1].Content != bodyText+"|"+bodyText {
b.Fatalf("unexpected repeated compiled render: session=%q message lengths=%v", rendered.SessionID, renderedMessageLengths(rendered))
}
},
)
for _, benchmark := range []compiledRenderingBenchmarkCase{small, large, repeated} {
b.Run(benchmark.name, func(b *testing.B) {
rendered, err := benchmark.renderer.Render(benchmark.ctx, benchmark.definition, benchmark.inputs, benchmark.vars)
if err != nil {
b.Fatalf("check compiled renderer output: %v", err)
}
benchmark.check(b, rendered)
b.ReportAllocs()
b.SetBytes(int64(benchmark.outputBytes))
b.ResetTimer()
for range b.N {
rendered, err := benchmark.renderer.Render(benchmark.ctx, benchmark.definition, benchmark.inputs, benchmark.vars)
if err != nil {
b.Fatalf("render compiled template: %v", err)
}
renderedPromptBenchmarkSink = rendered
}
b.StopTimer()
runtime.KeepAlive(renderedPromptBenchmarkSink)
})
}
}
// BenchmarkRetainedRenderedOutput reports post-GC live heap for a fixed batch
// of rendered outputs above the same fixture's setup-only baseline.
func BenchmarkRetainedRenderedOutput(b *testing.B) {
body := bytes.Repeat([]byte("x"), rendererBenchmarkArtifactSize)
bodyText := string(body)
content := `{{input "document"}}`
definition := &domain.PromptDefinition{
ID: "retained-render-benchmark", Version: stringPointerForTest("v1"),
Inputs: []domain.PromptInput{{Name: "document"}},
Templates: []domain.PromptMessageTemplate{{Role: "user", Content: content}},
}
compiled := CompiledTemplates{Messages: []*template.Template{mustCompileRendererBenchmarkTemplate(b, content)}}
renderer := NewCompiledRenderer(rendererBenchmarkLookup{
identity: rendererBenchmarkIdentity{id: definition.ID},
templates: compiled,
})
inputs := map[string]*domain.Artifact{"document": {Name: "document", Body: body}}
ctx := context.Background()
const retainedBatchSize = 4
retained := make([]*domain.RenderedPrompt, retainedBatchSize)
check, err := renderer.Render(ctx, definition, inputs, nil)
if err != nil {
b.Fatalf("check retained-output fixture: %v", err)
}
if len(check.Messages) != 1 || check.Messages[0].Content != bodyText {
b.Fatalf("retained-output fixture returned unexpected content")
}
check = nil
// Repeated -count runs must start from the same setup-only heap baseline.
renderedPromptBenchmarkSink = nil
renderedPromptsBenchmarkSink = nil
runtime.GC()
var setupMemory runtime.MemStats
runtime.ReadMemStats(&setupMemory)
runtime.KeepAlive(renderer)
runtime.KeepAlive(definition)
runtime.KeepAlive(inputs)
b.ReportAllocs()
b.SetBytes(int64(len(bodyText)))
b.ResetTimer()
for iteration := 0; iteration < b.N; iteration++ {
rendered, err := renderer.Render(ctx, definition, inputs, nil)
if err != nil {
b.Fatalf("render retained-output fixture: %v", err)
}
retained[iteration%len(retained)] = rendered
}
b.StopTimer()
for i, rendered := range retained {
if rendered != nil {
continue
}
rendered, err := renderer.Render(ctx, definition, inputs, nil)
if err != nil {
b.Fatalf("fill retained-output fixture: %v", err)
}
retained[i] = rendered
}
renderedPromptsBenchmarkSink = retained
runtime.GC()
var retainedMemory runtime.MemStats
runtime.ReadMemStats(&retainedMemory)
b.ReportMetric(float64(int64(retainedMemory.HeapAlloc)-int64(setupMemory.HeapAlloc)), "retained-B")
runtime.KeepAlive(retained)
runtime.KeepAlive(renderer)
runtime.KeepAlive(definition)
runtime.KeepAlive(inputs)
}
func newCompiledRenderingBenchmarkCase(
b *testing.B,
name string,
session string,
messages []string,
definition *domain.PromptDefinition,
inputs map[string]*domain.Artifact,
vars map[string]string,
outputBytes int,
check func(*testing.B, *domain.RenderedPrompt),
) compiledRenderingBenchmarkCase {
compiled := CompiledTemplates{}
if session != "" {
compiled.Session = mustCompileRendererBenchmarkTemplate(b, session)
}
for _, content := range messages {
compiled.Messages = append(compiled.Messages, mustCompileRendererBenchmarkTemplate(b, content))
}
return compiledRenderingBenchmarkCase{
name: name,
renderer: NewCompiledRenderer(rendererBenchmarkLookup{
identity: rendererBenchmarkIdentity{id: definition.ID},
templates: compiled,
}),
definition: definition, inputs: inputs, vars: vars, ctx: context.Background(),
outputBytes: outputBytes, check: check,
}
}
func mustCompileRendererBenchmarkTemplate(b *testing.B, source string) *template.Template {
b.Helper()
compiled, err := CompileTemplate("benchmark", source)
if err != nil {
b.Fatalf("compile benchmark template: %v", err)
}
return compiled
}
func renderedMessageLengths(rendered *domain.RenderedPrompt) []int {
lengths := make([]int, len(rendered.Messages))
for i, message := range rendered.Messages {
lengths[i] = len(message.Content)
}
return lengths
}

View File

@@ -1,7 +1,6 @@
package prompt
import (
"bytes"
"context"
"errors"
"fmt"
@@ -60,7 +59,7 @@ func (r *goRenderer) Render(ctx context.Context, definition *domain.PromptDefini
var compiled CompiledTemplates
if r.templates != nil {
var found bool
compiled, found = r.templates.GetCompiledTemplates(definition.ID, definition.Version)
compiled, found = r.templates.GetCompiledTemplates(definition.ID)
if !found || len(compiled.Messages) != len(definition.Templates) {
return nil, fmt.Errorf("%w: captured templates are unavailable", ErrRenderFailure)
}
@@ -126,7 +125,7 @@ func (r *goRenderer) Render(ctx context.Context, definition *domain.PromptDefini
if err := ctx.Err(); err != nil {
return nil, err
}
var buf bytes.Buffer
var buf strings.Builder
executeErr := tmpl.Execute(&buf, vars)
if err := ctx.Err(); err != nil {
return nil, err
@@ -235,7 +234,7 @@ func executeSessionTemplate(ctx context.Context, tmpl *template.Template, vars m
if err := ctx.Err(); err != nil {
return "", err
}
var buf bytes.Buffer
var buf strings.Builder
executeErr := tmpl.Execute(&buf, vars)
if err := ctx.Err(); err != nil {
return "", err

View File

@@ -4,8 +4,11 @@ import (
"bytes"
"context"
"errors"
"fmt"
"strings"
"sync"
"testing"
"text/template"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
)
@@ -501,6 +504,76 @@ func TestGoRendererArtifactTextLifecycle(t *testing.T) {
}
}
func TestCompiledRendererKeepsConcurrentRequestsIndependent(t *testing.T) {
const (
promptID = "concurrent-render"
version = "v1"
)
sessionTemplate, err := CompileTemplate("session", "session-{{.request}}")
if err != nil {
t.Fatalf("compile session template: %v", err)
}
messageTemplate, err := CompileTemplate("message", `request={{.request}};document={{input "document"}}`)
if err != nil {
t.Fatalf("compile message template: %v", err)
}
renderer := NewCompiledRenderer(compiledRendererTestLookup{
id: promptID,
templates: CompiledTemplates{
Session: sessionTemplate,
Messages: []*template.Template{messageTemplate},
},
})
definition := &domain.PromptDefinition{
ID: promptID, Version: stringPointerForTest(version), SessionID: "session-{{.request}}",
Inputs: []domain.PromptInput{{Name: "document"}},
Templates: []domain.PromptMessageTemplate{{Role: "user", Content: `request={{.request}};document={{input "document"}}`}},
}
const requestCount = 24
errs := make(chan error, requestCount)
var wait sync.WaitGroup
wait.Add(requestCount)
for i := 0; i < requestCount; i++ {
go func(i int) {
defer wait.Done()
request := fmt.Sprintf("request-%d", i)
document := fmt.Sprintf("document-%d", i)
rendered, err := renderer.Render(context.Background(), definition,
map[string]*domain.Artifact{"document": {Body: []byte(document)}},
map[string]string{"request": request},
)
if err != nil {
errs <- fmt.Errorf("render %s: %w", request, err)
return
}
if rendered.SessionID != "session-"+request || len(rendered.Messages) != 1 ||
rendered.Messages[0].Content != "request="+request+";document="+document {
errs <- fmt.Errorf("render %s returned session %q and messages %#v", request, rendered.SessionID, rendered.Messages)
}
}(i)
}
wait.Wait()
close(errs)
for err := range errs {
t.Error(err)
}
}
type compiledRendererTestLookup struct {
id string
templates CompiledTemplates
}
func (lookup compiledRendererTestLookup) GetCompiledTemplates(id string) (CompiledTemplates, bool) {
if id != lookup.id {
return CompiledTemplates{}, false
}
return lookup.templates, true
}
// BenchmarkGoRendererArtifactReferences measures NewGoRenderer's per-call
// template-parsing fallback rather than the captured compiled path.
func BenchmarkGoRendererArtifactReferences(b *testing.B) {
body := bytes.Repeat([]byte("document content "), (artifactTextChunkSize*4)/len("document content "))
inputs := map[string]*domain.Artifact{

View File

@@ -0,0 +1,978 @@
package prompt
import (
"context"
"sort"
"text/template"
"text/template/parse"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
)
type requirementReachability uint8
const (
requirementDefinite requirementReachability = iota
requirementPossible
requirementInactive
)
type dotIdentity uint8
const (
dotRoot dotIdentity = iota
dotNonRoot
dotUnknown
)
type inputEvidence struct {
name string
kind domain.RequirementKind
declared bool
applicable bool
contentType *string
description *string
explanation string
}
type variableEvidence struct {
name string
kind domain.RequirementKind
applicable bool
explanation string
}
type requirementIssueKey struct {
code domain.RequirementIssueCode
inputName string
variableName string
}
type conditionEvaluation struct {
value any
known bool
unsupported bool
inputNameUnknown bool
missingVariables map[string]struct{}
contentInputs map[string]struct{}
}
func (evaluation conditionEvaluation) truth() bool {
truth, ok := template.IsTrue(evaluation.value)
return evaluation.known && ok && truth
}
func (walker *requirementWalker) evaluateCondition(pipe *parse.PipeNode, reach requirementReachability, dot dotIdentity) conditionEvaluation {
if pipe == nil {
return conditionEvaluation{unsupported: true}
}
if len(pipe.Cmds) == 1 && len(pipe.Decl) == 0 && !pipe.IsAssign {
return walker.evaluateCommand(pipe.Cmds[0], reach, dot)
}
if name, ok := directInputPipeline(pipe); ok {
return walker.evaluateInputName(name, reach, dot)
}
before := walker.dependencies
visitReach := requirementPossible
if reach == requirementInactive {
visitReach = requirementInactive
}
if err := walker.visitPipe(pipe, visitReach, dot); err != nil {
return conditionEvaluation{unsupported: true}
}
return conditionEvaluation{unsupported: walker.dependencies != before}
}
func (walker *requirementWalker) evaluateCommand(command *parse.CommandNode, reach requirementReachability, dot dotIdentity) conditionEvaluation {
if command == nil || len(command.Args) == 0 {
return conditionEvaluation{unsupported: true}
}
identifier, isIdentifier := command.Args[0].(*parse.IdentifierNode)
if isIdentifier {
switch identifier.Ident {
case "input":
if reach == requirementInactive {
if len(command.Args) == 2 {
if name, ok := literalStringValue(command.Args[1]); ok {
walker.recordInput(name, requirementInactive)
return conditionEvaluation{}
}
}
for _, argument := range command.Args[1:] {
_ = walker.visit(argument, requirementInactive, dot)
}
return conditionEvaluation{}
}
if len(command.Args) == 2 {
if name, ok := command.Args[1].(*parse.StringNode); ok {
walker.recordInput(name.Text, reach)
return conditionEvaluation{contentInputs: map[string]struct{}{name.Text: {}}}
}
return walker.evaluateInputName(command.Args[1], reach, dot)
}
for _, argument := range command.Args[1:] {
_ = walker.visit(argument, requirementPossible, dot)
}
return walker.unresolvedInputName(conditionEvaluation{unsupported: true})
case "index":
if len(command.Args) == 3 {
if key, ok := command.Args[2].(*parse.StringNode); ok && isRootMap(command.Args[1], dot, walker.rootKnown) {
walker.recordOptionalVariable(key.Text, reach)
value, supplied := walker.vars[key.Text]
if !supplied {
value = ""
}
return conditionEvaluation{value: value, known: true}
}
}
case "not":
if len(command.Args) == 2 {
evaluation := walker.evaluateValue(command.Args[1], reach, dot)
if evaluation.known {
truth, ok := template.IsTrue(evaluation.value)
if ok {
evaluation.value = !truth
} else {
evaluation.known = false
evaluation.unsupported = true
}
}
return evaluation
}
case "eq", "ne":
if len(command.Args) == 3 {
left := walker.evaluateValue(command.Args[1], reach, dot)
right := walker.evaluateValue(command.Args[2], reach, dot)
result := mergeConditionEvaluations(left, right)
if left.known && right.known {
leftString, leftOK := left.value.(string)
rightString, rightOK := right.value.(string)
if leftOK && rightOK {
equal := leftString == rightString
if identifier.Ident == "ne" {
equal = !equal
}
result.value = equal
result.known = true
return result
}
result.unsupported = true
}
return result
}
}
}
if !isIdentifier && len(command.Args) == 1 {
return walker.evaluateValue(command.Args[0], reach, dot)
}
if reach == requirementInactive {
_ = walker.visitCommand(command, reach, dot)
return conditionEvaluation{}
}
before := walker.dependencies
_ = walker.visitCommand(command, requirementPossible, dot)
return conditionEvaluation{unsupported: walker.dependencies != before}
}
func (walker *requirementWalker) evaluateValue(node parse.Node, reach requirementReachability, dot dotIdentity) conditionEvaluation {
switch node := node.(type) {
case *parse.DotNode:
if dot == dotRoot {
return conditionEvaluation{value: walker.vars, known: true}
}
if dot == dotUnknown {
walker.unresolvedVariableScope(reach)
}
return conditionEvaluation{unsupported: true}
case *parse.StringNode:
return conditionEvaluation{value: node.Text, known: true}
case *parse.BoolNode:
return conditionEvaluation{value: node.True, known: true}
case *parse.NilNode:
return conditionEvaluation{value: nil, known: true}
case *parse.FieldNode:
if dot != dotRoot || len(node.Ident) == 0 {
if dot == dotUnknown {
walker.unresolvedVariableScope(reach)
}
return conditionEvaluation{unsupported: true}
}
name := node.Ident[0]
walker.recordVariable(name, reach)
if len(node.Ident) > 1 {
walker.addIssue("", name, "A chained field access is outside the direct-key scope.")
return conditionEvaluation{unsupported: true}
}
value, supplied := walker.vars[name]
if !supplied {
return conditionEvaluation{missingVariables: map[string]struct{}{name: {}}}
}
return conditionEvaluation{value: value, known: true}
case *parse.VariableNode:
if len(node.Ident) == 1 && node.Ident[0] == "$" && walker.rootKnown {
return conditionEvaluation{value: walker.vars, known: true}
}
if len(node.Ident) < 2 || node.Ident[0] != "$" || !walker.rootKnown {
walker.visitVariable(node, reach)
return conditionEvaluation{unsupported: true}
}
name := node.Ident[1]
walker.recordVariable(name, reach)
if len(node.Ident) > 2 {
walker.addIssue("", name, "A chained field access is outside the direct-key scope.")
return conditionEvaluation{unsupported: true}
}
value, supplied := walker.vars[name]
if !supplied {
return conditionEvaluation{missingVariables: map[string]struct{}{name: {}}}
}
return conditionEvaluation{value: value, known: true}
case *parse.CommandNode:
return walker.evaluateCommand(node, reach, dot)
case *parse.PipeNode:
return walker.evaluateCondition(node, reach, dot)
case *parse.ChainNode:
before := walker.dependencies
visitReach := requirementPossible
if reach == requirementInactive {
visitReach = requirementInactive
}
_ = walker.visit(node.Node, visitReach, dot)
return conditionEvaluation{unsupported: walker.dependencies != before}
default:
return conditionEvaluation{unsupported: true}
}
}
func (walker *requirementWalker) evaluateInputName(node parse.Node, reach requirementReachability, dot dotIdentity) conditionEvaluation {
evaluation := walker.evaluateValue(node, reach, dot)
if evaluation.known {
if name, ok := evaluation.value.(string); ok {
walker.recordInput(name, reach)
return conditionEvaluation{contentInputs: map[string]struct{}{name: {}}}
}
evaluation.unsupported = true
}
// Value analysis has already visited recognizable references, including
// the conservative fallback for unsupported expressions. Do not rescan
// the same subtree: nested computed names would multiply that work.
return walker.unresolvedInputName(evaluation)
}
func (walker *requirementWalker) unresolvedInputName(evaluation conditionEvaluation) conditionEvaluation {
for name := range evaluation.missingVariables {
walker.addIssueCode(domain.RequirementIssueMissingVariable, "", name, "A missing variable prevents resolving an input name.")
}
for name := range evaluation.contentInputs {
walker.addIssueCode(domain.RequirementIssueInputContent, name, "", "Captured input content affects an input name.")
}
walker.addIssueCode(domain.RequirementIssueDynamicInputName, "", "", "An input helper name is not statically available.")
walker.unknownInputEffect = true
evaluation.inputNameUnknown = true
return evaluation
}
func directInputPipeline(pipe *parse.PipeNode) (parse.Node, bool) {
if pipe == nil || len(pipe.Cmds) != 2 || len(pipe.Decl) != 0 || pipe.IsAssign {
return nil, false
}
input := pipe.Cmds[1]
if len(input.Args) != 1 {
return nil, false
}
identifier, ok := input.Args[0].(*parse.IdentifierNode)
if !ok || identifier.Ident != "input" {
return nil, false
}
value := pipe.Cmds[0]
if len(value.Args) > 0 {
valueIdentifier, isValueIdentifier := value.Args[0].(*parse.IdentifierNode)
if isValueIdentifier && valueIdentifier.Ident == "input" && len(value.Args) == 2 {
return value, true
}
}
if len(value.Args) == 1 {
return value.Args[0], true
}
if len(value.Args) == 3 {
function, ok := value.Args[0].(*parse.IdentifierNode)
if ok && function.Ident == "index" {
return value, true
}
}
return nil, false
}
func literalStringValue(node parse.Node) (string, bool) {
switch node := node.(type) {
case *parse.StringNode:
return node.Text, true
case *parse.PipeNode:
if len(node.Cmds) == 1 && len(node.Cmds[0].Args) == 1 && len(node.Decl) == 0 && !node.IsAssign {
return literalStringValue(node.Cmds[0].Args[0])
}
}
return "", false
}
func mergeConditionEvaluations(left, right conditionEvaluation) conditionEvaluation {
result := conditionEvaluation{
unsupported: left.unsupported || right.unsupported,
inputNameUnknown: left.inputNameUnknown || right.inputNameUnknown,
missingVariables: make(map[string]struct{}, len(left.missingVariables)+len(right.missingVariables)),
contentInputs: make(map[string]struct{}, len(left.contentInputs)+len(right.contentInputs)),
}
for name := range left.missingVariables {
result.missingVariables[name] = struct{}{}
}
for name := range right.missingVariables {
result.missingVariables[name] = struct{}{}
}
for name := range left.contentInputs {
result.contentInputs[name] = struct{}{}
}
for name := range right.contentInputs {
result.contentInputs[name] = struct{}{}
}
return result
}
func branchReach(parent requirementReachability, known, truth, selected bool) requirementReachability {
if parent == requirementInactive {
return requirementInactive
}
if !known {
return requirementPossible
}
if truth == selected {
if parent == requirementPossible {
return requirementPossible
}
return requirementDefinite
}
return requirementInactive
}
type requirementWalker struct {
ctx context.Context
vars map[string]string
inputs map[string]*inputEvidence
variables map[string]*variableEvidence
issues map[requirementIssueKey]domain.RequirementIssue
complete bool
rootKnown bool
unknownInputEffect bool
unknownVariableEffect bool
dependencies int
templateSet *template.Template
visitedTrees map[*parse.Tree]bool
}
// AnalyzeRequirements collects the prompt dependencies recognizable from its
// captured syntax. Branch bodies remain conditional until their conditions are
// classified; unsupported dependency-bearing syntax makes the result partial.
func AnalyzeRequirements(
ctx context.Context,
definition *domain.PromptDefinition,
snapshot RequirementSnapshot,
vars map[string]string,
sessionBypassed bool,
) (domain.RequirementInspection, error) {
if err := ctx.Err(); err != nil {
return domain.RequirementInspection{}, err
}
result := domain.RequirementInspection{Complete: true}
if definition != nil {
result.PromptID = definition.ID
result.PromptVersion = copyString(definition.Version)
}
walker := &requirementWalker{
ctx: ctx,
vars: vars,
inputs: make(map[string]*inputEvidence, len(snapshot.Inputs)),
variables: make(map[string]*variableEvidence),
issues: make(map[requirementIssueKey]domain.RequirementIssue),
complete: true,
rootKnown: true,
}
for _, input := range snapshot.Inputs {
if err := ctx.Err(); err != nil {
return domain.RequirementInspection{}, err
}
kind := domain.RequirementOptional
if input.Required {
kind = domain.RequirementRequired
}
walker.inputs[input.Name] = &inputEvidence{
name: input.Name,
kind: kind,
declared: true,
contentType: copyString(input.ContentType),
description: copyString(input.Description),
}
if kind == domain.RequirementOptional {
walker.inputs[input.Name].explanation = "Declared as an optional input."
}
}
for _, message := range snapshot.Templates.Messages {
if err := walker.walkTemplate(message); err != nil {
return domain.RequirementInspection{}, err
}
}
if !sessionBypassed && snapshot.Templates.Session != nil {
if err := walker.walkTemplate(snapshot.Templates.Session); err != nil {
return domain.RequirementInspection{}, err
}
}
if err := ctx.Err(); err != nil {
return domain.RequirementInspection{}, err
}
return walker.result(result), nil
}
func (walker *requirementWalker) walkTemplate(compiled *template.Template) error {
if err := walker.ctx.Err(); err != nil {
return err
}
if compiled == nil || compiled.Tree == nil || compiled.Tree.Root == nil {
walker.addIssue("", "", "A captured template has no parsed root.")
return nil
}
previousSet := walker.templateSet
previousVisited := walker.visitedTrees
walker.templateSet = compiled
walker.visitedTrees = map[*parse.Tree]bool{compiled.Tree: true}
walker.rootKnown = true
err := walker.visit(compiled.Tree.Root, requirementDefinite, dotRoot)
walker.templateSet = previousSet
walker.visitedTrees = previousVisited
return err
}
func (walker *requirementWalker) visit(node parse.Node, reach requirementReachability, dot dotIdentity) error {
if err := walker.ctx.Err(); err != nil {
return err
}
if node == nil {
return nil
}
switch node := node.(type) {
case *parse.ListNode:
if node == nil {
return nil
}
for _, child := range node.Nodes {
if err := walker.visit(child, reach, dot); err != nil {
return err
}
}
case *parse.ActionNode:
return walker.visit(node.Pipe, reach, dot)
case *parse.IfNode:
if reach == requirementInactive {
if err := walker.visit(node.Pipe, requirementInactive, dot); err != nil {
return err
}
if err := walker.visit(node.List, requirementInactive, dot); err != nil {
return err
}
return walker.visit(node.ElseList, requirementInactive, dot)
}
condition := walker.evaluateCondition(node.Pipe, reach, dot)
before := walker.dependencies
thenReach := branchReach(reach, condition.known, condition.truth(), true)
elseReach := branchReach(reach, condition.known, condition.truth(), false)
if err := walker.visit(node.List, thenReach, dot); err != nil {
return err
}
if err := walker.visit(node.ElseList, elseReach, dot); err != nil {
return err
}
if !condition.known && walker.dependencies != before {
walker.recordConditionLimitation(condition)
}
return nil
case *parse.WithNode:
if reach == requirementInactive {
if err := walker.visit(node.Pipe, requirementInactive, dot); err != nil {
return err
}
bodyDot := walker.withDotIdentity(node.Pipe, dot, conditionEvaluation{})
if err := walker.visit(node.List, requirementInactive, bodyDot); err != nil {
return err
}
return walker.visit(node.ElseList, requirementInactive, dot)
}
condition := walker.evaluateCondition(node.Pipe, reach, dot)
before := walker.dependencies
thenReach := branchReach(reach, condition.known, condition.truth(), true)
elseReach := branchReach(reach, condition.known, condition.truth(), false)
bodyDot := walker.withDotIdentity(node.Pipe, dot, condition)
if err := walker.visit(node.List, thenReach, bodyDot); err != nil {
return err
}
if err := walker.visit(node.ElseList, elseReach, dot); err != nil {
return err
}
if !condition.known && walker.dependencies != before {
walker.recordConditionLimitation(condition)
}
return nil
case *parse.RangeNode:
if err := walker.visit(node.Pipe, reach, dot); err != nil {
return err
}
before := walker.dependencies
bodyReach := requirementPossible
if reach == requirementInactive {
bodyReach = requirementInactive
}
if err := walker.visit(node.List, bodyReach, dotNonRoot); err != nil {
return err
}
if err := walker.visit(node.ElseList, bodyReach, dot); err != nil {
return err
}
if reach != requirementInactive && walker.dependencies != before {
walker.addIssue("", "", "Dependencies inside a range are only conservatively enumerated.")
}
case *parse.TemplateNode:
pipeReach := requirementPossible
if reach == requirementInactive {
pipeReach = requirementInactive
}
if err := walker.visit(node.Pipe, pipeReach, dot); err != nil {
return err
}
if reach == requirementInactive {
return nil
}
called := walker.templateSet.Lookup(node.Name)
if called == nil || called.Tree == nil || called.Tree.Root == nil {
walker.addIssue("", "", "An invoked named template is unavailable for syntax inspection.")
return nil
}
if walker.visitedTrees[called.Tree] {
walker.addIssue("", "", "An invoked named template is scanned conservatively.")
return nil
}
walker.visitedTrees[called.Tree] = true
rootKnown := walker.rootKnown
walker.rootKnown = false
walker.unknownVariableEffect = true
err := walker.visit(called.Tree.Root, requirementPossible, dotNonRoot)
walker.rootKnown = rootKnown
if err != nil {
return err
}
walker.addIssue("", "", "An invoked named template is scanned conservatively.")
case *parse.PipeNode:
return walker.visitPipe(node, reach, dot)
case *parse.CommandNode:
return walker.visitCommand(node, reach, dot)
case *parse.FieldNode:
if dot == dotRoot && len(node.Ident) > 0 {
walker.recordVariable(node.Ident[0], reach)
if len(node.Ident) > 1 && reach != requirementInactive {
walker.addIssue("", node.Ident[0], "A chained field access is outside the direct-key scope.")
}
}
if dot == dotUnknown {
walker.unresolvedVariableScope(reach)
}
case *parse.DotNode:
if dot == dotUnknown {
walker.unresolvedVariableScope(reach)
}
case *parse.VariableNode:
walker.visitVariable(node, reach)
case *parse.ChainNode:
before := walker.dependencies
chainReach := requirementPossible
if reach == requirementInactive {
chainReach = requirementInactive
}
if err := walker.visit(node.Node, chainReach, dot); err != nil {
return err
}
if reach != requirementInactive && walker.dependencies != before {
walker.addIssue("", "", "A chained expression is not interpreted.")
}
case *parse.BreakNode, *parse.ContinueNode:
if reach != requirementInactive {
walker.addIssue("", "", "Loop control is outside the supported syntax scope.")
}
}
return nil
}
func (walker *requirementWalker) visitPipe(pipe *parse.PipeNode, reach requirementReachability, dot dotIdentity) error {
if pipe == nil {
return nil
}
if len(pipe.Cmds) == 1 && len(pipe.Decl) == 0 && !pipe.IsAssign {
return walker.visitCommand(pipe.Cmds[0], reach, dot)
}
if name, ok := directInputPipeline(pipe); ok {
if reach == requirementInactive {
if literal, known := literalStringValue(name); known {
walker.recordInput(literal, requirementInactive)
return nil
}
return walker.visit(name, requirementInactive, dot)
}
walker.evaluateInputName(name, reach, dot)
return nil
}
before := walker.dependencies
commandReach := requirementPossible
if reach == requirementInactive {
commandReach = requirementInactive
}
for _, command := range pipe.Cmds {
if err := walker.visitCommand(command, commandReach, dot); err != nil {
return err
}
}
if reach != requirementInactive && walker.dependencies != before {
walker.addIssue("", "", "A pipeline or variable assignment is not interpreted.")
}
if reach != requirementInactive {
for _, declaration := range pipe.Decl {
if len(declaration.Ident) > 0 && declaration.Ident[0] == "$" {
walker.rootKnown = false
walker.unknownVariableEffect = true
walker.addIssue("", "", "The root template binding is reassigned.")
}
}
}
return nil
}
func (walker *requirementWalker) visitCommand(command *parse.CommandNode, reach requirementReachability, dot dotIdentity) error {
if command == nil || len(command.Args) == 0 {
return nil
}
identifier, isIdentifier := command.Args[0].(*parse.IdentifierNode)
if isIdentifier && identifier.Ident == "input" {
if reach == requirementInactive {
if len(command.Args) == 2 {
if name, ok := literalStringValue(command.Args[1]); ok {
walker.recordInput(name, reach)
return nil
}
}
for _, argument := range command.Args[1:] {
if err := walker.visit(argument, requirementInactive, dot); err != nil {
return err
}
}
return nil
}
if len(command.Args) == 2 {
if name, ok := command.Args[1].(*parse.StringNode); ok {
walker.recordInput(name.Text, reach)
return nil
}
walker.evaluateInputName(command.Args[1], reach, dot)
return nil
}
for _, argument := range command.Args[1:] {
if err := walker.visit(argument, requirementPossible, dot); err != nil {
return err
}
}
walker.unresolvedInputName(conditionEvaluation{unsupported: true})
return nil
}
if isIdentifier && identifier.Ident == "index" && len(command.Args) == 3 {
if _, ok := command.Args[2].(*parse.StringNode); ok && isRootMap(command.Args[1], dot, walker.rootKnown) {
key := command.Args[2].(*parse.StringNode).Text
walker.recordOptionalVariable(key, reach)
return nil
}
// A local map may alias the caller root even though its value is not
// tracked. A lookup on it must not vanish from complete discovery.
if variable, ok := unwrapSingleValue(command.Args[1]).(*parse.VariableNode); ok && len(variable.Ident) == 1 {
walker.unresolvedVariableScope(reach)
}
}
if !isIdentifier && len(command.Args) == 1 {
return walker.visit(command.Args[0], reach, dot)
}
before := walker.dependencies
for index, argument := range command.Args {
if isIdentifier && index == 0 {
continue
}
argumentReach := requirementPossible
if reach == requirementInactive {
argumentReach = requirementInactive
}
if err := walker.visit(argument, argumentReach, dot); err != nil {
return err
}
}
if reach != requirementInactive && walker.dependencies != before {
walker.addIssue("", "", "A function call or computed expression is not interpreted.")
}
return nil
}
func (walker *requirementWalker) visitVariable(variable *parse.VariableNode, reach requirementReachability) {
if len(variable.Ident) < 2 {
return
}
if variable.Ident[0] != "$" {
walker.unresolvedVariableScope(reach)
return
}
if !walker.rootKnown {
if reach != requirementInactive {
walker.unknownVariableEffect = true
walker.addIssue("", "", "The root template binding is not known.")
}
return
}
walker.recordVariable(variable.Ident[1], reach)
if len(variable.Ident) > 2 && reach != requirementInactive {
walker.addIssue("", variable.Ident[1], "A chained field access is outside the direct-key scope.")
}
}
func (walker *requirementWalker) unresolvedVariableScope(reach requirementReachability) {
if reach != requirementInactive {
walker.unknownVariableEffect = true
walker.addIssue("", "", "A caller-variable scope is not statically known.")
}
}
func (walker *requirementWalker) withDotIdentity(pipe *parse.PipeNode, dot dotIdentity, condition conditionEvaluation) dotIdentity {
if pipe != nil && len(pipe.Cmds) == 1 && len(pipe.Decl) == 0 && !pipe.IsAssign && len(pipe.Cmds[0].Args) == 1 &&
isRootMap(pipe.Cmds[0].Args[0], dot, walker.rootKnown) {
return dotRoot
}
if condition.known {
if _, root := condition.value.(map[string]string); root {
return dotRoot
}
return dotNonRoot
}
if condition.inputNameUnknown ||
!condition.unsupported && (len(condition.missingVariables) > 0 || len(condition.contentInputs) > 0) {
// Supported caller values and input bodies are strings; supported
// comparisons and negation return booleans, never the root map.
return dotNonRoot
}
return dotUnknown
}
func isRootMap(node parse.Node, dot dotIdentity, rootKnown bool) bool {
switch node := unwrapSingleValue(node).(type) {
case *parse.DotNode:
return dot == dotRoot
case *parse.VariableNode:
return rootKnown && len(node.Ident) == 1 && node.Ident[0] == "$"
default:
return false
}
}
func unwrapSingleValue(node parse.Node) parse.Node {
for {
pipe, ok := node.(*parse.PipeNode)
if !ok || len(pipe.Cmds) != 1 || len(pipe.Decl) != 0 || pipe.IsAssign || len(pipe.Cmds[0].Args) != 1 {
return node
}
node = pipe.Cmds[0].Args[0]
}
}
func (walker *requirementWalker) recordInput(name string, reach requirementReachability) {
evidence := walker.inputs[name]
if evidence == nil {
evidence = &inputEvidence{
name: name, kind: domain.RequirementOptional,
explanation: "Declared as an optional input.",
}
walker.inputs[name] = evidence
}
if reach == requirementInactive {
if !evidence.applicable && evidence.kind != domain.RequirementRequired {
evidence.explanation = "This input is referenced only in a skipped template path."
}
} else {
evidence.applicable = true
walker.promoteInput(evidence, kindForReach(reach))
if evidence.kind == domain.RequirementRequired {
evidence.explanation = ""
} else if reach == requirementPossible {
evidence.explanation = "This reference is not known to be reached on every execution path."
}
}
if reach != requirementInactive {
walker.dependencies++
}
}
func (walker *requirementWalker) recordVariable(name string, reach requirementReachability) {
evidence := walker.variables[name]
if evidence == nil {
evidence = &variableEvidence{name: name, kind: domain.RequirementOptional}
walker.variables[name] = evidence
}
if reach == requirementInactive {
if !evidence.applicable {
evidence.explanation = "This variable is referenced only in a skipped template path."
}
} else {
evidence.applicable = true
walker.promoteVariable(evidence, kindForReach(reach))
if evidence.kind == domain.RequirementRequired {
evidence.explanation = ""
} else if reach == requirementPossible {
evidence.explanation = "This reference is not known to be reached on every execution path."
}
}
if reach != requirementInactive {
walker.dependencies++
}
}
func (walker *requirementWalker) recordOptionalVariable(name string, reach requirementReachability) {
evidence := walker.variables[name]
if evidence == nil {
evidence = &variableEvidence{name: name, kind: domain.RequirementOptional}
walker.variables[name] = evidence
}
if reach == requirementInactive {
if !evidence.applicable {
evidence.explanation = "This variable is referenced only in a skipped template path."
}
} else {
evidence.applicable = true
}
if evidence.kind == domain.RequirementOptional && reach != requirementInactive {
evidence.explanation = "This key is read through an optional map lookup."
}
if reach != requirementInactive {
walker.dependencies++
}
}
func (walker *requirementWalker) promoteInput(evidence *inputEvidence, kind domain.RequirementKind) {
if requirementRank(kind) > requirementRank(evidence.kind) {
evidence.kind = kind
}
if evidence.kind == domain.RequirementRequired {
evidence.explanation = ""
}
}
func (walker *requirementWalker) promoteVariable(evidence *variableEvidence, kind domain.RequirementKind) {
if requirementRank(kind) > requirementRank(evidence.kind) {
evidence.kind = kind
}
if evidence.kind == domain.RequirementRequired {
evidence.explanation = ""
}
}
func kindForReach(reach requirementReachability) domain.RequirementKind {
if reach == requirementPossible {
return domain.RequirementConditional
}
return domain.RequirementRequired
}
func requirementRank(kind domain.RequirementKind) int {
switch kind {
case domain.RequirementRequired:
return 3
case domain.RequirementConditional:
return 2
default:
return 1
}
}
func (walker *requirementWalker) addIssue(inputName, variableName, description string) {
walker.addIssueCode(domain.RequirementIssueUnsupportedConstruct, inputName, variableName, description)
}
func (walker *requirementWalker) addIssueCode(code domain.RequirementIssueCode, inputName, variableName, description string) {
walker.complete = false
issue := domain.RequirementIssue{
Code: code,
InputName: inputName,
VariableName: variableName,
Description: description,
}
key := requirementIssueKey{code: issue.Code, inputName: inputName, variableName: variableName}
walker.issues[key] = issue
}
func (walker *requirementWalker) recordConditionLimitation(condition conditionEvaluation) {
for name := range condition.missingVariables {
walker.addIssueCode(domain.RequirementIssueMissingVariable, "", name, "A missing variable prevents selecting a dependency-bearing branch.")
}
for name := range condition.contentInputs {
walker.addIssueCode(domain.RequirementIssueInputContent, name, "", "Captured input content affects branch selection.")
}
if condition.unsupported && !condition.inputNameUnknown ||
(len(condition.missingVariables) == 0 && len(condition.contentInputs) == 0 && !condition.inputNameUnknown) {
walker.addIssue("", "", "A dependency-bearing branch condition is not resolved by the supported syntax analysis.")
}
}
func (walker *requirementWalker) result(result domain.RequirementInspection) domain.RequirementInspection {
result.Complete = walker.complete
result.Inputs = make([]domain.InputRequirement, 0, len(walker.inputs))
for _, evidence := range walker.inputs {
if walker.unknownInputEffect && evidence.kind != domain.RequirementRequired {
evidence.kind = domain.RequirementConditional
evidence.explanation = "An unresolved input name could refer to this input."
}
result.Inputs = append(result.Inputs, domain.InputRequirement{
Name: evidence.name, Kind: evidence.kind, Declared: evidence.declared,
ContentType: copyString(evidence.contentType), Description: copyString(evidence.description),
Explanation: evidence.explanation,
})
}
sort.Slice(result.Inputs, func(i, j int) bool { return result.Inputs[i].Name < result.Inputs[j].Name })
result.Variables = make([]domain.VariableRequirement, 0, len(walker.variables))
for _, evidence := range walker.variables {
if walker.unknownVariableEffect && evidence.kind != domain.RequirementRequired {
evidence.kind = domain.RequirementConditional
evidence.explanation = "An unresolved caller-variable scope could refer to this variable."
}
_, supplied := walker.vars[evidence.name]
result.Variables = append(result.Variables, domain.VariableRequirement{
Name: evidence.name, Kind: evidence.kind, Supplied: supplied, Explanation: evidence.explanation,
})
}
sort.Slice(result.Variables, func(i, j int) bool { return result.Variables[i].Name < result.Variables[j].Name })
result.Issues = make([]domain.RequirementIssue, 0, len(walker.issues))
for _, issue := range walker.issues {
result.Issues = append(result.Issues, issue)
}
sort.Slice(result.Issues, func(i, j int) bool {
left, right := result.Issues[i], result.Issues[j]
if left.Code != right.Code {
return left.Code < right.Code
}
if left.InputName != right.InputName {
return left.InputName < right.InputName
}
return left.VariableName < right.VariableName
})
return result
}
func copyString(value *string) *string {
if value == nil {
return nil
}
copy := *value
return &copy
}

Some files were not shown because too many files have changed in this diff Show More