Compare commits
193 Commits
v0.11.1
...
a70fd7bb65
| Author | SHA1 | Date | |
|---|---|---|---|
| a70fd7bb65 | |||
| 7091b51c87 | |||
| 8627c751a4 | |||
| 6fff3d822c | |||
| a1f12f3185 | |||
| 165cef7aec | |||
| 0221cd4610 | |||
| dfd8da843e | |||
| 29f104fbc0 | |||
| 8669294cf6 | |||
| c9b623b374 | |||
| 15f79ce7b1 | |||
| aa54b8d1c4 | |||
| 60c58260ec | |||
| 8dfe35a1f0 | |||
| 13b60a72ac | |||
| ee40a1bdd2 | |||
| 473fc1f6d7 | |||
| a10a9053b7 | |||
| ec3245c97e | |||
| 8cbb7c0daf | |||
| d40a010538 | |||
| 0cd833513c | |||
| 5e58b6944b | |||
| 0a85ac8cd1 | |||
| 89af2a1281 | |||
| 683689a4f8 | |||
| ddda2f8767 | |||
| 190ec69345 | |||
| b5866dd749 | |||
| c7490b783b | |||
| c7d1b0dca7 | |||
| 9e5ed08635 | |||
| 4bddbbe8f4 | |||
| 1516631a30 | |||
| cf997c2034 | |||
| a0ec67c256 | |||
| 75891b6561 | |||
| 5dc6370a97 | |||
| 4cda3674c4 | |||
| 5ea6f06a6d | |||
| 116bd0e39b | |||
| 208ff42915 | |||
| 972a103a97 | |||
| 2f735aedf9 | |||
| f4c721d934 | |||
| b7a951685b | |||
| abb2461244 | |||
| b268c55783 | |||
| f0182fb2c9 | |||
| 8d05b594e2 | |||
| edaaecc982 | |||
| 2e81c9e61b | |||
| 2b723760d1 | |||
| c5aed7a451 | |||
| d91afc61a4 | |||
| 6eeb0ee9c0 | |||
| b9c232ee63 | |||
| 7291d23587 | |||
| da1d196dc5 | |||
| b0de49a87a | |||
| 27ff580dae | |||
| ece764f19e | |||
| f91a02548e | |||
| 8fbd058b90 | |||
| c8f9a4def9 | |||
| d8b6c0b471 | |||
| 52cd94fcd1 | |||
| f1f72dfe8f | |||
| d4676bc870 | |||
| 64769883ae | |||
| 635674c066 | |||
| 50bc35516a | |||
| 18309f50db | |||
| 60784e7875 | |||
| f3148bfe52 | |||
| f315620bf7 | |||
| 97357d62b9 | |||
| 0c3651fceb | |||
| 5a5d47d36e | |||
| 143c6e9b3a | |||
| 48e4784fcd | |||
| 78af5069c1 | |||
| 18963266d7 | |||
| d039a019cb | |||
| d1c0281053 | |||
| 870e4ca4e6 | |||
| fd82524977 | |||
| 3f76c75919 | |||
| fe764f72d8 | |||
| 26ab566f38 | |||
| 18ca21fa45 | |||
| e32fdc22f4 | |||
| b02b802427 | |||
| 301192b974 | |||
| f2aeb5eb30 | |||
| a27ee8f7da | |||
| 4d1895c223 | |||
| d7fe70520a | |||
| 7325a34c07 | |||
| 03c44923c8 | |||
| dc0adf5751 | |||
| eae21797f7 | |||
| ea6c7d5f0f | |||
| 389ef26db3 | |||
| 8a5c0de162 | |||
| 80aecb646d | |||
| b7b4000aab | |||
| 11ecb923ba | |||
| f46d5a4591 | |||
| 743e78d514 | |||
| 3743221aa5 | |||
| a8863d49d0 | |||
| c5f733af45 | |||
| c09d2c6e20 | |||
| 6c5dec761c | |||
| 74b91fae99 | |||
| b7ff37118c | |||
| ef012ea1e5 | |||
| 04d5d695e0 | |||
| 78bff91395 | |||
| c63ca5352f | |||
| 3f157150e4 | |||
| f1e1a45a05 | |||
| 996295d5c1 | |||
| 58a5bbe790 | |||
| dd89be3b03 | |||
| ed7c27324d | |||
| 05fe13d80b | |||
| 0eb009ad5a | |||
| 3fde1f6cbb | |||
| 481e2d494c | |||
| eb830bb613 | |||
| dd6d4591ae | |||
| de6443f3ae | |||
| 5a63b649f9 | |||
| c7e22c0dbe | |||
| e69a847cf2 | |||
| 21815ad7c0 | |||
| a051edd82d | |||
| ee0ad99e17 | |||
| bc5a2da0f2 | |||
| 089e824a92 | |||
| 93e8874813 | |||
| cc0b6183bf | |||
| bf472dad8a | |||
| a6e0ffea8f | |||
| e459c7d54b | |||
| 71f6683fda | |||
| 7827ca043b | |||
| b595dca8c7 | |||
| d5f34642d7 | |||
| b1866946d1 | |||
| 05082c6fdf | |||
| 51b072965c | |||
| d5940856be | |||
| 0d163f8165 | |||
| 1889915a81 | |||
| 3ca01ae0b0 | |||
| 6e283d16c3 | |||
| 41b2021d8c | |||
| 3add7024db | |||
| a148a2b1f4 | |||
| 7310ef34a4 | |||
| 4b8169da28 | |||
| 62a4a20529 | |||
| 8c68b28421 | |||
| 66c55b7397 | |||
| 48343300e2 | |||
| f17a38f60b | |||
| aba5e960aa | |||
| d818e0d162 | |||
| ba9c972faa | |||
| 17b6c7bca5 | |||
| 2969a83a71 | |||
| 7f58261498 | |||
| defb240b6e | |||
| b158a8ec2d | |||
| 2b847ac2ae | |||
| 5f7b19403f | |||
| 80131fa160 | |||
| 521291d80f | |||
| 0f43460f84 | |||
| 08409dd472 | |||
| 34f5980973 | |||
| 7b012033a6 | |||
| 37f8dba6c3 | |||
| c7bdaa076b | |||
| 12d6d80024 | |||
| d8147cac49 | |||
| a16c2a38d6 | |||
| 843d07b636 | |||
| dc6fd2752d |
4
.gitignore
vendored
4
.gitignore
vendored
@@ -22,6 +22,9 @@
|
||||
go.work
|
||||
go.work.sum
|
||||
|
||||
# Local ReleaseKit executable
|
||||
.releasekit/
|
||||
|
||||
# env file
|
||||
.env
|
||||
|
||||
@@ -67,4 +70,3 @@ Icon
|
||||
.AppleDesktop
|
||||
Network Trash Folder
|
||||
Temporary Items
|
||||
.apdisk
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
Please review `docs/development.md` for initial orientation in this repository
|
||||
and follow its task-specific reading guide.
|
||||
Start with `docs/development.md` for orientation and use its policy reading guide
|
||||
to select the policies applicable to the task. Do not read the entire suite by
|
||||
default.
|
||||
|
||||
Review and follow `docs/policy/agents.md` before recommending, planning, or
|
||||
implementing any changes to this repository.
|
||||
Read and follow `docs/policy/agents.md` before recommending, planning, or
|
||||
implementing repository changes.
|
||||
|
||||
@@ -16,4 +16,8 @@ but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Lesser General Public License for more details.
|
||||
|
||||
Third-party code and assets are subject to their respective license terms.
|
||||
Policy documents copied or adapted from Standard Repository Policies remain licensed
|
||||
under CC-BY-SA-4.0. See docs/oss/attribution.md for the affected material,
|
||||
upstream credit, and license links.
|
||||
|
||||
Third-party code and assets are subject to their respective license terms.
|
||||
|
||||
35
README.md
35
README.md
@@ -8,7 +8,8 @@ gitea.maximumdirect.net/eric/promptkit
|
||||
```
|
||||
|
||||
The root `promptkit` package provides the supported public engine. Consumers
|
||||
can configure filesystem or in-memory prompt, profile, and schema sources,
|
||||
can configure file, embedded, or programmatic backend, profile, and prompt
|
||||
definitions alongside filesystem or in-memory schemas,
|
||||
prepare requests without generation, run requests with the built-in
|
||||
OpenAI-compatible client, or inject their own model client and artifact reader.
|
||||
|
||||
@@ -20,12 +21,21 @@ Run the maintained offline preparation example from the repository root:
|
||||
go run ./examples/go-library/prepare
|
||||
```
|
||||
|
||||
It loads a repository-local prompt, supplies an in-memory profile and inline
|
||||
input, and prints deterministic preparation metadata without contacting a
|
||||
provider or requiring credentials. Read the
|
||||
[example source](examples/go-library/prepare/main.go), the
|
||||
It composes two explicitly selected synthetic collections, imports
|
||||
source-relative prompt content, supplies an inline input, and prepares through
|
||||
a scoped view. The example also demonstrates collection inventories and
|
||||
namespace-filtered discovery, without contacting a provider or requiring
|
||||
credentials.
|
||||
The [consumer guide](docs/consumers/pkg-promptkit.md) also shows a local backend,
|
||||
ordered file/embedded sources, programmatic mappings, and how to replace an
|
||||
engine when source files change. Applications adapting the prior keyed
|
||||
configuration or versioned-prompt format can consult the
|
||||
[configuration migration guide](docs/consumers/configuration-migration.md).
|
||||
See the [example source](examples/go-library/prepare/main.go), the
|
||||
[Go package consumer guide](docs/consumers/pkg-promptkit.md), and the
|
||||
[framework format reference](docs/formats.md) to build a consumer workflow.
|
||||
The consumer guide also covers explicit, live smoke diagnostics and interpreting
|
||||
their factual reports; they are separate from the offline quickstart.
|
||||
|
||||
Contributors should start with the [development guide](docs/development.md).
|
||||
The [architecture policy](docs/policy/architecture.md) defines the library
|
||||
@@ -33,6 +43,21 @@ boundary and constraints that framework work must preserve.
|
||||
|
||||
## Release Guidance
|
||||
|
||||
The [v0.16.0 release notes](docs/releases/v0.16.0.md) cover explicit smoke
|
||||
diagnostics, configuration collections, scoped engine views, and discovery.
|
||||
Upgrading consumers should review the breaking changes to bundled configuration
|
||||
and prompt version selection and follow the linked migration guide.
|
||||
|
||||
The [v0.15.0 release notes](docs/releases/v0.15.0.md) cover error-only
|
||||
validation of concrete requests with `Engine.ValidateRequest`.
|
||||
|
||||
Consumers upgrading to `v0.12.0` should read the
|
||||
[v0.12.0 release notes](docs/releases/v0.12.0.md) for the intentional ConfigKit
|
||||
configuration and public construction API migration.
|
||||
|
||||
The [v0.11.2 release notes](docs/releases/v0.11.2.md) cover policy alignment
|
||||
and copied-policy attribution updates.
|
||||
|
||||
The [v0.11.1 release notes](docs/releases/v0.11.1.md) cover the MIT-licensed
|
||||
catalog updates and licensing documentation improvements.
|
||||
|
||||
|
||||
@@ -29,15 +29,7 @@ func TestAppendedMessagesComposeFrozenEffectivePrompt(t *testing.T) {
|
||||
{Role: promptkit.RoleUser, Content: "corrective request", CacheControl: cacheControl},
|
||||
}
|
||||
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "output"}}
|
||||
engine, err := promptkit.NewEngine(
|
||||
promptkit.Config{},
|
||||
promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "configured prompt"), "."),
|
||||
promptkit.WithProfiles(promptkit.Profile{ID: "profile", Endpoint: "http://example.test/v1", Model: "model"}),
|
||||
promptkit.WithLLMClient(client),
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("construct engine: %v", err)
|
||||
}
|
||||
engine := newSimpleEngine(t, "configured prompt", promptkit.WithLLMClient(client))
|
||||
request := promptkit.RunRequest{PromptID: "prompt", AppendedMessages: appended}
|
||||
|
||||
plain, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
|
||||
@@ -101,14 +93,9 @@ func TestAppendedMessagesComposeFrozenEffectivePrompt(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestInvalidAppendedMessagesFailBeforeSourceOrModelWork(t *testing.T) {
|
||||
promptSource := &inspectionCountingFS{}
|
||||
func TestInvalidAppendedMessagesFailBeforeModelWork(t *testing.T) {
|
||||
var modelCalls atomic.Int64
|
||||
engine, err := promptkit.NewEngine(
|
||||
promptkit.Config{},
|
||||
promptkit.WithPromptFS(promptSource, "."),
|
||||
promptkit.WithLLMClient(countingLLMClient{calls: &modelCalls}),
|
||||
)
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{}, promptkit.WithLLMClient(countingLLMClient{calls: &modelCalls}))
|
||||
if err != nil {
|
||||
t.Fatalf("construct engine: %v", err)
|
||||
}
|
||||
@@ -136,9 +123,6 @@ func TestInvalidAppendedMessagesFailBeforeSourceOrModelWork(t *testing.T) {
|
||||
}
|
||||
})
|
||||
}
|
||||
if promptSource.opens.Load() != 0 {
|
||||
t.Fatalf("invalid appended message opened prompt sources %d times", promptSource.opens.Load())
|
||||
}
|
||||
if modelCalls.Load() != 0 {
|
||||
t.Fatalf("invalid appended message invoked the model %d times", modelCalls.Load())
|
||||
}
|
||||
|
||||
238
assembled_workflow_benchmark_test.go
Normal file
238
assembled_workflow_benchmark_test.go
Normal file
@@ -0,0 +1,238 @@
|
||||
package promptkit_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"testing/fstest"
|
||||
|
||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
var (
|
||||
assembledPreparedSink *promptkit.PreparedRun
|
||||
assembledResultSink *promptkit.RunResult
|
||||
assembledDetailsSink promptkit.PreparedRun
|
||||
assembledHandleSink *promptkit.PreparedExecution
|
||||
)
|
||||
|
||||
type assembledWorkflowClient struct{ content string }
|
||||
|
||||
func (client assembledWorkflowClient) Generate(
|
||||
context.Context,
|
||||
promptkit.GenerateRequest,
|
||||
) (*promptkit.GenerateResponse, error) {
|
||||
return &promptkit.GenerateResponse{Content: client.content}, nil
|
||||
}
|
||||
|
||||
type assembledWorkflowFixture struct {
|
||||
engine *promptkit.Engine
|
||||
request promptkit.RunRequest
|
||||
expectedMessage string
|
||||
response string
|
||||
}
|
||||
|
||||
func assembledWorkflowSchema(b *testing.B) []byte {
|
||||
b.Helper()
|
||||
properties := make(map[string]any, 256)
|
||||
for index := range 256 {
|
||||
properties[fmt.Sprintf("property-%03d", index)] = map[string]any{"type": "string"}
|
||||
}
|
||||
data, err := json.Marshal(map[string]any{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"type": "object",
|
||||
"properties": properties,
|
||||
"additionalProperties": false,
|
||||
})
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
return data
|
||||
}
|
||||
|
||||
func assembledWorkflowExtras(entries int) map[string]any {
|
||||
providers := make(map[string]any, entries)
|
||||
for index := range entries {
|
||||
providers[fmt.Sprintf("provider-%03d", index)] = map[string]any{
|
||||
"enabled": true,
|
||||
"labels": []any{fmt.Sprintf("tier-%d", index%4), fmt.Sprintf("region-%d", index%3)},
|
||||
"limits": map[string]any{"burst": index + 1, "period": "minute"},
|
||||
"routes": []any{map[string]any{"weight": index + 1, "fallback": true}},
|
||||
}
|
||||
}
|
||||
return map[string]any{"provider_options": providers}
|
||||
}
|
||||
|
||||
func newAssembledWorkflowFixture(b *testing.B, name string) assembledWorkflowFixture {
|
||||
b.Helper()
|
||||
message := "Hello {{.name}}"
|
||||
response := "ok"
|
||||
output := map[string]any{"format": "text", "validation_mode": "none"}
|
||||
profile := map[string]any{"endpoint": "http://example.test/v1", "model": "benchmark-model"}
|
||||
options := []promptkit.Option{promptkit.WithLLMClient(assembledWorkflowClient{content: response})}
|
||||
var expectedMessage string
|
||||
|
||||
switch name {
|
||||
case "small":
|
||||
expectedMessage = "Hello Ada"
|
||||
case "large_message":
|
||||
prefix := strings.Repeat("x", 256*1024)
|
||||
message = prefix + "{{.name}}"
|
||||
expectedMessage = prefix + "Ada"
|
||||
case "mutable_payload":
|
||||
message = `{"subject":"{{.name}}"}`
|
||||
expectedMessage = `{"subject":"Ada"}`
|
||||
response = "{}"
|
||||
profile["extra_params"] = assembledWorkflowExtras(64)
|
||||
output = map[string]any{
|
||||
"format": "json", "validation_mode": "json_schema", "schema_path": "workflow-schema.json",
|
||||
}
|
||||
options[0] = promptkit.WithLLMClient(assembledWorkflowClient{content: response})
|
||||
options = append(options, promptkit.WithSchemaFS(fstest.MapFS{
|
||||
"workflow-schema.json": &fstest.MapFile{Data: assembledWorkflowSchema(b)},
|
||||
}, "."))
|
||||
default:
|
||||
b.Fatalf("unknown assembled workflow fixture %q", name)
|
||||
}
|
||||
|
||||
config := map[string]any{
|
||||
"profiles": map[string]any{"profile": profile},
|
||||
"prompts": map[string]any{"workflow": map[string]any{
|
||||
"version": "v1",
|
||||
"default_profile": "profile",
|
||||
"messages": []any{map[string]any{"role": "user", "content": message}},
|
||||
"output": output,
|
||||
}},
|
||||
}
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
|
||||
Source: configkit.Mapping("assembled workflow benchmark", config),
|
||||
}}}, options...)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
request := promptkit.RunRequest{
|
||||
PromptID: "workflow", Vars: map[string]string{"name": "Ada"},
|
||||
}
|
||||
return assembledWorkflowFixture{
|
||||
engine: engine, request: request, expectedMessage: expectedMessage, response: response,
|
||||
}
|
||||
}
|
||||
|
||||
func verifyAssembledWorkflowFixture(b *testing.B, fixture assembledWorkflowFixture) {
|
||||
b.Helper()
|
||||
ctx := context.Background()
|
||||
prepared, err := fixture.engine.Prepare(ctx, fixture.request)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
if prepared.PromptID != "workflow" || prepared.PromptVersion == nil || *prepared.PromptVersion != "v1" || len(prepared.Messages) != 1 ||
|
||||
prepared.Messages[0].Content != fixture.expectedMessage {
|
||||
b.Fatalf("prepared fixture output is invalid: %+v", prepared)
|
||||
}
|
||||
handle, err := fixture.engine.PrepareExecution(ctx, fixture.request)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
details := handle.Details()
|
||||
if details.PromptID != "workflow" || len(details.Messages) != 1 || details.Messages[0].Content != fixture.expectedMessage {
|
||||
handle.Discard()
|
||||
b.Fatalf("prepared execution details are invalid: %+v", details)
|
||||
}
|
||||
handle.Discard()
|
||||
result, err := fixture.engine.Run(ctx, fixture.request)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
if result.RawOutput != fixture.response {
|
||||
b.Fatalf("run output=%q, want %q", result.RawOutput, fixture.response)
|
||||
}
|
||||
if fixture.response == "{}" && result.Validation.Status != promptkit.ValidationPassed {
|
||||
b.Fatalf("structured response validation=%+v, want passed", result.Validation)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkEnginePreparation(b *testing.B) {
|
||||
for _, name := range []string{"small", "large_message", "mutable_payload"} {
|
||||
b.Run(name, func(b *testing.B) {
|
||||
fixture := newAssembledWorkflowFixture(b, name)
|
||||
verifyAssembledWorkflowFixture(b, fixture)
|
||||
ctx := context.Background()
|
||||
b.Run("prepare", func(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for b.Loop() {
|
||||
prepared, err := fixture.engine.Prepare(ctx, fixture.request)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
assembledPreparedSink = prepared
|
||||
}
|
||||
b.StopTimer()
|
||||
assembledPreparedSink = nil
|
||||
})
|
||||
b.Run("prepared_execution_creation_and_discard", func(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for b.Loop() {
|
||||
handle, err := fixture.engine.PrepareExecution(ctx, fixture.request)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
handle.Discard()
|
||||
assembledHandleSink = handle
|
||||
}
|
||||
b.StopTimer()
|
||||
assembledHandleSink = nil
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkEngineRun(b *testing.B) {
|
||||
for _, name := range []string{"small", "large_message", "mutable_payload"} {
|
||||
b.Run(name, func(b *testing.B) {
|
||||
fixture := newAssembledWorkflowFixture(b, name)
|
||||
verifyAssembledWorkflowFixture(b, fixture)
|
||||
ctx := context.Background()
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for b.Loop() {
|
||||
result, err := fixture.engine.Run(ctx, fixture.request)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
assembledResultSink = result
|
||||
}
|
||||
b.StopTimer()
|
||||
assembledResultSink = nil
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkPreparedExecutionDetails(b *testing.B) {
|
||||
for _, name := range []string{"small", "large_message", "mutable_payload"} {
|
||||
b.Run(name, func(b *testing.B) {
|
||||
fixture := newAssembledWorkflowFixture(b, name)
|
||||
verifyAssembledWorkflowFixture(b, fixture)
|
||||
handle, err := fixture.engine.PrepareExecution(context.Background(), fixture.request)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
defer handle.Discard()
|
||||
initial := handle.Details()
|
||||
if len(initial.Messages) != 1 || initial.Messages[0].Content != fixture.expectedMessage {
|
||||
b.Fatalf("prepared details fixture is invalid: %+v", initial)
|
||||
}
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for b.Loop() {
|
||||
assembledDetailsSink = handle.Details()
|
||||
}
|
||||
b.StopTimer()
|
||||
assembledDetailsSink = promptkit.PreparedRun{}
|
||||
})
|
||||
}
|
||||
}
|
||||
202
backends.go
202
backends.go
@@ -1,107 +1,131 @@
|
||||
package promptkit
|
||||
|
||||
import (
|
||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/backend"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
)
|
||||
|
||||
// BackendOpenRouter is the reserved ID of Promptkit's built-in OpenRouter
|
||||
// backend.
|
||||
// BackendOpenRouter is the conventional ID for an OpenRouter backend.
|
||||
const BackendOpenRouter = backend.OpenRouterID
|
||||
|
||||
// BackendRakestrawHome is the reserved ID of Promptkit's built-in
|
||||
// Rakestrawhome backend.
|
||||
// BackendRakestrawHome is the conventional ID for a Rakestrawhome backend.
|
||||
const BackendRakestrawHome = backend.RakestrawHomeID
|
||||
|
||||
// BackendLocal is the case-sensitive conventional ID used by [LocalBackend].
|
||||
// It is not a built-in or reserved backend and must be registered with
|
||||
// [WithBackend].
|
||||
// BackendLocal is a conventional ID for a caller-configured local backend.
|
||||
const BackendLocal = "local"
|
||||
|
||||
// Backend configures one engine-scoped OpenAI-compatible backend.
|
||||
//
|
||||
// Backend has no stable JSON representation. Use keyed literals so additions
|
||||
// to this configuration value do not break source compatibility.
|
||||
type Backend struct {
|
||||
// ID is the stable, case-sensitive registry key. NewEngine trims it and
|
||||
// requires a non-blank value. Built-in backend IDs are reserved.
|
||||
ID string
|
||||
// Endpoint is the OpenAI-compatible base endpoint. NewEngine trims it and
|
||||
// requires an absolute HTTP or HTTPS URL with a host and without user
|
||||
// information, a query string, or a fragment. Paths are allowed.
|
||||
Endpoint string
|
||||
// APIKeyEnv optionally names an environment lookup source for an API key.
|
||||
// NewEngine trims it and requires the portable form [A-Za-z_][A-Za-z0-9_]*.
|
||||
// A direct RunRequest.APIKey takes precedence. When no usable credential is
|
||||
// available, the built-in client omits Authorization; injected clients own
|
||||
// their own credential-resolution behavior. Store only the name, never a
|
||||
// credential value.
|
||||
APIKeyEnv string
|
||||
// ExtraParams contains backend-wide request defaults. Values must be
|
||||
// JSON-compatible, finite, acyclic, and keyed by non-empty strings. Keys
|
||||
// must not be model, session_id, messages, temperature, max_tokens, top_p,
|
||||
// service_tier, reasoning_effort, or response_format. An empty map supplies
|
||||
// no defaults. NewEngine deeply copies the map and rejects excessively deep
|
||||
// or large values for safety.
|
||||
ExtraParams map[string]any
|
||||
// ConcurrencyLimit is the maximum number of simultaneous model-generation
|
||||
// calls allowed for this backend within one Engine. Zero leaves the backend
|
||||
// unlimited. A negative value makes NewEngine fail with ErrInvalidConfig.
|
||||
ConcurrencyLimit int
|
||||
// QueueCapacity controls how many additional Run or RunPrepared calls may
|
||||
// be admitted beyond ConcurrencyLimit. Nil uses 1024 when ConcurrencyLimit
|
||||
// is positive; a pointer uses its exact value, including zero. The pointed-to
|
||||
// value must be non-negative, and QueueCapacity must be nil when
|
||||
// ConcurrencyLimit is zero. Their sum must fit in an int. WithBackend copies
|
||||
// the value and does not retain the pointer.
|
||||
// BackendConfig is a presence-aware convenience value for a keyed backend
|
||||
// definition. Nil pointer fields are omitted, while non-nil pointers preserve
|
||||
// explicit zero or blank values. A nil ExtraParams map is omitted; a non-nil
|
||||
// empty map supplies an empty mapping. A nil SmokeTests pointer is omitted; a
|
||||
// non-nil pointer supplies a smoke_tests mapping. Use raw ConfigKit Mapping
|
||||
// inputs for explicit null. BackendConfig has no stable JSON representation.
|
||||
type BackendConfig struct {
|
||||
// Endpoint is the required absolute HTTP(S) base URL after all layers merge.
|
||||
Endpoint *string
|
||||
// APIKeyEnv optionally names an environment variable, not a credential.
|
||||
APIKeyEnv *string
|
||||
// ConcurrencyLimit is zero for unlimited generation; negative is invalid.
|
||||
ConcurrencyLimit *int
|
||||
// QueueCapacity defaults to 1024 for positive concurrency when omitted.
|
||||
// An explicit value, including zero, requires positive concurrency.
|
||||
QueueCapacity *int
|
||||
// ExtraParams supplies JSON-compatible provider request defaults.
|
||||
ExtraParams map[string]any
|
||||
// SmokeTests supplies optional diagnostic defaults and HTTP probe settings.
|
||||
SmokeTests *BackendSmokeTestConfig
|
||||
}
|
||||
|
||||
// LocalBackend returns a caller-owned Backend for a conventional local
|
||||
// OpenAI-compatible endpoint. It sets ID to BackendLocal and copies endpoint
|
||||
// and concurrencyLimit into Endpoint and ConcurrencyLimit without
|
||||
// normalization or validation. APIKeyEnv, ExtraParams, and QueueCapacity keep
|
||||
// their zero values.
|
||||
//
|
||||
// LocalBackend does not read environment variables, register the value, or
|
||||
// mutate engine or package state. Supply the returned value through
|
||||
// [WithBackend]; [NewEngine] then applies the ordinary backend validation and
|
||||
// concurrency semantics, including default queue capacity for a positive
|
||||
// limit, unlimited behavior for zero, and ErrInvalidConfig for a negative
|
||||
// limit.
|
||||
func LocalBackend(endpoint string, concurrencyLimit int) Backend {
|
||||
return Backend{
|
||||
ID: BackendLocal,
|
||||
Endpoint: endpoint,
|
||||
ConcurrencyLimit: concurrencyLimit,
|
||||
}
|
||||
// BackendSmokeTestConfig contains optional diagnostic settings associated with
|
||||
// a backend. Nil nested fields are omitted by BackendSource.
|
||||
type BackendSmokeTestConfig struct {
|
||||
// DefaultProfile is the exact profile ID selected as the backend's
|
||||
// diagnostic inference default. A supplied value must not be blank; the ID
|
||||
// is resolved only when that default is selected.
|
||||
DefaultProfile *string
|
||||
// Reachability optionally selects a full HTTP destination for reachability
|
||||
// diagnostics. A nil endpoint uses the effective backend endpoint.
|
||||
Reachability *ReachabilityProbeConfig
|
||||
// Authentication optionally declares a named authentication diagnostic.
|
||||
// A supplied object requires Kind.
|
||||
Authentication *AuthenticationProbeConfig
|
||||
}
|
||||
|
||||
// WithBackend adds one Backend registration to the constructed Engine.
|
||||
//
|
||||
// Registrations accumulate in option order. Every normalized ID must be unique
|
||||
// across consumer registrations and built-ins; a duplicate or invalid
|
||||
// definition makes NewEngine fail with ErrInvalidConfig. Built-in IDs,
|
||||
// including [BackendOpenRouter] and [BackendRakestrawHome], cannot be
|
||||
// replaced. The immutable registration is scoped to the resulting Engine and
|
||||
// cannot be enumerated, replaced, removed, or mutated after construction.
|
||||
// WithBackend does not install package-global state.
|
||||
func WithBackend(backend Backend) Option {
|
||||
queueCapacity := 0
|
||||
queueCapacitySet := backend.QueueCapacity != nil
|
||||
if queueCapacitySet {
|
||||
queueCapacity = *backend.QueueCapacity
|
||||
}
|
||||
return optionFunc(func(options *engineOptions) error {
|
||||
options.internal.AddBackend(domain.Backend{
|
||||
ID: backend.ID,
|
||||
Endpoint: backend.Endpoint,
|
||||
APIKeyEnv: backend.APIKeyEnv,
|
||||
ExtraParams: backend.ExtraParams,
|
||||
ConcurrencyLimit: backend.ConcurrencyLimit,
|
||||
QueueCapacity: queueCapacity,
|
||||
QueueCapacitySet: queueCapacitySet,
|
||||
})
|
||||
return nil
|
||||
})
|
||||
// ReachabilityProbeConfig configures an optional reachability destination.
|
||||
type ReachabilityProbeConfig struct {
|
||||
// Endpoint is an absolute HTTP(S) URL, including any desired path or query.
|
||||
// Nil selects the effective backend endpoint; an explicit blank value is
|
||||
// invalid.
|
||||
Endpoint *string
|
||||
}
|
||||
|
||||
// AuthenticationProbeConfig configures an optional named authentication
|
||||
// diagnostic.
|
||||
type AuthenticationProbeConfig struct {
|
||||
// Kind selects the authentication adapter. Supported values are "none" and
|
||||
// "openrouter_key"; surrounding whitespace and case are normalized.
|
||||
Kind *string
|
||||
// Endpoint is an absolute HTTP(S) URL, including any desired path or query.
|
||||
// Nil selects the adapter's backend-relative destination; an explicit blank
|
||||
// value is invalid.
|
||||
Endpoint *string
|
||||
}
|
||||
|
||||
// BackendSource creates a labeled ConfigKit mapping from keyed backend values.
|
||||
// It encodes fields without validating them; NewEngine validates the final
|
||||
// effective configuration after all selected source layers are combined.
|
||||
// Mapping captures a copy of supported input values at creation; unsupported
|
||||
// values fail when the source is loaded.
|
||||
func BackendSource(label string, backends map[string]BackendConfig) configkit.Source {
|
||||
definitions := make(map[string]any, len(backends))
|
||||
for id, config := range backends {
|
||||
fields := make(map[string]any)
|
||||
if config.Endpoint != nil {
|
||||
fields["endpoint"] = *config.Endpoint
|
||||
}
|
||||
if config.APIKeyEnv != nil {
|
||||
fields["api_key_env"] = *config.APIKeyEnv
|
||||
}
|
||||
if config.ConcurrencyLimit != nil {
|
||||
fields["concurrency_limit"] = *config.ConcurrencyLimit
|
||||
}
|
||||
if config.QueueCapacity != nil {
|
||||
fields["queue_capacity"] = *config.QueueCapacity
|
||||
}
|
||||
if config.ExtraParams != nil {
|
||||
fields["extra_params"] = providerMapForMapping(config.ExtraParams)
|
||||
}
|
||||
if config.SmokeTests != nil {
|
||||
smokeTests := make(map[string]any)
|
||||
if config.SmokeTests.DefaultProfile != nil {
|
||||
smokeTests["default_profile"] = *config.SmokeTests.DefaultProfile
|
||||
}
|
||||
if config.SmokeTests.Reachability != nil {
|
||||
reachability := make(map[string]any)
|
||||
if config.SmokeTests.Reachability.Endpoint != nil {
|
||||
reachability["endpoint"] = *config.SmokeTests.Reachability.Endpoint
|
||||
}
|
||||
smokeTests["reachability"] = reachability
|
||||
}
|
||||
if config.SmokeTests.Authentication != nil {
|
||||
authentication := make(map[string]any)
|
||||
if config.SmokeTests.Authentication.Kind != nil {
|
||||
authentication["kind"] = *config.SmokeTests.Authentication.Kind
|
||||
}
|
||||
if config.SmokeTests.Authentication.Endpoint != nil {
|
||||
authentication["endpoint"] = *config.SmokeTests.Authentication.Endpoint
|
||||
}
|
||||
smokeTests["authentication"] = authentication
|
||||
}
|
||||
fields["smoke_tests"] = smokeTests
|
||||
}
|
||||
definitions[id] = fields
|
||||
}
|
||||
return configkit.Mapping(label, map[string]any{"backends": definitions})
|
||||
}
|
||||
|
||||
// LocalBackend supplies endpoint and concurrency pointers for a caller-chosen
|
||||
// backend ID; it does not select an ID or validate either value.
|
||||
func LocalBackend(endpoint string, concurrencyLimit int) BackendConfig {
|
||||
return BackendConfig{Endpoint: &endpoint, ConcurrencyLimit: &concurrencyLimit}
|
||||
}
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
@@ -150,17 +151,7 @@ func TestUnlimitedBackendsRetainInjectedClientConcurrency(t *testing.T) {
|
||||
{
|
||||
name: "endpoint-only profile",
|
||||
configure: func(t *testing.T, client promptkit.LLMClient) *promptkit.Engine {
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{},
|
||||
promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."),
|
||||
promptkit.WithProfiles(promptkit.Profile{
|
||||
ID: "profile", Endpoint: "http://endpoint.example/v1", Model: "model",
|
||||
}),
|
||||
promptkit.WithLLMClient(client),
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("construct endpoint-only engine: %v", err)
|
||||
}
|
||||
return engine
|
||||
return newSimpleEngine(t, "message", promptkit.WithLLMClient(client))
|
||||
},
|
||||
},
|
||||
}
|
||||
@@ -250,27 +241,35 @@ func newBackendCapacityEngine(
|
||||
reader promptkit.ArtifactReader,
|
||||
) *promptkit.Engine {
|
||||
t.Helper()
|
||||
promptFS := contractPromptFS("prompt", "profile", "message")
|
||||
promptID, content := "prompt", "message"
|
||||
var inputs []any
|
||||
if reader != nil {
|
||||
promptFS = contractInputPromptFS()
|
||||
promptID, content = "input-prompt", `{{input "input"}}`
|
||||
inputs = []any{map[string]any{"name": "input", "required": true}}
|
||||
}
|
||||
options := []promptkit.Option{
|
||||
promptkit.WithPromptFS(promptFS, "."),
|
||||
promptkit.WithBackend(promptkit.Backend{
|
||||
ID: "limited",
|
||||
Endpoint: "http://backend.example/v1",
|
||||
ConcurrencyLimit: limit,
|
||||
QueueCapacity: queueCapacity,
|
||||
}),
|
||||
promptkit.WithProfiles(promptkit.Profile{
|
||||
ID: "profile", BackendID: "limited", Model: "model",
|
||||
}),
|
||||
promptkit.WithLLMClient(client),
|
||||
backend := map[string]any{"endpoint": "http://backend.example/v1", "concurrency_limit": limit}
|
||||
if queueCapacity != nil {
|
||||
backend["queue_capacity"] = *queueCapacity
|
||||
}
|
||||
definition := map[string]any{
|
||||
"default_profile": "profile",
|
||||
"messages": []any{map[string]any{"role": "user", "content": content}},
|
||||
"output": map[string]any{"format": "text", "validation_mode": "none"},
|
||||
}
|
||||
if inputs != nil {
|
||||
definition["inputs"] = inputs
|
||||
}
|
||||
options := []promptkit.Option{promptkit.WithLLMClient(client)}
|
||||
if reader != nil {
|
||||
options = append(options, promptkit.WithArtifactReader(reader))
|
||||
}
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{}, options...)
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
|
||||
Source: configkit.Mapping("capacity fixture", map[string]any{
|
||||
"backends": map[string]any{"limited": backend},
|
||||
"profiles": map[string]any{"profile": map[string]any{"backend": "limited", "model": "model"}},
|
||||
"prompts": map[string]any{promptID: promptDefinitionWithVersion("1", definition)},
|
||||
}),
|
||||
}}}, options...)
|
||||
if err != nil {
|
||||
t.Fatalf("construct capacity engine: %v", err)
|
||||
}
|
||||
|
||||
243
configuration_diff_contract_test.go
Normal file
243
configuration_diff_contract_test.go
Normal file
@@ -0,0 +1,243 @@
|
||||
package promptkit_test
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
func TestConfigurationInspectionDiffsCapturedValues(t *testing.T) {
|
||||
left := comparisonInspection(t, "earlier-model")
|
||||
right := comparisonInspection(t, "later-model")
|
||||
|
||||
differences, err := left.Diff(right)
|
||||
if err != nil {
|
||||
t.Fatalf("compare captured configurations: %v", err)
|
||||
}
|
||||
if len(differences) != 1 {
|
||||
t.Fatalf("difference count = %d, want 1: %+v", len(differences), differences)
|
||||
}
|
||||
difference := differences[0]
|
||||
if got, want := difference.Path.String(), "/profiles/offline/model"; got != want {
|
||||
t.Fatalf("difference path = %q, want %q", got, want)
|
||||
}
|
||||
if !strings.Contains(difference.Left.YAML, "earlier-model") || !strings.Contains(difference.Right.YAML, "later-model") {
|
||||
t.Fatalf("difference snapshots = (%q, %q)", difference.Left.YAML, difference.Right.YAML)
|
||||
}
|
||||
|
||||
reversed, err := right.Diff(left)
|
||||
if err != nil {
|
||||
t.Fatalf("compare reversed configurations: %v", err)
|
||||
}
|
||||
if len(reversed) != 1 || reversed[0].Path.String() != difference.Path.String() ||
|
||||
!strings.Contains(reversed[0].Left.YAML, "later-model") ||
|
||||
!strings.Contains(reversed[0].Right.YAML, "earlier-model") {
|
||||
t.Fatalf("reversed difference = %+v", reversed)
|
||||
}
|
||||
|
||||
equal := comparisonInspection(t, "earlier-model")
|
||||
if equal, err := left.Diff(equal); err != nil || len(equal) != 0 {
|
||||
t.Fatalf("compare equal configurations = (%+v, %v), want no differences", equal, err)
|
||||
}
|
||||
if equal, err := left.Diff(left); err != nil || len(equal) != 0 {
|
||||
t.Fatalf("compare view with itself = (%+v, %v), want no differences", equal, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigurationInspectionDiffRejectsZeroViews(t *testing.T) {
|
||||
valid := comparisonInspection(t, "current-model")
|
||||
var zero promptkit.ConfigurationInspection
|
||||
|
||||
for _, comparison := range []struct {
|
||||
name string
|
||||
left, right promptkit.ConfigurationInspection
|
||||
}{
|
||||
{name: "zero left", left: zero, right: valid},
|
||||
{name: "zero right", left: valid, right: zero},
|
||||
{name: "both zero", left: zero, right: zero},
|
||||
} {
|
||||
t.Run(comparison.name, func(t *testing.T) {
|
||||
differences, err := comparison.left.Diff(comparison.right)
|
||||
if !errors.Is(err, promptkit.ErrInvalidConfig) || len(differences) != 0 {
|
||||
t.Fatalf("Diff() = (%+v, %v), want ErrInvalidConfig and no records", differences, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigurationInspectionDiffIgnoresSourceHistory(t *testing.T) {
|
||||
left := comparisonInspectionFrom(t, "earlier source", "same-model", nil)
|
||||
right := comparisonInspectionFrom(t, "later source", "same-model", nil)
|
||||
if !hasSourceLabel(left, "earlier source") || !hasSourceLabel(right, "later source") {
|
||||
t.Fatal("comparison fixtures did not retain their distinct source labels")
|
||||
}
|
||||
|
||||
differences, err := left.Diff(right)
|
||||
if err != nil || len(differences) != 0 {
|
||||
t.Fatalf("compare equal effective values with different sources = (%+v, %v), want no differences", differences, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigurationInspectionDiffReportsPresenceAndSequences(t *testing.T) {
|
||||
left := comparisonInspectionFrom(t, "before", "same-model", map[string]any{
|
||||
"sequence": []any{"earlier", "unchanged"},
|
||||
})
|
||||
right := comparisonInspectionFrom(t, "after", "same-model", map[string]any{
|
||||
"nullable": nil,
|
||||
"sequence": []any{"later", "unchanged"},
|
||||
})
|
||||
|
||||
differences, err := left.Diff(right)
|
||||
if err != nil {
|
||||
t.Fatalf("compare presence and sequence values: %v", err)
|
||||
}
|
||||
if len(differences) != 2 {
|
||||
t.Fatalf("difference count = %d, want 2: %+v", len(differences), differences)
|
||||
}
|
||||
byPath := differencesByPath(differences)
|
||||
nullable, ok := byPath["/backends/local/extra_params/nullable"]
|
||||
if !ok || nullable.Left.Present || !nullable.Right.Present || strings.TrimSpace(nullable.Right.YAML) != "null" {
|
||||
t.Fatalf("absent-to-null difference = (%+v, %t)", nullable, ok)
|
||||
}
|
||||
sequence, ok := byPath["/backends/local/extra_params/sequence"]
|
||||
if !ok || !sequence.Left.Present || !sequence.Right.Present ||
|
||||
!strings.Contains(sequence.Left.YAML, "earlier") || !strings.Contains(sequence.Right.YAML, "later") {
|
||||
t.Fatalf("atomic sequence difference = (%+v, %t)", sequence, ok)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigurationInspectionDiffProtectsSensitiveValuesAndReturnedRecords(t *testing.T) {
|
||||
left := comparisonInspectionFrom(t, "before", "same-model", map[string]any{
|
||||
"token": "left-secret-sentinel",
|
||||
"visible": "before-value",
|
||||
})
|
||||
right := comparisonInspectionFrom(t, "after", "same-model", map[string]any{
|
||||
"token": "right-secret-sentinel",
|
||||
"visible": "after-value",
|
||||
})
|
||||
|
||||
differences, err := left.Diff(right)
|
||||
if err != nil {
|
||||
t.Fatalf("compare sensitive values: %v", err)
|
||||
}
|
||||
if len(differences) != 2 {
|
||||
t.Fatalf("difference count = %d, want 2: %+v", len(differences), differences)
|
||||
}
|
||||
byPath := differencesByPath(differences)
|
||||
secret, ok := byPath["/backends/local/extra_params/token"]
|
||||
if !ok || !secret.Left.Redacted || !secret.Right.Redacted || !secret.HiddenChanged {
|
||||
t.Fatalf("sensitive difference = (%+v, %t)", secret, ok)
|
||||
}
|
||||
if strings.Contains(secret.Left.YAML, "left-secret-sentinel") || strings.Contains(secret.Left.YAML, "right-secret-sentinel") ||
|
||||
strings.Contains(secret.Right.YAML, "left-secret-sentinel") || strings.Contains(secret.Right.YAML, "right-secret-sentinel") {
|
||||
t.Fatalf("sensitive snapshots disclosed a value: %+v", secret)
|
||||
}
|
||||
visible, ok := byPath["/backends/local/extra_params/visible"]
|
||||
if !ok || visible.Left.Redacted || visible.Right.Redacted || visible.HiddenChanged ||
|
||||
!strings.Contains(visible.Left.YAML, "before-value") || !strings.Contains(visible.Right.YAML, "after-value") {
|
||||
t.Fatalf("public sibling difference = (%+v, %t)", visible, ok)
|
||||
}
|
||||
|
||||
differences[0].Path = configkit.Keys("caller", "mutation")
|
||||
differences[0].Left.YAML = "caller mutation"
|
||||
differences[0].Right.YAML = "caller mutation"
|
||||
again, err := left.Diff(right)
|
||||
if err != nil {
|
||||
t.Fatalf("repeat comparison after mutating returned records: %v", err)
|
||||
}
|
||||
if len(again) != 2 {
|
||||
t.Fatalf("repeat difference count = %d, want 2: %+v", len(again), again)
|
||||
}
|
||||
againByPath := differencesByPath(again)
|
||||
if _, exists := againByPath["/caller/mutation"]; exists {
|
||||
t.Fatal("mutating returned path reached retained inspection")
|
||||
}
|
||||
if _, exists := againByPath["/backends/local/extra_params/token"]; !exists {
|
||||
t.Fatalf("original sensitive path was lost after mutation: %+v", again)
|
||||
}
|
||||
if _, exists := againByPath["/backends/local/extra_params/visible"]; !exists {
|
||||
t.Fatalf("original public path was lost after mutation: %+v", again)
|
||||
}
|
||||
for _, difference := range again {
|
||||
if difference.Left.YAML == "caller mutation" || difference.Right.YAML == "caller mutation" {
|
||||
t.Fatalf("mutating returned snapshots reached retained inspection: %+v", difference)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigurationInspectionDiffMasksSensitiveShapeReplacement(t *testing.T) {
|
||||
left := comparisonInspectionFrom(t, "before", "same-model", map[string]any{
|
||||
"details": map[string]any{
|
||||
"token": "nested-secret-sentinel",
|
||||
"label": "public detail",
|
||||
},
|
||||
})
|
||||
right := comparisonInspectionFrom(t, "after", "same-model", map[string]any{
|
||||
"details": "replacement",
|
||||
})
|
||||
|
||||
differences, err := left.Diff(right)
|
||||
if err != nil {
|
||||
t.Fatalf("compare sensitive shape replacement: %v", err)
|
||||
}
|
||||
if len(differences) != 1 || differences[0].Path.String() != "/backends/local/extra_params/details" {
|
||||
t.Fatalf("shape replacement differences = %+v", differences)
|
||||
}
|
||||
difference := differences[0]
|
||||
if !difference.Left.Redacted || !difference.Right.Redacted || !difference.HiddenChanged {
|
||||
t.Fatalf("shape replacement masking = %+v", difference)
|
||||
}
|
||||
if strings.Contains(difference.Left.YAML, "nested-secret-sentinel") || strings.Contains(difference.Right.YAML, "nested-secret-sentinel") {
|
||||
t.Fatalf("shape replacement disclosed a sensitive value: %+v", difference)
|
||||
}
|
||||
}
|
||||
|
||||
func comparisonInspection(t *testing.T, model string) promptkit.ConfigurationInspection {
|
||||
t.Helper()
|
||||
return comparisonInspectionFrom(t, "comparison fixture", model, nil)
|
||||
}
|
||||
|
||||
func comparisonInspectionFrom(t *testing.T, source, model string, extraParams map[string]any) promptkit.ConfigurationInspection {
|
||||
t.Helper()
|
||||
backend := map[string]any{"endpoint": "https://example.invalid/v1"}
|
||||
if extraParams != nil {
|
||||
backend["extra_params"] = extraParams
|
||||
}
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: configkit.Mapping(source, map[string]any{
|
||||
"backends": map[string]any{"local": backend},
|
||||
"profiles": map[string]any{"offline": map[string]any{
|
||||
"backend": "local",
|
||||
"model": model,
|
||||
}},
|
||||
})}}}},
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatalf("construct comparison engine: %v", err)
|
||||
}
|
||||
inspection, err := engine.InspectConfiguration()
|
||||
if err != nil {
|
||||
t.Fatalf("inspect comparison engine: %v", err)
|
||||
}
|
||||
return inspection
|
||||
}
|
||||
|
||||
func differencesByPath(differences []configkit.Difference) map[string]configkit.Difference {
|
||||
byPath := make(map[string]configkit.Difference, len(differences))
|
||||
for _, difference := range differences {
|
||||
byPath[difference.Path.String()] = difference
|
||||
}
|
||||
return byPath
|
||||
}
|
||||
|
||||
func hasSourceLabel(view promptkit.ConfigurationInspection, label string) bool {
|
||||
for _, source := range view.Sources() {
|
||||
if source.Label == label {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
99
configuration_inspection.go
Normal file
99
configuration_inspection.go
Normal file
@@ -0,0 +1,99 @@
|
||||
package promptkit
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/configuration"
|
||||
)
|
||||
|
||||
// ConfigurationInspection is a read-only view of one Engine's captured
|
||||
// effective configuration and source history. It does not expose typed
|
||||
// configuration values, filesystem contexts, or runtime credentials.
|
||||
// Report content may include application prompt and provider data, so callers
|
||||
// choose where and to whom to disclose it.
|
||||
type ConfigurationInspection struct {
|
||||
source configuration.Inspection
|
||||
}
|
||||
|
||||
// InspectConfiguration returns the ConfigKit inspection view captured by this
|
||||
// Engine. It does not reload sources, resolve credentials, or contact a model.
|
||||
// A nil Engine returns ErrInvalidConfig.
|
||||
func (e *Engine) InspectConfiguration() (ConfigurationInspection, error) {
|
||||
if e == nil || e.internal == nil {
|
||||
return ConfigurationInspection{}, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
|
||||
}
|
||||
return ConfigurationInspection{source: e.internal.InspectConfiguration()}, nil
|
||||
}
|
||||
|
||||
// Diff compares this view's captured effective configuration with other. This
|
||||
// view is the earlier (left) value and other is the later (right) value. The
|
||||
// comparison delegates to ConfigKit's frozen effective-value semantics, so
|
||||
// source or resolution history alone does not create differences. It does not
|
||||
// reread external sources or compare referenced file contents, resolved
|
||||
// credential contents, later execution preparation, or request-specific
|
||||
// overrides. Either zero view returns ErrInvalidConfig and no records; other
|
||||
// ConfigKit errors are returned unchanged. An empty diff is not evidence of
|
||||
// equivalent engine behavior or execution. Records can contain ordinary
|
||||
// application data, so callers choose where and to whom to disclose them.
|
||||
func (v ConfigurationInspection) Diff(other ConfigurationInspection) ([]configkit.Difference, error) {
|
||||
if v.source == nil || other.source == nil {
|
||||
return nil, ErrInvalidConfig
|
||||
}
|
||||
return configuration.Diff(v.source, other.source)
|
||||
}
|
||||
|
||||
// Metadata returns detached effective presence and history records.
|
||||
func (v ConfigurationInspection) Metadata() configkit.Metadata {
|
||||
if v.source == nil {
|
||||
return configkit.Metadata{}
|
||||
}
|
||||
return v.source.Metadata()
|
||||
}
|
||||
|
||||
// Sources returns a detached list of loaded source descriptions.
|
||||
func (v ConfigurationInspection) Sources() []configkit.SourceInfo {
|
||||
if v.source == nil {
|
||||
return nil
|
||||
}
|
||||
return v.source.Sources()
|
||||
}
|
||||
|
||||
// Diagnostics returns detached nonfatal load diagnostics.
|
||||
func (v ConfigurationInspection) Diagnostics() []configkit.Diagnostic {
|
||||
if v.source == nil {
|
||||
return nil
|
||||
}
|
||||
return v.source.Diagnostics()
|
||||
}
|
||||
|
||||
// Explain returns detached effective and historical values for a known path.
|
||||
// Dynamic credential-shaped provider keys are redacted by ConfigKit; ordinary
|
||||
// prompt and provider content is not automatically safe to disclose.
|
||||
func (v ConfigurationInspection) Explain(path configkit.Path) (configkit.Explanation, bool) {
|
||||
if v.source == nil {
|
||||
return configkit.Explanation{}, false
|
||||
}
|
||||
return v.source.Explain(path)
|
||||
}
|
||||
|
||||
// WriteEffective writes ConfigKit's redacted effective YAML. Writer errors
|
||||
// remain available through errors.Is and errors.As. The zero view returns
|
||||
// ErrInvalidConfig.
|
||||
func (v ConfigurationInspection) WriteEffective(w io.Writer) error {
|
||||
if v.source == nil {
|
||||
return ErrInvalidConfig
|
||||
}
|
||||
return v.source.WriteEffective(w)
|
||||
}
|
||||
|
||||
// WriteExplanation writes ConfigKit's redacted history for path. Unknown paths
|
||||
// produce no output; writer errors remain inspectable. The zero view returns
|
||||
// ErrInvalidConfig.
|
||||
func (v ConfigurationInspection) WriteExplanation(w io.Writer, path configkit.Path) error {
|
||||
if v.source == nil {
|
||||
return ErrInvalidConfig
|
||||
}
|
||||
return v.source.WriteExplanation(w, path)
|
||||
}
|
||||
286
configuration_inspection_contract_test.go
Normal file
286
configuration_inspection_contract_test.go
Normal file
@@ -0,0 +1,286 @@
|
||||
package promptkit_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"testing/fstest"
|
||||
|
||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
func TestConfigurationInspectionReportsLiteralKeysAndRedactedHistory(t *testing.T) {
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||
Underlays: []configkit.Underlay{
|
||||
{Inputs: []configkit.Input{{Source: configkit.Mapping("defaults", map[string]any{
|
||||
"backends": map[string]any{"local": map[string]any{
|
||||
"endpoint": "https://example.invalid/v1", "api_key_env": "OPTIONAL_KEY",
|
||||
"extra_params": map[string]any{"token": "private-sentinel", "visible": "public-value"},
|
||||
}},
|
||||
"collections": map[string]any{"literal": map[string]any{
|
||||
"backends": map[string]any{"local": map[string]any{
|
||||
"endpoint": "https://example.invalid/v1",
|
||||
"extra_params": map[string]any{"token": "namespaced-private-sentinel"},
|
||||
}},
|
||||
"profiles": map[string]any{
|
||||
"base.v1": map[string]any{"backend": "local", "model": "default-model"},
|
||||
"id": map[string]any{"extends": "base.v1"},
|
||||
},
|
||||
}},
|
||||
})}}},
|
||||
{Inputs: []configkit.Input{{Source: configkit.Mapping("operator", map[string]any{
|
||||
"collections": map[string]any{"literal": map[string]any{"profiles": map[string]any{"base.v1": map[string]any{"model": "operator-model"}}}},
|
||||
})}}},
|
||||
},
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatalf("construct layered engine: %v", err)
|
||||
}
|
||||
view, err := engine.InspectConfiguration()
|
||||
if err != nil {
|
||||
t.Fatalf("inspect configuration: %v", err)
|
||||
}
|
||||
modelPath := configkit.Keys("collections", "literal", "profiles", "base.v1", "model")
|
||||
model, found := view.Explain(modelPath)
|
||||
if !found || !strings.Contains(model.Effective.YAML, "operator-model") || len(model.History) < 2 {
|
||||
t.Fatalf("layered model explanation = (%+v, %t)", model, found)
|
||||
}
|
||||
inheritedPath := configkit.Keys("collections", "literal", "profiles", "id", "model")
|
||||
inherited, found := view.Explain(inheritedPath)
|
||||
if !found || !strings.Contains(inherited.Effective.YAML, "operator-model") || len(inherited.History) == 0 || !view.Metadata().IsExplicit(inheritedPath) {
|
||||
t.Fatalf("literal-key inheritance explanation = (%+v, %t)", inherited, found)
|
||||
}
|
||||
if inheritedPath.String() != "/collections/literal/profiles/id/model" {
|
||||
t.Fatalf("literal-key path = %q", inheritedPath)
|
||||
}
|
||||
secretPath := configkit.Keys("backends", "local", "extra_params", "token")
|
||||
secret, found := view.Explain(secretPath)
|
||||
if !found || !secret.Effective.Redacted || strings.Contains(secret.Effective.YAML, "private-sentinel") {
|
||||
t.Fatalf("credential-shaped provider value was not redacted: (%+v, %t)", secret, found)
|
||||
}
|
||||
namespacedSecretPath := configkit.Keys("collections", "literal", "backends", "local", "extra_params", "token")
|
||||
namespacedSecret, found := view.Explain(namespacedSecretPath)
|
||||
if !found || !namespacedSecret.Effective.Redacted || strings.Contains(namespacedSecret.Effective.YAML, "namespaced-private-sentinel") {
|
||||
t.Fatalf("namespaced provider value was not redacted: (%+v, %t)", namespacedSecret, found)
|
||||
}
|
||||
var effective bytes.Buffer
|
||||
if err := view.WriteEffective(&effective); err != nil {
|
||||
t.Fatalf("write effective configuration: %v", err)
|
||||
}
|
||||
if strings.Contains(effective.String(), "private-sentinel") || !strings.Contains(effective.String(), "OPTIONAL_KEY") || !strings.Contains(effective.String(), "public-value") {
|
||||
t.Fatalf("unexpected effective report: %s", effective.String())
|
||||
}
|
||||
var explanation bytes.Buffer
|
||||
if err := view.WriteExplanation(&explanation, namespacedSecretPath); err != nil || strings.Contains(explanation.String(), "namespaced-private-sentinel") {
|
||||
t.Fatalf("secret explanation = (%q, %v)", explanation.String(), err)
|
||||
}
|
||||
|
||||
sources := view.Sources()
|
||||
if len(sources) == 0 {
|
||||
t.Fatal("inspection returned no sources")
|
||||
}
|
||||
originalLabel := sources[0].Label
|
||||
sources[0].Label = "changed"
|
||||
if view.Sources()[0].Label != originalLabel {
|
||||
t.Fatal("source mutation reached retained inspection")
|
||||
}
|
||||
model.History[0].After.YAML = "changed"
|
||||
if again, ok := view.Explain(modelPath); !ok || again.History[0].After.YAML == "changed" {
|
||||
t.Fatal("explanation mutation reached retained inspection")
|
||||
}
|
||||
history := view.Metadata().History(modelPath)
|
||||
if len(history) == 0 {
|
||||
t.Fatal("metadata history is empty")
|
||||
}
|
||||
history[0].Reasons = []string{"changed"}
|
||||
if again := view.Metadata().History(modelPath); len(again) == 0 || (len(again[0].Reasons) > 0 && again[0].Reasons[0] == "changed") {
|
||||
t.Fatal("metadata mutation reached retained inspection")
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigurationInspectionRetainsMainSpecificationDeclaration(t *testing.T) {
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
|
||||
Source: configkit.Mapping("main", map[string]any{"version": "1"}),
|
||||
}}})
|
||||
if err != nil {
|
||||
t.Fatalf("construct engine with accepted declaration: %v", err)
|
||||
}
|
||||
inspection, err := engine.InspectConfiguration()
|
||||
if err != nil {
|
||||
t.Fatalf("inspect configuration: %v", err)
|
||||
}
|
||||
path := configkit.Keys("version")
|
||||
field, found := inspection.Explain(path)
|
||||
if !found || !inspection.Metadata().IsExplicit(path) || !strings.Contains(field.Effective.YAML, "1") {
|
||||
t.Fatalf("main specification declaration was not retained: (%+v, %t)", field, found)
|
||||
}
|
||||
var effective bytes.Buffer
|
||||
if err := inspection.WriteEffective(&effective); err != nil || !strings.Contains(effective.String(), "version") {
|
||||
t.Fatalf("effective inspection = (%q, %v), want retained version", effective.String(), err)
|
||||
}
|
||||
}
|
||||
|
||||
type failingInspectionWriter struct{ err error }
|
||||
|
||||
func (writer failingInspectionWriter) Write([]byte) (int, error) { return 0, writer.err }
|
||||
|
||||
func TestConfigurationInspectionZeroAndWriterErrors(t *testing.T) {
|
||||
var engine *promptkit.Engine
|
||||
if _, err := engine.InspectConfiguration(); !errors.Is(err, promptkit.ErrInvalidConfig) {
|
||||
t.Fatalf("nil engine inspection = %v", err)
|
||||
}
|
||||
var empty promptkit.ConfigurationInspection
|
||||
path := configkit.Keys("profiles", "missing")
|
||||
if _, found := empty.Metadata().Lookup(path); found || len(empty.Sources()) != 0 || len(empty.Diagnostics()) != 0 {
|
||||
t.Fatal("zero inspection reported configuration")
|
||||
}
|
||||
if _, found := empty.Explain(path); found {
|
||||
t.Fatal("zero inspection explained an unknown path")
|
||||
}
|
||||
if err := empty.WriteEffective(io.Discard); !errors.Is(err, promptkit.ErrInvalidConfig) {
|
||||
t.Fatalf("zero effective writer = %v", err)
|
||||
}
|
||||
if err := empty.WriteExplanation(io.Discard, path); !errors.Is(err, promptkit.ErrInvalidConfig) {
|
||||
t.Fatalf("zero explanation writer = %v", err)
|
||||
}
|
||||
|
||||
endpoint, model := "https://configured.example/v1", "configured-model"
|
||||
profile := configkit.Input{Source: promptkit.ProfileSource("inspection test profile", map[string]promptkit.ProfileConfig{
|
||||
"inspection-profile": {Endpoint: &endpoint, Model: &model},
|
||||
})}
|
||||
valid, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{profile}}},
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatalf("construct engine: %v", err)
|
||||
}
|
||||
view, err := valid.InspectConfiguration()
|
||||
if err != nil {
|
||||
t.Fatalf("inspect configuration: %v", err)
|
||||
}
|
||||
writeErr := errors.New("report writer failed")
|
||||
writer := failingInspectionWriter{err: writeErr}
|
||||
if err := view.WriteEffective(writer); !errors.Is(err, writeErr) {
|
||||
t.Fatalf("effective writer error = %v", err)
|
||||
}
|
||||
if err := view.WriteExplanation(writer, configkit.Keys("profiles", "inspection-profile", "model")); !errors.Is(err, writeErr) {
|
||||
t.Fatalf("explanation writer error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigurationInspectionUsesCapturedSourcesWithoutProviderCalls(t *testing.T) {
|
||||
source := &unreadableAfterConstructionFS{FS: fstest.MapFS{
|
||||
"config/prompt.yaml": &fstest.MapFile{Data: []byte(capturedPromptDocument)},
|
||||
"config/body.tmpl": &fstest.MapFile{Data: []byte("hello")},
|
||||
}}
|
||||
inputs, err := promptkit.FilesystemInputs("inspection source", source, "config")
|
||||
if err != nil {
|
||||
t.Fatalf("select inputs: %v", err)
|
||||
}
|
||||
otherSource := &unreadableAfterConstructionFS{FS: fstest.MapFS{
|
||||
"config/prompt.yaml": &fstest.MapFile{Data: []byte(capturedPromptDocument)},
|
||||
"config/body.tmpl": &fstest.MapFile{Data: []byte("hello")},
|
||||
}}
|
||||
otherInputs, err := promptkit.FilesystemInputs("comparison source", otherSource, "config")
|
||||
if err != nil {
|
||||
t.Fatalf("select comparison inputs: %v", err)
|
||||
}
|
||||
var modelCalls atomic.Int64
|
||||
client := countingLLMClient{calls: &modelCalls}
|
||||
endpoint, model, otherModel := "https://example.invalid/v1", "offline-model", "comparison-model"
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||
Underlays: []configkit.Underlay{{Inputs: inputs}, {Inputs: []configkit.Input{{Source: promptkit.ProfileSource("offline profile", map[string]promptkit.ProfileConfig{
|
||||
"offline": {Endpoint: &endpoint, Model: &model},
|
||||
})}}}},
|
||||
}}, promptkit.WithLLMClient(client))
|
||||
if err != nil {
|
||||
t.Fatalf("construct engine: %v", err)
|
||||
}
|
||||
otherEngine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||
Underlays: []configkit.Underlay{{Inputs: otherInputs}, {Inputs: []configkit.Input{{Source: promptkit.ProfileSource("comparison profile", map[string]promptkit.ProfileConfig{
|
||||
"offline": {Endpoint: &endpoint, Model: &otherModel},
|
||||
})}}}},
|
||||
}}, promptkit.WithLLMClient(client))
|
||||
if err != nil {
|
||||
t.Fatalf("construct comparison engine: %v", err)
|
||||
}
|
||||
source.closed.Store(true)
|
||||
otherSource.closed.Store(true)
|
||||
view, err := engine.InspectConfiguration()
|
||||
if err != nil {
|
||||
t.Fatalf("inspect configuration: %v", err)
|
||||
}
|
||||
otherView, err := otherEngine.InspectConfiguration()
|
||||
if err != nil {
|
||||
t.Fatalf("inspect comparison configuration: %v", err)
|
||||
}
|
||||
var expected configkit.Difference
|
||||
for index := 0; index < 3; index++ {
|
||||
differences, err := view.Diff(otherView)
|
||||
if err != nil || len(differences) != 1 {
|
||||
t.Fatalf("compare captured configurations = (%+v, %v), want one difference", differences, err)
|
||||
}
|
||||
if differences[0].Path.String() != "/profiles/offline/model" ||
|
||||
!strings.Contains(differences[0].Left.YAML, model) ||
|
||||
!strings.Contains(differences[0].Right.YAML, otherModel) {
|
||||
t.Fatalf("captured difference = %+v", differences[0])
|
||||
}
|
||||
if index == 0 {
|
||||
expected = differences[0]
|
||||
} else if differences[0].Path.String() != expected.Path.String() ||
|
||||
differences[0].Left.YAML != expected.Left.YAML || differences[0].Right.YAML != expected.Right.YAML {
|
||||
t.Fatalf("captured difference changed between reads: first=%+v next=%+v", expected, differences[0])
|
||||
}
|
||||
}
|
||||
if _, found := view.Explain(configkit.Keys("prompts", "captured", "messages").Index(0).Key("content_file")); !found {
|
||||
t.Fatal("captured prompt content path was not inspectable")
|
||||
}
|
||||
var report bytes.Buffer
|
||||
if err := view.WriteEffective(&report); err != nil || !strings.Contains(report.String(), "captured") {
|
||||
t.Fatalf("write captured report = (%q, %v)", report.String(), err)
|
||||
}
|
||||
if source.reads.Load() != 0 || otherSource.reads.Load() != 0 || modelCalls.Load() != 0 {
|
||||
t.Fatalf("inspection or comparison touched source or provider: reads=(%d,%d) requests=%d", source.reads.Load(), otherSource.reads.Load(), modelCalls.Load())
|
||||
}
|
||||
|
||||
const calls = 24
|
||||
var wait sync.WaitGroup
|
||||
errs := make(chan error, calls)
|
||||
for index := 0; index < calls; index++ {
|
||||
wait.Add(1)
|
||||
go func(index int) {
|
||||
defer wait.Done()
|
||||
switch index % 3 {
|
||||
case 0:
|
||||
_, err := engine.InspectConfiguration()
|
||||
errs <- err
|
||||
case 1:
|
||||
_, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "captured", SessionID: "direct"})
|
||||
errs <- err
|
||||
case 2:
|
||||
differences, err := view.Diff(otherView)
|
||||
if err == nil && (len(differences) != 1 || differences[0].Path.String() != expected.Path.String() ||
|
||||
differences[0].Left.YAML != expected.Left.YAML || differences[0].Right.YAML != expected.Right.YAML) {
|
||||
err = errors.New("concurrent comparison returned an unexpected difference")
|
||||
}
|
||||
errs <- err
|
||||
}
|
||||
}(index)
|
||||
}
|
||||
wait.Wait()
|
||||
close(errs)
|
||||
for err := range errs {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if source.reads.Load() != 0 || otherSource.reads.Load() != 0 || modelCalls.Load() != calls/3 {
|
||||
t.Fatalf("concurrent inspection/comparison/execution = source reads (%d,%d), model calls %d", source.reads.Load(), otherSource.reads.Load(), modelCalls.Load())
|
||||
}
|
||||
}
|
||||
100
configuration_sources.go
Normal file
100
configuration_sources.go
Normal file
@@ -0,0 +1,100 @@
|
||||
package promptkit
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/filecatalog"
|
||||
)
|
||||
|
||||
// CollectionInput mounts source at the selected collection's configuration
|
||||
// destination, collections/<namespace>, and enables ConfigKit imports for that
|
||||
// input. An invalid namespace returns an error matching ErrInvalidConfig. The
|
||||
// source is not read or validated until ConfigKit loads it. Callers may adjust
|
||||
// the returned input, including its filesystem context or imports setting.
|
||||
func CollectionInput(namespace string, source configkit.Source) (configkit.Input, error) {
|
||||
if !domain.IsValidResourceNamespace(namespace) {
|
||||
return configkit.Input{}, fmt.Errorf("%w: invalid collection namespace %q", ErrInvalidConfig, namespace)
|
||||
}
|
||||
return configkit.Input{
|
||||
Source: source,
|
||||
Destination: configkit.Keys("collections", namespace),
|
||||
Imports: true,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// DirectoryInputs selects the YAML files beneath an explicitly chosen host
|
||||
// directory as one strict-composition group. The returned order is deterministic;
|
||||
// it does not assign precedence among those files. Discovery is recursive over
|
||||
// .yaml and .yml files. An unavailable root fails; an empty directory yields
|
||||
// no inputs. Imports are disabled unless callers enable them on the inputs.
|
||||
func DirectoryInputs(directory string) ([]configkit.Input, error) {
|
||||
root, err := filepath.Abs(directory)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("resolve configuration directory: %w", err)
|
||||
}
|
||||
info, err := os.Stat(root)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("inspect configuration directory: %w", err)
|
||||
}
|
||||
if !info.IsDir() {
|
||||
return nil, fmt.Errorf("configuration root is not a directory: %s", root)
|
||||
}
|
||||
root, err = filepath.EvalSymlinks(root)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("resolve configuration directory: %w", err)
|
||||
}
|
||||
files, err := filecatalog.FindYAMLFiles(context.Background(), root)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("discover configuration files: %w", err)
|
||||
}
|
||||
inputs := make([]configkit.Input, 0, len(files))
|
||||
for _, file := range files {
|
||||
inputs = append(inputs, configkit.Input{
|
||||
Source: configkit.File(file),
|
||||
Filesystem: &configkit.FilesystemContext{Root: root},
|
||||
})
|
||||
}
|
||||
return inputs, nil
|
||||
}
|
||||
|
||||
// FilesystemInputs selects the YAML files beneath root in a caller-owned
|
||||
// filesystem as one strict-composition group. Discovery is recursive over
|
||||
// .yaml and .yml files, with deterministic ordering and no precedence among
|
||||
// files. The caller retains ownership of the filesystem; an unavailable root
|
||||
// fails and an empty directory yields no inputs. Imports remain disabled unless
|
||||
// callers enable them on inputs.
|
||||
func FilesystemInputs(label string, filesystem fs.FS, root string) ([]configkit.Input, error) {
|
||||
if filesystem == nil {
|
||||
return nil, fmt.Errorf("configuration filesystem is nil")
|
||||
}
|
||||
root = filecatalog.CleanFSRoot(root)
|
||||
if !fs.ValidPath(root) {
|
||||
return nil, fmt.Errorf("configuration root is not a valid filesystem path: %s", root)
|
||||
}
|
||||
info, err := fs.Stat(filesystem, root)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("inspect configuration root: %w", err)
|
||||
}
|
||||
if !info.IsDir() {
|
||||
return nil, fmt.Errorf("configuration root is not a directory: %s", root)
|
||||
}
|
||||
files, err := filecatalog.FindFSYAMLFiles(context.Background(), filesystem, root)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("discover configuration files: %w", err)
|
||||
}
|
||||
namespace := configkit.NewFilesystem(label, filesystem)
|
||||
inputs := make([]configkit.Input, 0, len(files))
|
||||
for _, file := range files {
|
||||
inputs = append(inputs, configkit.Input{
|
||||
Source: namespace.File(file),
|
||||
Filesystem: &configkit.FilesystemContext{FS: &namespace, Root: root},
|
||||
})
|
||||
}
|
||||
return inputs, nil
|
||||
}
|
||||
157
configuration_sources_test.go
Normal file
157
configuration_sources_test.go
Normal file
@@ -0,0 +1,157 @@
|
||||
package promptkit_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"testing/fstest"
|
||||
|
||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
type sourceDocument struct {
|
||||
First string `yaml:"first"`
|
||||
Second string `yaml:"second"`
|
||||
}
|
||||
|
||||
func TestCollectionInputValidatesWithoutReadingSource(t *testing.T) {
|
||||
input, err := promptkit.CollectionInput("docs", configkit.File(filepath.Join(t.TempDir(), "not-read.yaml")))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if input.Destination.String() != "/collections/docs" || !input.Imports {
|
||||
t.Fatalf("collection input = %+v, want destination /collections/docs with imports enabled", input)
|
||||
}
|
||||
for _, namespace := range []string{"", " ", "docs/api"} {
|
||||
if _, err := promptkit.CollectionInput(namespace, configkit.NoSource()); !errors.Is(err, promptkit.ErrInvalidConfig) {
|
||||
t.Errorf("namespace %q error = %v, want ErrInvalidConfig", namespace, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCollectionInputLoadsNativeImportsAndLaterOverride(t *testing.T) {
|
||||
directory := t.TempDir()
|
||||
mainPath := filepath.Join(directory, "collection.yaml")
|
||||
defaultsPath := filepath.Join(directory, "defaults.yaml")
|
||||
if err := os.WriteFile(mainPath, []byte("configkit:\n imports:\n - path: defaults.yaml\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(defaultsPath, []byte("prompts:\n greeting:\n version: imported\n messages:\n - role: user\n content: Hello\n output:\n format: text\n validation_mode: none\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
input, err := promptkit.CollectionInput("docs", configkit.File(mainPath))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{input}}},
|
||||
Main: &configkit.Input{
|
||||
Source: configkit.Mapping("later collection override", map[string]any{"prompts": map[string]any{"greeting": map[string]any{"version": "overridden"}}}),
|
||||
Destination: configkit.Keys("collections", "docs"),
|
||||
},
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatalf("construct engine with native collection import: %v", err)
|
||||
}
|
||||
inspection, err := engine.InspectPrompt(context.Background(), "docs/greeting")
|
||||
if err != nil || inspection.PromptVersion == nil || *inspection.PromptVersion != "overridden" {
|
||||
t.Fatalf("inspect imported and overlaid prompt = (%+v, %v)", inspection, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCollectionInputLoadsEmbeddedImportsAndLaterOverride(t *testing.T) {
|
||||
files := fstest.MapFS{
|
||||
"collection.yaml": {Data: []byte("configkit:\n imports:\n - path: defaults.yaml\n")},
|
||||
"defaults.yaml": {Data: []byte("prompts:\n greeting:\n version: imported\n messages:\n - role: user\n content: Hello\n output:\n format: text\n validation_mode: none\n")},
|
||||
}
|
||||
filesystem := configkit.NewFilesystem("embedded collection", files)
|
||||
input, err := promptkit.CollectionInput("docs", filesystem.File("collection.yaml"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
input.Filesystem = &configkit.FilesystemContext{FS: &filesystem, Root: "."}
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{input}}},
|
||||
Main: &configkit.Input{
|
||||
Source: configkit.Mapping("later collection override", map[string]any{"prompts": map[string]any{"greeting": map[string]any{"version": "overridden"}}}),
|
||||
Destination: configkit.Keys("collections", "docs"),
|
||||
},
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatalf("construct engine with embedded collection import: %v", err)
|
||||
}
|
||||
inspection, err := engine.InspectPrompt(context.Background(), "docs/greeting")
|
||||
if err != nil || inspection.PromptVersion == nil || *inspection.PromptVersion != "overridden" {
|
||||
t.Fatalf("inspect imported and overlaid prompt = (%+v, %v)", inspection, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDirectoryInputsSelectsSortedFilesWithRootContext(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
if err := os.Mkdir(filepath.Join(root, "nested"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for name, data := range map[string]string{
|
||||
"a.yaml": "first: one\n", "nested/b.yml": "second: two\n", "nested/body.md": "message",
|
||||
} {
|
||||
if err := os.WriteFile(filepath.Join(root, name), []byte(data), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
inputs, err := promptkit.DirectoryInputs(root)
|
||||
if err != nil || len(inputs) != 2 || inputs[0].Imports || inputs[1].Imports {
|
||||
t.Fatalf("directory inputs: %+v, %v", inputs, err)
|
||||
}
|
||||
assertSourceInputs(t, inputs, "message")
|
||||
if _, err := promptkit.DirectoryInputs(filepath.Join(root, "missing")); err == nil {
|
||||
t.Fatal("missing selected directory succeeded")
|
||||
}
|
||||
if empty, err := promptkit.DirectoryInputs(t.TempDir()); err != nil || len(empty) != 0 {
|
||||
t.Fatalf("empty directory: %+v, %v", empty, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFilesystemInputsSelectsEmbeddedFilesWithRootContext(t *testing.T) {
|
||||
files := fstest.MapFS{
|
||||
"settings/a.yaml": {Data: []byte("first: one\n")},
|
||||
"settings/nested/b.yml": {Data: []byte("second: two\n")},
|
||||
"settings/nested/body.md": {Data: []byte("message")},
|
||||
}
|
||||
inputs, err := promptkit.FilesystemInputs("embedded", files, "settings")
|
||||
if err != nil || len(inputs) != 2 || inputs[0].Imports || inputs[1].Imports {
|
||||
t.Fatalf("filesystem inputs: %+v, %v", inputs, err)
|
||||
}
|
||||
assertSourceInputs(t, inputs, "message")
|
||||
if _, err := promptkit.FilesystemInputs("embedded", files, "missing"); err == nil {
|
||||
t.Fatal("missing selected root succeeded")
|
||||
}
|
||||
}
|
||||
|
||||
func assertSourceInputs(t *testing.T, inputs []configkit.Input, expected string) {
|
||||
t.Helper()
|
||||
result, err := configkit.LoadLayered(configkit.Input{Source: configkit.NoSource()}, configkit.LayeredOptions[sourceDocument]{
|
||||
Underlays: []configkit.Underlay{{Inputs: inputs}},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result.Config.First != "one" || result.Config.Second != "two" {
|
||||
t.Fatalf("configuration files did not compose: %+v", result.Config)
|
||||
}
|
||||
field, found := result.Metadata().Lookup(configkit.Keys("second"))
|
||||
if !found || len(field.Origin.Locations) != 1 {
|
||||
t.Fatalf("missing second-field origin: %+v", field)
|
||||
}
|
||||
context, found := result.SourceContext(field.Origin.Locations[0].SourceID)
|
||||
if !found || context.BaseDir != "nested" {
|
||||
t.Fatalf("unexpected rooted source context: %+v", context)
|
||||
}
|
||||
data, err := fs.ReadFile(context.FS, "nested/body.md")
|
||||
if err != nil || string(data) != expected {
|
||||
t.Fatalf("rooted resource read: %q, %v", data, err)
|
||||
}
|
||||
}
|
||||
314
construction_contract_test.go
Normal file
314
construction_contract_test.go
Normal file
@@ -0,0 +1,314 @@
|
||||
package promptkit_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"testing/fstest"
|
||||
|
||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
func TestProfileEnvironmentLookupAcceptsNonShellNames(t *testing.T) {
|
||||
const name = "promptkit.test-api-key"
|
||||
for _, test := range []struct {
|
||||
name, value, direct, want string
|
||||
unset, clear bool
|
||||
}{
|
||||
{name: "set", value: " synthetic-key ", want: "Bearer synthetic-key"},
|
||||
{name: "empty"},
|
||||
{name: "whitespace", value: " "},
|
||||
{name: "unset", unset: true},
|
||||
{name: "direct precedence", value: "synthetic-key", direct: "direct-key", want: "Bearer direct-key"},
|
||||
{name: "explicit clear", value: "synthetic-key", clear: true},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
t.Setenv(name, test.value)
|
||||
if test.unset {
|
||||
if err := os.Unsetenv(name); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
cfg := simpleConfig("hello")
|
||||
cfg.Sources.Underlays = []configkit.Underlay{{Inputs: []configkit.Input{*cfg.Sources.Main}}}
|
||||
lowerName := name
|
||||
cfg.Sources.Underlays = append(cfg.Sources.Underlays, configkit.Underlay{Inputs: []configkit.Input{{
|
||||
Source: promptkit.ProfileSource("default credentials", map[string]promptkit.ProfileConfig{
|
||||
"profile": {APIKeyEnv: &lowerName},
|
||||
}),
|
||||
}}})
|
||||
env := " " + name + " "
|
||||
if test.clear {
|
||||
env = ""
|
||||
}
|
||||
cfg.Sources.Main = &configkit.Input{Source: promptkit.ProfileSource("credentials", map[string]promptkit.ProfileConfig{
|
||||
"profile": {APIKeyEnv: &env},
|
||||
})}
|
||||
var calls int
|
||||
var auth string
|
||||
cfg.HTTPClient = &http.Client{Transport: roundTripFunc(func(request *http.Request) (*http.Response, error) {
|
||||
calls++
|
||||
auth = request.Header.Get("Authorization")
|
||||
return &http.Response{StatusCode: http.StatusOK, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{"choices":[{"message":{"content":"ok"}}]}`))}, nil
|
||||
})}
|
||||
engine, err := promptkit.NewEngine(cfg)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt", APIKey: test.direct})
|
||||
if err != nil || calls != 1 || auth != test.want {
|
||||
t.Fatalf("run: err=%v calls=%d authorization=%q, want %q", err, calls, auth, test.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPublicConstructionWithLocalBackendAndProgrammaticProfile(t *testing.T) {
|
||||
backendID, model := "workstation", "offline-model"
|
||||
prompt := configkit.Input{Source: configkit.Mapping("prompt", map[string]any{
|
||||
"prompts": map[string]any{"hello": map[string]any{
|
||||
"version": "1",
|
||||
"default_profile": "offline",
|
||||
"messages": []any{map[string]any{"role": "user", "content": "Hello"}},
|
||||
"output": map[string]any{"format": "text", "validation_mode": "none"},
|
||||
}},
|
||||
})}
|
||||
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "reply"}}
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{
|
||||
{Source: promptkit.BackendSource("local backend", map[string]promptkit.BackendConfig{
|
||||
backendID: promptkit.LocalBackend("http://localhost:8000/v1", 2),
|
||||
})},
|
||||
{Source: promptkit.ProfileSource("profile", map[string]promptkit.ProfileConfig{
|
||||
"offline": {Backend: &backendID, Model: &model},
|
||||
})},
|
||||
}}},
|
||||
Main: &prompt,
|
||||
}}, promptkit.WithLLMClient(client))
|
||||
if err != nil {
|
||||
t.Fatalf("construct engine: %v", err)
|
||||
}
|
||||
result, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "hello"})
|
||||
if err != nil {
|
||||
t.Fatalf("run prompt: %v", err)
|
||||
}
|
||||
if result.RawOutput != "reply" || result.SelectedBackendID != backendID || result.EffectiveModelParams.Model != model {
|
||||
t.Fatalf("unexpected result: %+v", result)
|
||||
}
|
||||
if len(client.requests) != 1 || client.requests[0].Target.Endpoint != "http://localhost:8000/v1" {
|
||||
t.Fatalf("unexpected model request: %+v", client.requests)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFormerCatalogBackendIDsWorkWhenConfigured(t *testing.T) {
|
||||
openRouterID, rakestrawhomeID := promptkit.BackendOpenRouter, promptkit.BackendRakestrawHome
|
||||
openRouterEndpoint, rakestrawhomeEndpoint := "https://openrouter.example/v1", "https://inference.example/v1"
|
||||
model := "configured-model"
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{
|
||||
{Source: promptkit.BackendSource("application backends", map[string]promptkit.BackendConfig{
|
||||
openRouterID: {Endpoint: &openRouterEndpoint},
|
||||
rakestrawhomeID: {Endpoint: &rakestrawhomeEndpoint},
|
||||
})},
|
||||
{Source: promptkit.ProfileSource("application profiles", map[string]promptkit.ProfileConfig{
|
||||
"openrouter-profile": {Backend: &openRouterID, Model: &model},
|
||||
"rakestrawhome-profile": {Backend: &rakestrawhomeID, Model: &model},
|
||||
})},
|
||||
}}},
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatalf("construct engine with explicitly configured provider IDs: %v", err)
|
||||
}
|
||||
for profileID, backendID := range map[string]string{
|
||||
"openrouter-profile": openRouterID,
|
||||
"rakestrawhome-profile": rakestrawhomeID,
|
||||
} {
|
||||
inspection, err := engine.InspectProfile(context.Background(), profileID)
|
||||
if err != nil || inspection.EffectiveModelParams.BackendID != backendID {
|
||||
t.Fatalf("inspect %q = (%+v, %v), want backend %q", profileID, inspection, err, backendID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPublicConstructionCombinesEmbeddedOperatorAndProgrammaticInputs(t *testing.T) {
|
||||
embedded, err := promptkit.FilesystemInputs("embedded defaults", fstest.MapFS{
|
||||
"config/default.yaml": &fstest.MapFile{Data: []byte(`
|
||||
profiles:
|
||||
application:
|
||||
endpoint: https://example.invalid/v1
|
||||
model: default-model
|
||||
prompts:
|
||||
hello:
|
||||
version: "1"
|
||||
default_profile: application
|
||||
messages:
|
||||
- role: user
|
||||
content: Hello
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
`)},
|
||||
}, "config")
|
||||
if err != nil {
|
||||
t.Fatalf("select embedded defaults: %v", err)
|
||||
}
|
||||
operatorDirectory := t.TempDir()
|
||||
if err := os.WriteFile(filepath.Join(operatorDirectory, "override.yaml"), []byte("profiles:\n application:\n model: operator-model\n"), 0o644); err != nil {
|
||||
t.Fatalf("write operator input: %v", err)
|
||||
}
|
||||
operator, err := promptkit.DirectoryInputs(operatorDirectory)
|
||||
if err != nil {
|
||||
t.Fatalf("select operator inputs: %v", err)
|
||||
}
|
||||
programmaticModel := "programmatic-model"
|
||||
programmatic := configkit.Input{Source: promptkit.ProfileSource("programmatic settings", map[string]promptkit.ProfileConfig{
|
||||
"application": {Model: &programmaticModel},
|
||||
})}
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
apply bool
|
||||
want string
|
||||
}{
|
||||
{name: "embedded", want: "default-model"},
|
||||
{name: "operator", apply: true, want: "operator-model"},
|
||||
{name: "programmatic", apply: true, want: "programmatic-model"},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
underlays := []configkit.Underlay{{Inputs: embedded}}
|
||||
if test.apply {
|
||||
underlays = append(underlays, configkit.Underlay{Inputs: operator})
|
||||
}
|
||||
if test.name == "programmatic" {
|
||||
underlays = append(underlays, configkit.Underlay{Inputs: []configkit.Input{programmatic}})
|
||||
}
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: underlays}})
|
||||
if err != nil {
|
||||
t.Fatalf("construct engine: %v", err)
|
||||
}
|
||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "hello"})
|
||||
if err != nil {
|
||||
t.Fatalf("prepare prompt: %v", err)
|
||||
}
|
||||
if prepared.EffectiveModelParams.Model != test.want {
|
||||
t.Fatalf("model = %q, want %q", prepared.EffectiveModelParams.Model, test.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPublicConstructionPreservesConfigKitLoadError(t *testing.T) {
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
|
||||
Source: configkit.Mapping("invalid", map[string]any{"profiles": map[string]any{"broken": map[string]any{"unknown": true}}}),
|
||||
}}})
|
||||
var loadErr *configkit.LoadError
|
||||
if engine != nil || !errors.Is(err, promptkit.ErrInvalidConfig) || !errors.As(err, &loadErr) {
|
||||
t.Fatalf("construction = (%#v, %v), want invalid config preserving LoadError", engine, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPublicConstructionRejectsSuppliedInvalidMainInput(t *testing.T) {
|
||||
main := configkit.Input{}
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &main}})
|
||||
if engine != nil || !errors.Is(err, promptkit.ErrInvalidConfig) {
|
||||
t.Fatalf("construction = (%#v, %v), want invalid supplied main input", engine, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPublicConstructionUsesProgrammaticBackendAndProfileSources(t *testing.T) {
|
||||
endpoint, backendID, model := "http://application.example/v1", "application-backend", "synthetic-model"
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{
|
||||
{Source: promptkit.BackendSource("operator backend", map[string]promptkit.BackendConfig{
|
||||
backendID: {Endpoint: &endpoint},
|
||||
})},
|
||||
{Source: promptkit.ProfileSource("operator profile", map[string]promptkit.ProfileConfig{
|
||||
"application-profile": {Backend: &backendID, Model: &model},
|
||||
})},
|
||||
}}},
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatalf("construct engine with backend overlay: %v", err)
|
||||
}
|
||||
inspection, err := engine.InspectProfile(context.Background(), "application-profile")
|
||||
if err != nil {
|
||||
t.Fatalf("inspect configured profile: %v", err)
|
||||
}
|
||||
if inspection.EffectiveModelParams.Endpoint != endpoint {
|
||||
t.Fatalf("effective endpoint = %q, want %q", inspection.EffectiveModelParams.Endpoint, endpoint)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBackendSourceOwnsSmokeTestValuesAndDefersDefaultLookup(t *testing.T) {
|
||||
backendEndpoint := "https://backend.example/v1"
|
||||
defaultProfile := "not-yet-listed"
|
||||
reachabilityEndpoint := " https://probe.example/ready?check=1 "
|
||||
authenticationKind := " OPENROUTER_KEY "
|
||||
authenticationEndpoint := "https://probe.example/key?scope=current"
|
||||
source := promptkit.BackendSource("diagnostic backend", map[string]promptkit.BackendConfig{
|
||||
"local": {
|
||||
Endpoint: &backendEndpoint,
|
||||
SmokeTests: &promptkit.BackendSmokeTestConfig{
|
||||
DefaultProfile: &defaultProfile,
|
||||
Reachability: &promptkit.ReachabilityProbeConfig{Endpoint: &reachabilityEndpoint},
|
||||
Authentication: &promptkit.AuthenticationProbeConfig{
|
||||
Kind: &authenticationKind, Endpoint: &authenticationEndpoint,
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
backendEndpoint = "https://mutated.example/v1"
|
||||
defaultProfile = "mutated-profile"
|
||||
reachabilityEndpoint = "https://mutated.example/ready"
|
||||
authenticationKind = "none"
|
||||
authenticationEndpoint = "https://mutated.example/key"
|
||||
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: source}}}},
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatalf("construct engine with dangling diagnostic default: %v", err)
|
||||
}
|
||||
inspection, err := engine.InspectConfiguration()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var effective bytes.Buffer
|
||||
if err := inspection.WriteEffective(&effective); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
contents := effective.String()
|
||||
for _, want := range []string{
|
||||
"https://backend.example/v1",
|
||||
"not-yet-listed",
|
||||
"https://probe.example/ready?check=1",
|
||||
"openrouter_key",
|
||||
"https://probe.example/key?scope=current",
|
||||
} {
|
||||
if !strings.Contains(contents, want) {
|
||||
t.Errorf("effective configuration does not contain %q:\n%s", want, contents)
|
||||
}
|
||||
}
|
||||
for _, unwanted := range []string{
|
||||
"mutated.example",
|
||||
"mutated-profile",
|
||||
} {
|
||||
if strings.Contains(contents, unwanted) {
|
||||
t.Errorf("effective configuration unexpectedly contains %q:\n%s", unwanted, contents)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreparationSelectsSinglePromptDefinitionByID(t *testing.T) {
|
||||
engine := newSimpleEngine(t, "selected")
|
||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
|
||||
if err != nil || prepared == nil || len(prepared.Messages) != 1 || prepared.Messages[0].Content != "selected" {
|
||||
t.Fatalf("prepare prompt by ID = (%+v, %v)", prepared, err)
|
||||
}
|
||||
}
|
||||
74
consumer_test_helpers_test.go
Normal file
74
consumer_test_helpers_test.go
Normal file
@@ -0,0 +1,74 @@
|
||||
package promptkit_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
type fakeLLMClient struct {
|
||||
response *promptkit.GenerateResponse
|
||||
responses []*promptkit.GenerateResponse
|
||||
err error
|
||||
requests []promptkit.GenerateRequest
|
||||
}
|
||||
|
||||
func (f *fakeLLMClient) Generate(_ context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
f.requests = append(f.requests, request)
|
||||
if f.err != nil {
|
||||
return nil, f.err
|
||||
}
|
||||
if len(f.responses) > 0 {
|
||||
index := len(f.requests) - 1
|
||||
if index >= len(f.responses) {
|
||||
return nil, fmt.Errorf("no response configured for generation %d", index+1)
|
||||
}
|
||||
return f.responses[index], nil
|
||||
}
|
||||
return f.response, nil
|
||||
}
|
||||
|
||||
type countingLLMClient struct{ calls *atomic.Int64 }
|
||||
|
||||
func (client countingLLMClient) Generate(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
if client.calls != nil {
|
||||
client.calls.Add(1)
|
||||
}
|
||||
return &promptkit.GenerateResponse{Content: "ok"}, nil
|
||||
}
|
||||
|
||||
type roundTripFunc func(*http.Request) (*http.Response, error)
|
||||
|
||||
func (f roundTripFunc) RoundTrip(request *http.Request) (*http.Response, error) { return f(request) }
|
||||
|
||||
// newSimpleEngine exercises the public constructor with a complete keyed
|
||||
// document. Tests that need layering or resources select their own inputs.
|
||||
func newSimpleEngine(t *testing.T, content string, options ...promptkit.Option) *promptkit.Engine {
|
||||
t.Helper()
|
||||
engine, err := promptkit.NewEngine(simpleConfig(content), options...)
|
||||
if err != nil {
|
||||
t.Fatalf("construct consumer test engine: %v", err)
|
||||
}
|
||||
return engine
|
||||
}
|
||||
|
||||
func simpleConfig(content string) promptkit.Config {
|
||||
return promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||
Main: &configkit.Input{Source: configkit.Mapping("consumer test", map[string]any{
|
||||
"profiles": map[string]any{"profile": map[string]any{
|
||||
"endpoint": "http://example.test/v1", "model": "model",
|
||||
}},
|
||||
"prompts": map[string]any{"prompt": map[string]any{
|
||||
"version": "1",
|
||||
"default_profile": "profile",
|
||||
"messages": []any{map[string]any{"role": "user", "content": content}},
|
||||
"output": map[string]any{"format": "text", "validation_mode": "none"},
|
||||
}},
|
||||
})},
|
||||
}}
|
||||
}
|
||||
15
convert.go
15
convert.go
@@ -25,7 +25,6 @@ func toDomainRunRequest(req RunRequest) (domain.RunRequest, error) {
|
||||
}
|
||||
return domain.RunRequest{
|
||||
PromptID: req.PromptID,
|
||||
PromptVersion: req.PromptVersion,
|
||||
ProfileID: req.ProfileID,
|
||||
SessionID: req.SessionID,
|
||||
APIKey: req.APIKey,
|
||||
@@ -73,7 +72,7 @@ func fromDomainPreparedRun(prepared *domain.PreparedRun) *PreparedRun {
|
||||
}
|
||||
return &PreparedRun{
|
||||
PromptID: prepared.PromptID,
|
||||
PromptVersion: prepared.PromptVersion,
|
||||
PromptVersion: copyStringPtr(prepared.PromptVersion),
|
||||
PromptHash: prepared.PromptHash,
|
||||
SelectedProfileID: prepared.SelectedProfileID,
|
||||
SelectedBackendID: prepared.SelectedBackendID,
|
||||
@@ -100,7 +99,7 @@ func fromDomainRunResult(result *domain.RunResult) *RunResult {
|
||||
RawOutput: result.RawOutput,
|
||||
Validation: fromDomainValidationResult(result.Validation),
|
||||
PromptID: result.PromptID,
|
||||
PromptVersion: result.PromptVersion,
|
||||
PromptVersion: copyStringPtr(result.PromptVersion),
|
||||
PromptHash: result.PromptHash,
|
||||
SessionID: result.SessionID,
|
||||
RenderedPromptHash: result.RenderedPromptHash,
|
||||
@@ -211,6 +210,12 @@ func toDomainExecutionTargetOverride(override *ExecutionTargetOverride) (*domain
|
||||
}
|
||||
|
||||
func fromDomainExecutionTarget(target domain.ExecutionTarget) ExecutionTarget {
|
||||
return fromDomainExecutionTargetWithOwnedParams(target, copyAnyMap(target.ExtraParams))
|
||||
}
|
||||
|
||||
// fromDomainExecutionTargetWithOwnedParams projects scalar settings and takes
|
||||
// ownership of extraParams, which the caller has already copied.
|
||||
func fromDomainExecutionTargetWithOwnedParams(target domain.ExecutionTarget, extraParams map[string]any) ExecutionTarget {
|
||||
return ExecutionTarget{
|
||||
BackendID: target.BackendID,
|
||||
Endpoint: target.Endpoint,
|
||||
@@ -222,7 +227,7 @@ func fromDomainExecutionTarget(target domain.ExecutionTarget) ExecutionTarget {
|
||||
ServiceTier: target.ServiceTier,
|
||||
ReasoningEffort: target.ReasoningEffort,
|
||||
APIKeyEnv: target.APIKeyEnv,
|
||||
ExtraParams: copyAnyMap(target.ExtraParams),
|
||||
ExtraParams: extraParams,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -252,7 +257,7 @@ func fromDomainPromptInspection(inspection *domain.PromptInspection) *PromptInsp
|
||||
}
|
||||
return &PromptInspection{
|
||||
PromptID: inspection.PromptID,
|
||||
PromptVersion: inspection.PromptVersion,
|
||||
PromptVersion: copyStringPtr(inspection.PromptVersion),
|
||||
PromptHash: inspection.PromptHash,
|
||||
DefaultProfileID: inspection.DefaultProfileID,
|
||||
Inputs: inputs,
|
||||
|
||||
37
doc.go
37
doc.go
@@ -1,27 +1,32 @@
|
||||
// Package promptkit provides an embeddable engine for preparing and executing
|
||||
// prompt-defined LLM workflows.
|
||||
//
|
||||
// Applications construct an [Engine] with [NewEngine], select filesystem or
|
||||
// in-memory sources and optional engine-scoped [Backend] registrations, and
|
||||
// call [Engine.InspectPrompt], [Engine.InspectProfile], [Engine.Prepare],
|
||||
// [Engine.PrepareExecution], [Engine.Run], or [Engine.RunPrepared]. Concrete
|
||||
// registries, repositories, validators, and the built-in OpenAI-compatible
|
||||
// client remain internal implementation details.
|
||||
// Applications construct an [Engine] with [NewEngine], select ConfigKit
|
||||
// sources for prompts, profiles, and backends, and
|
||||
// call [Engine.InspectConfiguration], [Engine.InspectPrompt],
|
||||
// [Engine.InspectRequirements], [Engine.InspectProfile], [Engine.ListProfiles],
|
||||
// [Engine.ListPrompts], [Engine.Prepare], [Engine.PrepareExecution],
|
||||
// [Engine.Run], or [Engine.RunPrepared]. Concrete registries, repositories,
|
||||
// validators, and the built-in OpenAI-compatible client remain internal
|
||||
// implementation details.
|
||||
//
|
||||
// # Concurrency and ownership
|
||||
//
|
||||
// An Engine supports concurrent InspectPrompt, InspectProfile, Prepare,
|
||||
// An Engine supports concurrent InspectConfiguration, InspectPrompt,
|
||||
// InspectRequirements, InspectProfile, ListProfiles, ListPrompts, Prepare,
|
||||
// PrepareExecution, Run, and RunPrepared calls. Engine-local backend policies
|
||||
// bound admitted Run and RunPrepared calls and model generations where
|
||||
// configured, while different backend pools and unlimited backends continue
|
||||
// independently. An injected [LLMClient] or [ArtifactReader] can therefore
|
||||
// still receive concurrent calls and must be safe for that use.
|
||||
//
|
||||
// NewEngine copies in-memory profiles and backend definitions. Prepare,
|
||||
// PrepareExecution, and Run copy request maps, slices, pointer values, and
|
||||
// JSON-compatible extra parameters before using them. InspectPrompt and
|
||||
// InspectProfile return copied inspection values. Returned values and values
|
||||
// passed to extension interfaces are likewise isolated from engine state.
|
||||
// ConfigKit captures the selected configuration when NewEngine constructs an
|
||||
// Engine. Prepare, PrepareExecution, and Run copy request maps, slices, pointer
|
||||
// values, and JSON-compatible extra parameters before using them. InspectPrompt,
|
||||
// InspectRequirements, and InspectProfile return copied inspection values.
|
||||
// ListProfiles and ListPrompts return caller-owned identity slices.
|
||||
// Returned values and values passed to extension interfaces are likewise
|
||||
// isolated from engine state.
|
||||
// Callers own those copies and may mutate them after the call that supplied or
|
||||
// returned them. [CapacityError] values are caller-owned and may be mutated
|
||||
// without affecting engine state or another error. Immutable [GenerationError]
|
||||
@@ -52,9 +57,11 @@
|
||||
// used by those values.
|
||||
//
|
||||
// Construction, inspection, handle, and error values, including [Config],
|
||||
// [Backend], [RunRequest], [ArtifactRef], [ExecutionTargetOverride], [Profile],
|
||||
// [OpenAICompatibleProfileConfig], [ProfileInspection],
|
||||
// [PromptInputDefinition], [PromptInspection], [PreparedExecution],
|
||||
// [ConfigurationSources], [ConfigurationInspection], [BackendConfig],
|
||||
// [ProfileConfig], [RunRequest],
|
||||
// [ArtifactRef], [ExecutionTargetOverride], [ProfileInspection],
|
||||
// [PromptIdentity], [PromptInputDefinition], [PromptInspection],
|
||||
// [RequirementInspectionRequest], [RequirementInspection], [PreparedExecution],
|
||||
// [CapacityError], [GenerationError], [RetryExhaustedError], and
|
||||
// [ExecutionEvent], do not have stable JSON
|
||||
// representations. Direct API keys are nevertheless excluded from JSON for
|
||||
|
||||
60
docs/adr/0001-configuration-boundary.md
Normal file
60
docs/adr/0001-configuration-boundary.md
Normal file
@@ -0,0 +1,60 @@
|
||||
# 0001: Configuration Ownership At The ConfigKit Handoff
|
||||
|
||||
## Status
|
||||
|
||||
Accepted.
|
||||
|
||||
## Date
|
||||
|
||||
2026-09-30
|
||||
|
||||
## Context
|
||||
|
||||
PromptKit is adopting ConfigKit to standardize configuration semantics and
|
||||
simplify its loading machinery. Configuration includes prompts, profiles, and
|
||||
backend definitions contributed by maintained catalogs and consuming applications.
|
||||
ConfigKit supplies loading, composition, precedence, inheritance, and inspection
|
||||
capabilities. PromptKit supplies LLM-specific interpretation and execution.
|
||||
|
||||
A clear handoff assigns each library responsibility for the configuration it
|
||||
owns and keeps validation aligned with the values an application actually uses.
|
||||
|
||||
## Decision
|
||||
|
||||
ConfigKit owns configuration processing up to the handoff of the final effective
|
||||
configuration. PromptKit takes ownership at that handoff and applies its domain
|
||||
interpretation and validation to the final configuration.
|
||||
|
||||
The handoff includes effective presence/null information and resource context
|
||||
supplied by ConfigKit. PromptKit uses this information to interpret effective
|
||||
settings and resolve referenced resources.
|
||||
|
||||
PromptKit passes through ConfigKit's inspection capabilities for explicit
|
||||
operator use. Configuration history and source explanations serve that
|
||||
inspection purpose; domain validation operates on the final effective values.
|
||||
|
||||
This boundary applies uniformly to application configuration and maintained
|
||||
catalog configuration, including backend definitions.
|
||||
|
||||
## Alternatives Considered
|
||||
|
||||
- **Independent source or catalog preflight validation:** preserves checks on
|
||||
individual contributions, but duplicates configuration processing and evaluates
|
||||
values that may be replaced before handoff. Raw-source and intermediate-state
|
||||
validation are outside PromptKit's scope, including equivalent test-only checks.
|
||||
- **History-based source auditing:** can identify overwritten contributions, but
|
||||
couples domain enforcement to configuration history. Historical inspection is
|
||||
an operator capability rather than a validation mechanism.
|
||||
|
||||
## Consequences
|
||||
|
||||
PromptKit's configuration logic and tests focus on the final effective values,
|
||||
their domain meaning, and resulting execution behavior. ConfigKit owns failures
|
||||
arising during its processing, which PromptKit preserves through its error
|
||||
boundary. Catalog maintainers own their repositories' publishing checks.
|
||||
|
||||
The adoption work routes backend configuration through the same handoff and
|
||||
retires PromptKit's source-level and intermediate-configuration validation.
|
||||
Effective resource interpretation and explicit operator inspection remain
|
||||
supported. This concentrates configuration semantics in ConfigKit and reduces
|
||||
duplicated machinery and maintenance in PromptKit.
|
||||
165
docs/consumers/configuration-migration.md
Normal file
165
docs/consumers/configuration-migration.md
Normal file
@@ -0,0 +1,165 @@
|
||||
# Configuration And API Migration
|
||||
|
||||
Promptkit no longer installs a provider catalog or selects prompt definitions
|
||||
by descriptive version labels. Applications now choose configuration inputs
|
||||
explicitly, and execution selects one prompt by its exact ID. This guide
|
||||
explains the shape changes and decisions an application must make; it is not an
|
||||
automatic migration tool. The [framework format reference](../formats.md) owns
|
||||
the current configuration contract, and the root package GoDoc owns exact API
|
||||
signatures and errors.
|
||||
|
||||
## Select And Compose The Configuration You Need
|
||||
|
||||
Supply each backend, profile, prompt, schema, and collection source that the
|
||||
application intends to use. Promptkit does not discover directories or install
|
||||
provider definitions implicitly. An application may select an explicit
|
||||
collection input with `CollectionInput`:
|
||||
|
||||
```go
|
||||
input, err := promptkit.CollectionInput(
|
||||
"openrouter",
|
||||
configkit.File("config/openrouter.yaml"),
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{
|
||||
Sources: promptkit.ConfigurationSources{
|
||||
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{input}}},
|
||||
},
|
||||
})
|
||||
```
|
||||
|
||||
Collection resource keys are local keys, while their runtime identities are
|
||||
qualified. For example, `collections.openrouter.profiles.gemini-flash-latest`
|
||||
is selected as `openrouter/gemini-flash-latest`. Stored backend, prompt-default,
|
||||
and other resource references must use their exact intended IDs; Promptkit does
|
||||
not rewrite references when a source is mounted. Profile `extends` remains a
|
||||
sibling local key in the same collection, not a cross-collection reference.
|
||||
Review collisions and all references when moving resources into collections.
|
||||
|
||||
The root `version` is an optional ConfigKit source header. A collection's
|
||||
`release_version` and a prompt's `version` are optional descriptive labels;
|
||||
neither selects a resource or changes execution. Preserve a label only when it
|
||||
is useful to your application. A formerly version-keyed prompt mapping must be
|
||||
reduced to one deliberately chosen definition per prompt ID, with its old label
|
||||
retained as prompt metadata if desired. Promptkit cannot choose a winner from a
|
||||
consumer's version history.
|
||||
|
||||
For example, change a version-keyed definition:
|
||||
|
||||
```yaml
|
||||
prompts:
|
||||
report:
|
||||
"2.0":
|
||||
messages:
|
||||
- role: user
|
||||
content: Review this report.
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
```
|
||||
|
||||
to one direct definition. Its optional `version` is only a label:
|
||||
|
||||
```yaml
|
||||
prompts:
|
||||
report:
|
||||
version: "2.0"
|
||||
messages:
|
||||
- role: user
|
||||
content: Review this report.
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
```
|
||||
|
||||
## Identity Mapping
|
||||
|
||||
The reviewed catalog identities have slash-free profile keys. Their documented
|
||||
mapping is:
|
||||
|
||||
| Previous identity | Namespaced identity or action |
|
||||
| --- | --- |
|
||||
| Backend `openrouter` | `openrouter/default` |
|
||||
| Backend `rakestrawhome` | `rakestrawhome/default` |
|
||||
| Each OpenRouter catalog profile `p` | `openrouter/p`, retaining the exact local key `p` |
|
||||
| Profile `rakestrawhome-gemma-4-31b` | `rakestrawhome/rakestrawhome-gemma-4-31b` |
|
||||
| Built-in `promptkit-openrouter-standard` | No automatic replacement; optionally define caller profile `openrouter/standard` |
|
||||
| Built-in `promptkit-openrouter-auto` | No automatic replacement; optionally define caller profile `openrouter/auto` |
|
||||
|
||||
These are conversion examples, not aliases or bundled resources. Mounting old
|
||||
`/v3` assets does not update backend references, profile inheritance, prompt
|
||||
defaults, or diagnostic defaults inside them. Edit those values to the exact
|
||||
qualified IDs you chose.
|
||||
|
||||
For example, an optional caller-owned standard profile may extend a selected
|
||||
Gemini profile in the same `openrouter` collection:
|
||||
|
||||
```yaml
|
||||
collections:
|
||||
openrouter:
|
||||
profiles:
|
||||
standard:
|
||||
extends: gemini-flash-latest
|
||||
```
|
||||
|
||||
An optional auto profile may instead select the exact backend and model values:
|
||||
|
||||
```yaml
|
||||
collections:
|
||||
openrouter:
|
||||
profiles:
|
||||
auto:
|
||||
backend: openrouter/default
|
||||
model: openrouter/auto
|
||||
```
|
||||
|
||||
Neither optional profile name is a Promptkit contract. Keep provider model
|
||||
strings and settings as selected by the application, including any meaningful
|
||||
prefixes. If a backend declares diagnostic defaults, update them to exact
|
||||
qualified IDs; an unselected diagnostic default is not automatically made
|
||||
resolvable by migration.
|
||||
|
||||
## Prompt Selection API
|
||||
|
||||
Prompt selection is ID-only. Remove version selectors from calls to `Run`,
|
||||
`Prepare`, `PrepareExecution`, `ValidateRequest`, `InspectRequirements`, and
|
||||
`SmokeTest`. Remove the `PromptVersion` field from `RunRequest`,
|
||||
`RequirementInspectionRequest`, and `SmokeTestRequest`; `InspectPrompt` now
|
||||
has the form `InspectPrompt(ctx, promptID)`. For example, select the prompt
|
||||
explicitly in the request rather than setting a version selector:
|
||||
|
||||
```go
|
||||
request := promptkit.RunRequest{
|
||||
PromptID: "support/report",
|
||||
ProfileID: "support/standard",
|
||||
}
|
||||
_, err = engine.Run(ctx, request)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
```
|
||||
|
||||
If the former format contained multiple versions under one ID, choose the one
|
||||
definition the application should install, and optionally keep its label in
|
||||
the prompt's `version` field. That label remains descriptive metadata and is
|
||||
reported by inspection or results where applicable; it never chooses the
|
||||
definition.
|
||||
|
||||
The public inventory APIs can help an operator inspect a captured effective
|
||||
configuration: `ListProfiles`, `ListPrompts`, and `ListBackends` return exact
|
||||
identities, while `ListNamespaces` and `ListCollections` expose resource-bearing
|
||||
namespaces and effective collection declarations. Inventory does not select
|
||||
which configuration version the application should deploy.
|
||||
|
||||
## Deployment Selection Remains With The Application
|
||||
|
||||
Applications decide which checked-in, embedded, operator-supplied, or
|
||||
programmatic inputs to compose for each deployment. They also decide when to
|
||||
construct a replacement engine after changing those sources. Promptkit does
|
||||
not fetch catalog updates, select provider availability, validate upstream
|
||||
catalog release status, or automatically migrate arbitrary consumer files.
|
||||
The maintained [offline collection example](../../examples/go-library/prepare/main.go)
|
||||
shows explicit source selection, local imports, layering, a scoped view, and
|
||||
operator inventory without a provider call.
|
||||
@@ -15,60 +15,313 @@ Import the package as:
|
||||
import "gitea.maximumdirect.net/eric/promptkit"
|
||||
```
|
||||
|
||||
The maintained [offline preparation example](../../examples/go-library/prepare/main.go)
|
||||
demonstrates explicit collection composition and inventory. Applications
|
||||
adapting older provider or versioned-prompt configuration can use the
|
||||
[configuration migration guide](configuration-migration.md) for the required
|
||||
caller-owned choices.
|
||||
|
||||
The following Go fragments are illustrative and omit surrounding package,
|
||||
import, and error-handling code. Use the maintained examples for complete
|
||||
programs.
|
||||
|
||||
## Go Compatibility
|
||||
|
||||
The `go` directive in the selected release's [module metadata](../../go.mod)
|
||||
declares its minimum supported consumer Go version. Maintainers build and
|
||||
validate releases with the locally selected Go patch release; the minimum
|
||||
does not simply track that patch version.
|
||||
|
||||
Successful release validation on the selected patch is sufficient to presume
|
||||
support across its Go series, subject to the declared minimum and concrete
|
||||
evidence of incompatibility. Specific compatibility concerns are investigated
|
||||
as they arise. Minor releases may raise the minimum; patch releases preserve
|
||||
it. Release notes explain increases and any required consumer action. See the
|
||||
[release policy](../policy/release.md#consumer-minimum-and-release-toolchain).
|
||||
|
||||
## Construct An Engine
|
||||
|
||||
Create an engine with
|
||||
[`NewEngine`](../../engine.go). A directory-backed setup supplies a prompt
|
||||
directory and may supply profile and schema directories:
|
||||
Create an engine with [`NewEngine`](../../engine.go). Select each directory
|
||||
explicitly and place its inputs in an ordered ConfigKit underlay. A group
|
||||
strictly composes its files; later groups can overlay earlier ones:
|
||||
|
||||
```go
|
||||
promptInputs, err := promptkit.DirectoryInputs("prompts")
|
||||
if err != nil { return err }
|
||||
profileInputs, err := promptkit.DirectoryInputs("profiles")
|
||||
if err != nil { return err }
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{
|
||||
PromptDir: "prompts",
|
||||
ProfileDir: "profiles",
|
||||
SchemaDir: "schemas",
|
||||
Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{
|
||||
{Inputs: promptInputs},
|
||||
{Inputs: profileInputs},
|
||||
}},
|
||||
SchemaDir: "schemas",
|
||||
})
|
||||
```
|
||||
|
||||
Options support single-file or `fs.FS` sources, in-memory profiles,
|
||||
engine-scoped backends, and injected artifact or model clients. Consult the
|
||||
[constructor and option GoDoc](../../engine.go) for composition, precedence,
|
||||
validation, and default transport behavior. Source discovery, format
|
||||
validation, and profile precedence are defined by the
|
||||
[framework format reference](../formats.md).
|
||||
For a single highest-priority document, set `Sources.Main` to a
|
||||
`*configkit.Input`; omit it for ordinary layered construction. A single file
|
||||
can use `configkit.File` directly. `ProfileSource`, `BackendSource`, and raw
|
||||
`configkit.Mapping` supply programmatic definitions. Schema and runtime
|
||||
collaborator options remain available. Construction validates the final
|
||||
effective configuration and captures referenced prompt content; construct a
|
||||
new engine to observe configuration edits. See the [constructor GoDoc](../../engine.go)
|
||||
for the exact public contract and the [framework format reference](../formats.md)
|
||||
for definition fields.
|
||||
|
||||
### Use Scoped Engine Views
|
||||
|
||||
Create a view when an application wants short request IDs to resolve within a
|
||||
selected collection:
|
||||
|
||||
```go
|
||||
scoped, err := engine.WithNamespaces(promptkit.Namespaces{
|
||||
Prompt: "support",
|
||||
Profile: "support",
|
||||
})
|
||||
if err != nil { return err }
|
||||
|
||||
prepared, err := scoped.Prepare(ctx, promptkit.RunRequest{
|
||||
PromptID: "answer",
|
||||
ProfileID: "concise",
|
||||
})
|
||||
```
|
||||
|
||||
Execution, preparation, prompt/profile inspection, requirement inspection, and
|
||||
smoke diagnostics qualify slashless caller selections with their respective
|
||||
namespaces. Already-qualified IDs pass through unchanged; configured defaults
|
||||
and references are not rewritten. Each call replaces the view's scope defaults,
|
||||
including clearing a scope with an empty field, without changing the receiver
|
||||
or rebuilding its captured configuration and execution collaborators.
|
||||
Namespace syntax is checked when creating the view; resource existence is
|
||||
checked when selected. Configuration inspection remains global. See
|
||||
[`WithNamespaces`](../../namespaces.go) for the exact public contract.
|
||||
|
||||
## Supply Embedded Application Defaults
|
||||
|
||||
Use `WithFallbackProfileFS` when an application packages profile definitions
|
||||
that should apply unless an operator provides an ordinary configured profile
|
||||
with the same ID. For example, an application can embed its defaults while
|
||||
continuing to use `ProfileDir` for operator overrides:
|
||||
Use `FilesystemInputs` for embedded defaults and `DirectoryInputs` for an
|
||||
explicitly chosen operator directory. A later programmatic group can override
|
||||
both. No directory name has built-in precedence:
|
||||
|
||||
```go
|
||||
//go:embed profiles/*.yaml
|
||||
var applicationProfiles embed.FS
|
||||
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{
|
||||
PromptDir: "prompts",
|
||||
ProfileDir: operatorProfileDir,
|
||||
},
|
||||
promptkit.WithFallbackProfileFS(applicationProfiles, "profiles"),
|
||||
)
|
||||
defaults, err := promptkit.FilesystemInputs("embedded defaults", applicationFS, "config")
|
||||
if err != nil { return err }
|
||||
operator, err := promptkit.DirectoryInputs(operatorDirectory)
|
||||
if err != nil { return err }
|
||||
profile := promptkit.ProfileSource("application choices", map[string]promptkit.ProfileConfig{
|
||||
"summary": {Model: &selectedModel},
|
||||
})
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||
Underlays: []configkit.Underlay{
|
||||
{Inputs: defaults},
|
||||
{Inputs: operator},
|
||||
{Inputs: []configkit.Input{{Source: profile}}},
|
||||
},
|
||||
}})
|
||||
```
|
||||
|
||||
Keep application-owned profile IDs and definitions in the embedded source.
|
||||
Use the ordinary configured profile source for operator overrides. Leave
|
||||
`operatorProfileDir` empty when the operator did not configure an override
|
||||
directory; a non-empty path names an authoritative higher-precedence source,
|
||||
so an unavailable or unreadable directory is an error rather than a reason to
|
||||
fall back. The
|
||||
[framework format reference](../formats.md#source-and-profile-precedence)
|
||||
owns the exact profile format and lookup order; the
|
||||
[`WithFallbackProfileFS` GoDoc](../../engine.go) owns its option contract and
|
||||
validation rules.
|
||||
Only add the operator group when the application selects that directory; an
|
||||
explicitly selected missing or unreadable root is an error. Each helper finds
|
||||
`.yaml` and `.yml` files recursively and disables imports by default. The
|
||||
caller may enable imports on individual inputs. A nil `Sources.Main` is an
|
||||
empty main source, not another directory lookup.
|
||||
|
||||
For a programmatic document with explicit null or a prompt definition, use a
|
||||
labeled mapping as an input. A mapping owns a snapshot of supported values:
|
||||
|
||||
```go
|
||||
document := configkit.Mapping("application configuration", map[string]any{
|
||||
"profiles": map[string]any{
|
||||
"local-summary": map[string]any{
|
||||
"backend": "local", "model": "example-model",
|
||||
"temperature": nil, // Suppress an inherited provider control.
|
||||
},
|
||||
},
|
||||
})
|
||||
input := configkit.Input{Source: document}
|
||||
// Add input to a later underlay, or set Sources.Main = &input.
|
||||
```
|
||||
|
||||
`ProfileSource` and `BackendSource` are smaller typed conveniences when null
|
||||
is not needed. Backend and profile definitions are supplied by the caller;
|
||||
there is no registration-only backend namespace.
|
||||
|
||||
When configuration or message files change, build and validate a new engine,
|
||||
then switch the application to that engine for subsequent work. Existing
|
||||
engines and prepared handles retain their captured definitions; schema and
|
||||
request artifact resources still follow their runtime lifecycle. Inspection
|
||||
does not watch or reload sources automatically.
|
||||
|
||||
## Discover Available Resources And Collections
|
||||
|
||||
Use [`Engine.ListProfiles`](../../listing.go),
|
||||
[`Engine.ListPrompts`](../../listing.go), and
|
||||
[`Engine.ListBackends`](../../listing.go) to discover exact identities from the
|
||||
engine's final captured configuration. A scoped view filters by its respective
|
||||
namespace by default; pass `NamespaceNamed`, `NamespaceUnqualified`, or
|
||||
`NamespaceAll` to override that listing filter without changing execution
|
||||
scope. Returned IDs remain full identities and can be used for exact selection:
|
||||
|
||||
```go
|
||||
profiles, err := engine.ListProfiles(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(profiles) > 0 {
|
||||
_, err := engine.InspectProfile(ctx, profiles[0])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
prompts, err := engine.ListPrompts(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(prompts) > 0 {
|
||||
selected := prompts[0]
|
||||
inspection, err := engine.InspectPrompt(ctx, selected.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_ = inspection
|
||||
}
|
||||
```
|
||||
|
||||
Use [`Engine.ListNamespaces`](../../listing.go) to discover resource-bearing
|
||||
namespaces across the entire captured configuration, independent of view
|
||||
scopes. [`Engine.ListCollections`](../../listing.go) reports effective
|
||||
collection declarations, including metadata-only entries; it does not infer
|
||||
collection metadata from resource membership or source history.
|
||||
|
||||
These lists describe effective membership, not ConfigKit contribution history,
|
||||
and do not promise that a profile is ready to execute. Use inspection for
|
||||
details and preparation to resolve request resources. The linked method GoDoc
|
||||
owns ordering, ownership, concurrency, and error behavior; identifiers may be
|
||||
sensitive application data.
|
||||
|
||||
## Inspect The Captured Configuration
|
||||
|
||||
Use [`Engine.InspectConfiguration`](../../configuration_inspection.go) for an
|
||||
operator-selected report of effective configuration, source descriptions,
|
||||
diagnostics, and path history. For a literal ID containing a slash, construct
|
||||
the semantic path with `configkit.Keys`:
|
||||
|
||||
```go
|
||||
view, err := engine.InspectConfiguration()
|
||||
if err != nil { return err }
|
||||
path := configkit.Keys("profiles", "team/summary", "model")
|
||||
explanation, found := view.Explain(path)
|
||||
if found && explanation.Effective.Redacted {
|
||||
// Apply the application's disclosure policy before sharing the report.
|
||||
}
|
||||
```
|
||||
|
||||
`WriteEffective` and `WriteExplanation` produce ConfigKit-redacted reports;
|
||||
the application decides whether those reports may be disclosed. Credential-
|
||||
shaped provider keys are redacted, but arbitrary prompt and provider content
|
||||
is not necessarily safe to share. Inspection uses the engine's captured result
|
||||
and does not reopen sources or resolve credentials. `InspectProfile` instead
|
||||
shows backend-completed execution settings for one profile, and `Prepare` or
|
||||
`PreparedExecution.Details` shows request-specific rendered messages, inputs,
|
||||
and validation resources. None of those views is a substitute for the others.
|
||||
|
||||
To review changes between two independently captured configurations, compare
|
||||
their inspection views:
|
||||
|
||||
```go
|
||||
earlier, err := earlierEngine.InspectConfiguration()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
later, err := candidateEngine.InspectConfiguration()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
differences, err := earlier.Diff(later)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, difference := range differences {
|
||||
if difference.HiddenChanged {
|
||||
// Report a protected change without inferring its value.
|
||||
}
|
||||
if difference.Left.Redacted || difference.Right.Redacted {
|
||||
// Apply the application's disclosure policy before presenting snapshots.
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
See the [`ConfigurationInspection.Diff` GoDoc](../../configuration_inspection.go)
|
||||
and ConfigKit's [v0.8.0 effective-configuration diff guidance](https://gitea.maximumdirect.net/eric/ConfigKit/src/tag/v0.8.0/docs/config.md#effective-configuration-diffs)
|
||||
for the exact comparison contract. The operation compares only frozen effective
|
||||
configuration values: it does not reread external files or compare their
|
||||
contents, resolved credential contents, later execution preparation, or
|
||||
request-specific overrides. An empty diff is not proof of equivalent execution.
|
||||
Records can contain ordinary application data, so use the application's
|
||||
disclosure policy when presenting them.
|
||||
|
||||
## Run Explicit Smoke Diagnostics
|
||||
|
||||
[`Engine.SmokeTest`](../../smoke_diagnostics.go) runs one explicit diagnostic
|
||||
using a target resolved from the engine's captured configuration. It does not
|
||||
execute the selected application prompt or its inputs, session, artifacts, or
|
||||
output schema. Unlike the offline examples and default test suite, calling it
|
||||
can contact the selected provider and use configured credentials. It is a
|
||||
consumer operation, not an automatic readiness verdict or a release check.
|
||||
|
||||
For example, a caller can request the built-in text inference check and treat
|
||||
its optional fact as an observation rather than assuming that a nil Go error
|
||||
means success:
|
||||
|
||||
```go
|
||||
report, err := engine.SmokeTest(ctx, promptkit.SmokeTestRequest{
|
||||
Kind: promptkit.SmokeTestTextInference,
|
||||
ProfileID: profileID,
|
||||
APIKey: apiKey,
|
||||
})
|
||||
if err != nil {
|
||||
if report != nil {
|
||||
retainDisclosureSafeSummary(report)
|
||||
}
|
||||
return err
|
||||
}
|
||||
if report != nil {
|
||||
matched := report.Facts.ExpectedTextMatched
|
||||
switch {
|
||||
case matched == nil:
|
||||
// The comparison was not evaluated or is unknown/inapplicable.
|
||||
case *matched:
|
||||
// The private expected-text comparison matched.
|
||||
default:
|
||||
// It did not match; the operation may still have concluded normally.
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
A nil Go error means the diagnostic concluded, not that its finding was
|
||||
positive. Fact pointers are tri-state: nil means unknown, not evaluated, or
|
||||
inapplicable. Reports contain factual observations and effective settings, not
|
||||
generated text or provider bodies; identifiers and settings can still be
|
||||
sensitive. The
|
||||
[`SmokeTest` GoDoc](../../smoke_diagnostics.go) owns selection precedence,
|
||||
exact fields, timeout and retry behavior, supported transports, and disclosure
|
||||
details.
|
||||
|
||||
Caller cancellation can return both an error and a partial report. Preserve the
|
||||
context error identity and inspect any available observations before returning:
|
||||
|
||||
```go
|
||||
report, err := engine.SmokeTest(ctx, request)
|
||||
if report != nil {
|
||||
retainDisclosureSafeSummary(report)
|
||||
}
|
||||
if err != nil {
|
||||
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
|
||||
// The report may describe work completed before interruption.
|
||||
}
|
||||
return err
|
||||
}
|
||||
```
|
||||
|
||||
## Inspect A Prompt Before Preparation
|
||||
|
||||
@@ -77,7 +330,7 @@ declared inputs and output workflow without creating placeholder inputs or
|
||||
resolving a profile:
|
||||
|
||||
```go
|
||||
inspection, err := engine.InspectPrompt(ctx, "meeting.summary", "")
|
||||
inspection, err := engine.InspectPrompt(ctx, "meeting.summary")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -94,6 +347,85 @@ rendered messages, and use prepared execution when that work must remain tied
|
||||
to later execution. The method's [GoDoc](../../engine.go) owns exact fields,
|
||||
hash, ownership, and error semantics.
|
||||
|
||||
## Inspect Requirements Before Preparation
|
||||
|
||||
Use [`Engine.InspectRequirements`](../../requirements.go) when the application
|
||||
needs the prompt's recognizable input and caller-variable dependencies for a
|
||||
particular context before it has input artifacts. Unlike `InspectPrompt`, it
|
||||
also walks captured template syntax and selects a profile identity; unlike
|
||||
`Prepare`, it does not load artifacts or render messages.
|
||||
|
||||
```go
|
||||
requirements, err := engine.InspectRequirements(ctx, promptkit.RequirementInspectionRequest{
|
||||
PromptID: "meeting.summary",
|
||||
Vars: map[string]string{"audience": "project leads"},
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, input := range requirements.Inputs {
|
||||
if input.Kind == promptkit.RequirementRequired {
|
||||
// Plan to bind this input before preparation.
|
||||
}
|
||||
if input.Kind == promptkit.RequirementOptional && input.Explanation != "" {
|
||||
// A recognizable reference may occur only on a known skipped path.
|
||||
}
|
||||
}
|
||||
for _, variable := range requirements.Variables {
|
||||
if variable.Kind == promptkit.RequirementRequired && !variable.Supplied {
|
||||
// The key is required even though this inspection did not receive it.
|
||||
}
|
||||
}
|
||||
if !requirements.Complete {
|
||||
// Preserve the partial inventory and account for its discovery issues.
|
||||
}
|
||||
```
|
||||
|
||||
Optional entries can include recognizable inputs and variables found only in
|
||||
inactive branches. A missing required variable is reported separately from
|
||||
discovery completeness: missing a value used only for direct text substitution
|
||||
does not by itself make the inventory incomplete. Conversely, an unresolved
|
||||
dependency-bearing branch can make discovery incomplete while retaining known
|
||||
conditional entries. Do not treat an empty partial inventory as proof that no
|
||||
input bindings are needed. See the method's [GoDoc](../../requirements.go) for
|
||||
the exact classifications, supported syntax, ownership, and error contract.
|
||||
|
||||
## Validate A Concrete Request
|
||||
|
||||
Use [`Engine.ValidateRequest`](../../engine.go) when the application has a
|
||||
complete `RunRequest` and needs to know whether that concrete invocation passes
|
||||
preparation:
|
||||
|
||||
```go
|
||||
request := promptkit.RunRequest{
|
||||
PromptID: "meeting.summary",
|
||||
Vars: map[string]string{"audience": "project leads"},
|
||||
Inputs: map[string]promptkit.ArtifactRef{
|
||||
"note": promptkit.Inline("Synthetic meeting notes"),
|
||||
},
|
||||
}
|
||||
if err := engine.ValidateRequest(ctx, request); err != nil {
|
||||
return err
|
||||
}
|
||||
```
|
||||
|
||||
Unlike `InspectRequirements`, this checks supplied values, the selected template
|
||||
branches, and request bindings through the existing preparation path. It checks
|
||||
only this invocation, does not aggregate errors, and loads every supplied input
|
||||
according to ordinary preparation behavior, even if a branch does not use it.
|
||||
It makes no model call and returns no prepared snapshot, but artifact and schema
|
||||
resource I/O and its time and memory costs remain. Success does not certify
|
||||
provider acceptance or later execution success. See the method's GoDoc for its
|
||||
exact error and context behavior.
|
||||
|
||||
Scriptorium can map the returned error to its own diagnostics and exit status.
|
||||
Its `validate` and `run` commands should construct equivalent requests for
|
||||
equivalent caller inputs so their preparation behavior agrees. Separate
|
||||
subprocess calls prepare independently; an opaque `PrepareExecution` handle
|
||||
cannot be transferred from `validate` to `run`. Use `PrepareExecution` when one
|
||||
process and engine must execute the exact prepared snapshot.
|
||||
|
||||
## Prepare Without Model Execution
|
||||
|
||||
[`Engine.Prepare`](../../engine.go) resolves the selected prompt and profile,
|
||||
@@ -114,8 +446,11 @@ prepared, err := engine.Prepare(ctx, promptkit.RunRequest{
|
||||
The maintained
|
||||
[offline preparation example](../../examples/go-library/prepare/main.go)
|
||||
shows a complete runnable setup with a prompt file, in-memory profile, and
|
||||
inline input. Exact request requirements and prepared-result fields belong to
|
||||
the [`RunRequest` and `PreparedRun` GoDoc](../../types.go).
|
||||
inline input. It demonstrates validation and preparation with the same request
|
||||
to make the two operations concrete; this performs preparation twice and is
|
||||
illustrative, not a requirement to validate before `Prepare` or `Run`. Exact
|
||||
request requirements and prepared-result fields belong to the
|
||||
[`RunRequest` and `PreparedRun` GoDoc](../../types.go).
|
||||
|
||||
## Prepare Now And Execute The Same Snapshot Later
|
||||
|
||||
@@ -268,8 +603,8 @@ result, err := engine.Run(ctx, request)
|
||||
|
||||
These messages are already rendered: Promptkit does not template or resolve
|
||||
files in them, and they can contain sensitive model output or application
|
||||
feedback. Promptkit remains stateless; every `Run` call re-resolves its current
|
||||
sources and the application owns any semantic retry budget. When a
|
||||
feedback. Every `Run` call uses the engine's captured configuration; the
|
||||
application owns any semantic retry budget. When a
|
||||
pre-execution equality check is required, use `PrepareExecution` and compare
|
||||
its opaque rendered-prompt hash before invoking `RunPrepared`.
|
||||
|
||||
@@ -282,84 +617,48 @@ replace execution settings or the complete output contract.
|
||||
The [public value GoDoc](../../types.go) defines nil, empty, zero, replacement,
|
||||
copy, and credential behavior. The
|
||||
[framework format reference](../formats.md) defines how those request values
|
||||
interact with prompt definitions, file-backed and application fallback
|
||||
profiles, built-ins, schemas, and framework defaults.
|
||||
interact with prompt definitions, layered profiles, built-ins, schemas, and
|
||||
framework defaults.
|
||||
|
||||
For programmatic profiles,
|
||||
[`OpenAICompatibleProfile`](../../profiles.go) converts ordinary
|
||||
OpenAI-compatible settings into a value accepted by `WithProfiles`.
|
||||
For programmatic profiles, use [`ProfileSource`](../../profiles.go) with keyed
|
||||
`ProfileConfig` values. Nil fields are omitted; pointers preserve explicit
|
||||
zero, false, and blank values. Raw `configkit.Mapping` expresses explicit null.
|
||||
|
||||
### Migrate From The Previous Configuration API
|
||||
|
||||
Replace `Config.PromptDir` and `Config.ProfileDir` with selected ConfigKit
|
||||
inputs in `Config.Sources`. The removed `WithPromptFS`, `WithPromptFile`,
|
||||
`WithProfileFS`, `WithProfileFile`, `WithFallbackProfileFS`, `WithProfiles`,
|
||||
and `WithBackend` options have no compatibility wrappers. Use
|
||||
`DirectoryInputs` or `FilesystemInputs` for a strict-composition file group,
|
||||
ConfigKit file inputs for individual files, and `ProfileSource`,
|
||||
`BackendSource`, or `configkit.Mapping` for programmatic values. Backend IDs
|
||||
are keys in configuration, not registrations. Choose caller underlays in
|
||||
fallback-to-operator-to-programmatic order; use `Sources.Main` for one final
|
||||
highest-priority input.
|
||||
|
||||
Move embedded IDs into keyed `profiles` and `prompts` maps; each prompt ID now
|
||||
contains one installed definition, with any descriptive `version` label inside
|
||||
that definition. Convert `base_profile` to ConfigKit `extends`. For the current
|
||||
collection identity mapping, ID-only prompt selection, and required
|
||||
caller-owned choices, see the
|
||||
[configuration migration guide](configuration-migration.md). Exact current
|
||||
format, reference, inheritance, and presence semantics belong to the
|
||||
[framework format reference](../formats.md). Prompt text and static templates
|
||||
are captured at construction, not automatically reloaded between runs. Replace
|
||||
the engine to pick up changed definitions or message files.
|
||||
|
||||
### Choose A Profile Ownership Model
|
||||
|
||||
Choose the profile source that matches the application's continuity needs:
|
||||
|
||||
- Select `promptkit-openrouter-standard` or `promptkit-openrouter-auto` when
|
||||
the maintained literal name is the desired consumer contract. Their current
|
||||
selections are in the [format reference](../formats.md#built-in-profile-catalog).
|
||||
- Select an external catalog profile directly when the application accepts that
|
||||
profile's catalog volatility. The selected external inventories are linked
|
||||
from the [format reference](../formats.md#external-profile-inventories).
|
||||
- Define a complete application-owned profile with an explicit backend or
|
||||
endpoint, model, and settings when the application owns those choices.
|
||||
|
||||
An application profile that inherits an external base follows later changes to
|
||||
that base and fails if the base disappears; it is not a frozen copy. Promptkit
|
||||
tests the catalog versions selected in its module graph. Independently advancing
|
||||
a catalog dependency can remove an inherited base and produce an untested
|
||||
combination. Pinning the resolved module graph preserves the embedded data for
|
||||
a build, but cannot preserve provider availability. Running binaries neither
|
||||
download catalog updates nor automatically replace withdrawn models. See the
|
||||
[release policy](../policy/release.md#catalog-profile-compatibility) for the exact
|
||||
cross-version guarantees.
|
||||
|
||||
### Alias A Built-In Profile
|
||||
|
||||
Give an application-owned profile ID a built-in base when prompts should select
|
||||
the application ID while inheriting the built-in target. The child can override
|
||||
only the setting it owns:
|
||||
|
||||
```go
|
||||
promptkit.WithProfiles(promptkit.Profile{
|
||||
ID: "weather-light",
|
||||
BaseProfileID: "promptkit-openrouter-standard",
|
||||
ReasoningEffort: "high",
|
||||
})
|
||||
```
|
||||
|
||||
Select `weather-light` in a prompt or `RunRequest.ProfileID`; it remains the
|
||||
reported selected profile. See the [profile inheritance format
|
||||
reference](../formats.md#profile-inheritance) and the
|
||||
[`Profile` GoDoc](../../types.go) for exact lookup, merging, and validation
|
||||
behavior. If the base is an external profile, apply the volatility guidance
|
||||
above when deciding whether to use an application-owned complete definition
|
||||
instead.
|
||||
|
||||
### Use The Rakestrawhome Built-In Profile
|
||||
|
||||
Set `RAKESTRAWHOME_INFERENCE_API_KEY` in the application environment, then
|
||||
select `rakestrawhome-gemma-4-31b` as an ordinary profile ID. For example, a
|
||||
prepared result identifies the selected built-in through
|
||||
`BackendRakestrawHome`:
|
||||
|
||||
```go
|
||||
prepared, err := engine.Prepare(ctx, promptkit.RunRequest{
|
||||
PromptID: "meeting.summary",
|
||||
ProfileID: "rakestrawhome-gemma-4-31b",
|
||||
Inputs: inputs,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if prepared.SelectedBackendID != promptkit.BackendRakestrawHome {
|
||||
return fmt.Errorf("unexpected backend %q", prepared.SelectedBackendID)
|
||||
}
|
||||
```
|
||||
|
||||
Do not register `rakestrawhome` manually. When adopting this built-in, remove
|
||||
an existing `WithBackend` registration with that exact ID; retaining it causes
|
||||
the intentional duplicate-ID configuration error. Direct request credentials
|
||||
and runtime endpoint overrides remain supported under their ordinary GoDoc and
|
||||
format contracts.
|
||||
Supply backend and profile definitions through the configuration sources your
|
||||
application selects. PromptKit does not install provider catalogs or built-in
|
||||
profile inventories; conventional backend ID constants are names only. Define
|
||||
the model, endpoint, credentials lookup, and settings that your application
|
||||
intends to support. A profile may inherit a sibling profile by its local key in
|
||||
the same root or collection mapping; its backend reference uses the exact
|
||||
configured identity. Caller-selected source layers determine the final values. The
|
||||
[format reference](../formats.md) describes the definition fields and
|
||||
composition rules.
|
||||
|
||||
### Inspect A Profile Before Prompt Work
|
||||
|
||||
@@ -424,19 +723,15 @@ Choose the smallest configuration that fits how the endpoint will be reused.
|
||||
|
||||
#### Use An Endpoint-Only Profile
|
||||
|
||||
Put the endpoint directly on an in-memory profile when only that profile needs
|
||||
Put the endpoint directly on a programmatic profile when only that profile needs
|
||||
it and shared backend identity or capacity policy is unnecessary:
|
||||
|
||||
```go
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{
|
||||
PromptDir: "prompts",
|
||||
},
|
||||
promptkit.WithProfiles(promptkit.Profile{
|
||||
ID: "local-summary",
|
||||
Endpoint: "http://localhost:8000/v1",
|
||||
Model: "example-model",
|
||||
}),
|
||||
)
|
||||
endpoint, model := "http://localhost:8000/v1", "example-model"
|
||||
profile := promptkit.ProfileSource("local profile", map[string]promptkit.ProfileConfig{
|
||||
"local-summary": {Endpoint: &endpoint, Model: &model},
|
||||
})
|
||||
// Add configkit.Input{Source: profile} to a caller underlay.
|
||||
```
|
||||
|
||||
Endpoint-only profiles have an empty backend ID and remain unrestricted by
|
||||
@@ -448,59 +743,44 @@ Use `LocalBackend` when profiles should share the conventional `local`
|
||||
identity, endpoint, and concurrency limit:
|
||||
|
||||
```go
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{
|
||||
PromptDir: "prompts",
|
||||
},
|
||||
promptkit.WithBackend(
|
||||
promptkit.LocalBackend("http://localhost:8000/v1", 2),
|
||||
),
|
||||
promptkit.WithProfiles(promptkit.Profile{
|
||||
ID: "local-summary",
|
||||
BackendID: promptkit.BackendLocal,
|
||||
Model: "example-model",
|
||||
}),
|
||||
)
|
||||
backendID, model := promptkit.BackendLocal, "example-model"
|
||||
backend := promptkit.BackendSource("local backend", map[string]promptkit.BackendConfig{
|
||||
backendID: promptkit.LocalBackend("http://localhost:8000/v1", 2),
|
||||
})
|
||||
profile := promptkit.ProfileSource("local profile", map[string]promptkit.ProfileConfig{
|
||||
"local-summary": {Backend: &backendID, Model: &model},
|
||||
})
|
||||
// Add both sources as inputs in a caller underlay.
|
||||
```
|
||||
|
||||
The helper is explicit: it does not pre-register a backend or read environment
|
||||
variables. Supplying a positive limit leaves queue capacity omitted, so normal
|
||||
backend registration selects the existing default waiting capacity of 1024.
|
||||
The returned value still enters the engine through `WithBackend`.
|
||||
The helper does not choose an ID or read environment variables. A positive
|
||||
concurrency limit with omitted queue capacity selects the default waiting
|
||||
capacity of 1024.
|
||||
|
||||
#### Configure A Complete Backend
|
||||
|
||||
Use a keyed `Backend` value for authentication, extra request parameters, an
|
||||
Use a keyed `BackendConfig` value for authentication, extra request parameters, an
|
||||
explicit queue capacity, a custom ID, or multiple local endpoints:
|
||||
|
||||
```go
|
||||
noWaiting := 0
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{
|
||||
PromptDir: "prompts",
|
||||
},
|
||||
promptkit.WithBackend(promptkit.Backend{
|
||||
ID: "local-gpu",
|
||||
Endpoint: "http://gpu-host:8000/v1",
|
||||
APIKeyEnv: "LOCAL_GPU_API_KEY",
|
||||
ExtraParams: map[string]any{"provider_option": "enabled"},
|
||||
ConcurrencyLimit: 2,
|
||||
QueueCapacity: &noWaiting,
|
||||
}),
|
||||
promptkit.WithProfiles(promptkit.Profile{
|
||||
ID: "gpu-summary",
|
||||
BackendID: "local-gpu",
|
||||
Model: "example-model",
|
||||
}),
|
||||
)
|
||||
endpoint, keyName, limit, noWaiting := "http://gpu-host:8000/v1", "LOCAL_GPU_API_KEY", 2, 0
|
||||
backend := promptkit.BackendSource("GPU backend", map[string]promptkit.BackendConfig{
|
||||
"local-gpu": {
|
||||
Endpoint: &endpoint, APIKeyEnv: &keyName,
|
||||
ConcurrencyLimit: &limit, QueueCapacity: &noWaiting,
|
||||
ExtraParams: map[string]any{"provider_option": "enabled"},
|
||||
},
|
||||
})
|
||||
```
|
||||
|
||||
Use distinct custom IDs when registering multiple local endpoints.
|
||||
Registrations belong to one engine and custom IDs cannot replace built-ins.
|
||||
The [`Backend`, `LocalBackend`, and `WithBackend` GoDoc](../../backends.go)
|
||||
defines exact construction, validation, copying, uniqueness, concurrency, and
|
||||
request-default behavior.
|
||||
Place `backend` in a caller underlay and select `"local-gpu"` from a
|
||||
`ProfileConfig.Backend` pointer or a keyed file profile. All backend
|
||||
contributions use the same ConfigKit layering rules, including overlays of
|
||||
maintained definitions. The [`BackendConfig` and `LocalBackend` GoDoc](../../backends.go)
|
||||
defines exact omission and construction behavior.
|
||||
|
||||
Both file-backed and in-memory profiles select a registration through
|
||||
`backend` or `Profile.BackendID`. Profile and request endpoint overrides retain
|
||||
Both file-backed and programmatic profiles select a backend through
|
||||
`backend` or `ProfileConfig.Backend`. Profile and request endpoint overrides retain
|
||||
that routing and capacity identity. `PreparedRun.SelectedBackendID`,
|
||||
`RunResult.SelectedBackendID`, and the effective `ExecutionTarget.BackendID`
|
||||
expose it to consumers and injected model clients. Endpoint-only profiles
|
||||
@@ -508,7 +788,7 @@ remain supported and expose an empty backend ID.
|
||||
|
||||
### Limit Backend Concurrency
|
||||
|
||||
Set `Backend.ConcurrencyLimit` when a backend needs protection from too many
|
||||
Set `BackendConfig.ConcurrencyLimit` when a backend needs protection from too many
|
||||
simultaneous model calls. Leaving `QueueCapacity` nil, as in the local-backend
|
||||
example above, selects the default waiting capacity of 1024.
|
||||
|
||||
@@ -517,24 +797,23 @@ zero:
|
||||
|
||||
```go
|
||||
noWaiting := 0
|
||||
backend := promptkit.Backend{
|
||||
ID: "local-gpu",
|
||||
Endpoint: "http://gpu-host:8000/v1",
|
||||
ConcurrencyLimit: 2,
|
||||
QueueCapacity: &noWaiting,
|
||||
backend := promptkit.BackendConfig{
|
||||
Endpoint: &endpoint,
|
||||
ConcurrencyLimit: &limit,
|
||||
QueueCapacity: &noWaiting,
|
||||
}
|
||||
```
|
||||
|
||||
The pointer distinguishes an explicit zero from omission. Keep using keyed
|
||||
`Backend` literals so additive configuration fields remain source-compatible.
|
||||
The pointer distinguishes an explicit zero from omission. Use keyed
|
||||
`BackendConfig` literals inside `BackendSource` for programmatic definitions.
|
||||
Capacity belongs to one engine and the selected backend ID; endpoint-only
|
||||
profiles and custom backends without a configured limit remain unrestricted.
|
||||
Exact validation, defaulting, ownership, and concurrency semantics belong to
|
||||
the [`Backend` GoDoc](../../backends.go).
|
||||
the [`BackendConfig` GoDoc](../../backends.go).
|
||||
|
||||
## Credentials
|
||||
|
||||
File-backed profiles name an environment variable; in-memory profiles can
|
||||
File-backed and programmatic profiles can name an environment variable or
|
||||
require a direct request key. Direct keys are request-scoped and are excluded
|
||||
from supported JSON values and the package's `String` and `GoString`
|
||||
summaries. The exact precedence and redaction guarantees belong to
|
||||
@@ -568,12 +847,13 @@ the application policy that motivated the injection.
|
||||
|
||||
Use `errors.Is` with the
|
||||
[public error sentinels and operation GoDoc](../../engine.go). The declarations
|
||||
distinguish invalid construction, invalid requests, absent sources,
|
||||
source-loading failures, collaborator failures, and operational validation
|
||||
distinguish invalid construction, invalid requests, absent selections,
|
||||
collaborator failures, and operational validation
|
||||
failures. Specific request conditions may also match the broader
|
||||
`ErrInvalidRequest`, and injected collaborator identities are preserved where
|
||||
documented. Invalid or duplicate backend registrations match
|
||||
`ErrInvalidConfig`; selecting an unknown backend matches `ErrProfileLoad`.
|
||||
documented. Invalid effective definitions, including unknown backend
|
||||
references, match `ErrInvalidConfig` during construction. ConfigKit load
|
||||
errors remain available through `errors.As`.
|
||||
|
||||
When a limited backend has admitted all active and waiting calls, handle
|
||||
`ErrCapacityExceeded` separately from request errors and provider failures:
|
||||
|
||||
@@ -1,9 +1,7 @@
|
||||
# Development
|
||||
|
||||
This is the contributor entry point for Promptkit, a reusable Go library. All
|
||||
contributors must read the
|
||||
[architecture policy](policy/architecture.md) before making changes. Agents must
|
||||
also follow the [agent policy](policy/agents.md) before recommendations or work.
|
||||
This is the contributor entry point for Promptkit, a reusable Go library.
|
||||
Use the Policy Reading Guide below to select policies applicable to the task.
|
||||
|
||||
## Initial Orientation
|
||||
|
||||
@@ -16,18 +14,27 @@ Before starting work:
|
||||
4. keep documentation limited to implemented behavior unless an accepted
|
||||
decision or temporary roadmap explicitly owns future work.
|
||||
|
||||
Start with:
|
||||
## Policy Reading Guide
|
||||
|
||||
- the [architecture policy](policy/architecture.md) for library boundaries,
|
||||
dependency direction, invariants, and non-goals;
|
||||
- the [internal component overview](internal/overview.md) for the current
|
||||
package and component inventory;
|
||||
- the [documentation policy](policy/documentation.md) before changing
|
||||
documentation;
|
||||
- the [testing policy](policy/testing.md) before adding, rewriting, or deleting
|
||||
tests; and
|
||||
- the [release policy](policy/release.md) and [release procedure](release.md)
|
||||
for version and publication work.
|
||||
| Policy | Read before… |
|
||||
| --- | --- |
|
||||
| [Policy set provenance](policy/meta.md) | Adopting policies, checking upstream revisions, or updating the adopted policy set. |
|
||||
| [Agents](policy/agents.md) | Recommending, planning, or implementing repository changes. |
|
||||
| [Architecture](policy/architecture.md) | Planning or changing library code or its structure. |
|
||||
| [Testing](policy/testing.md) | Selecting validation for code changes, or adding, changing, removing, or evaluating tests. |
|
||||
| [Documentation](policy/documentation.md) | Writing, revising, or reorganizing documentation. |
|
||||
| [OSS](policy/oss.md) | Adding or updating dependencies or incorporated third-party material, or changing licensing or attribution. |
|
||||
| [Release](policy/release.md) | Preparing or publishing a release, changing release procedures, or making versioning and compatibility decisions. |
|
||||
|
||||
- Read applicable policies before making decisions they govern; multiple policies
|
||||
may apply.
|
||||
- Select policies using the task triggers above. Follow cross-references to
|
||||
relevant sections when needed to resolve a concrete question.
|
||||
- Reuse policies already available in context; reread when changed or no longer
|
||||
available.
|
||||
- Expand reading when the task's scope expands.
|
||||
- When revising a policy itself, read that policy and any provisions elsewhere
|
||||
affected by the revision.
|
||||
|
||||
## Task-Specific Reading Guide
|
||||
|
||||
@@ -36,13 +43,13 @@ Start with:
|
||||
| Root public API | The [architecture policy](policy/architecture.md), [consumer guide](consumers/pkg-promptkit.md), [testing policy](policy/testing.md), and existing GoDoc. |
|
||||
| Prompt, profile, or schema formats | The [framework format reference](formats.md), owning parser or validator package, and [documentation policy](policy/documentation.md). |
|
||||
| Source loading or validation | The [framework format reference](formats.md), [internal source document](internal/sources.md), and owning package tests. |
|
||||
| Maintained external catalogs or built-ins | The [internal source document](internal/sources.md), [framework format reference](formats.md), [testing policy](policy/testing.md), both catalog module repositories, each catalog's release procedure, and the selected module versions in `go.mod`. |
|
||||
| Caller-supplied configuration sources | The [internal source document](internal/sources.md), [framework format reference](formats.md), and [testing policy](policy/testing.md). |
|
||||
| Model-client behavior | The [OpenAI-compatible integration contract](integrations/openai-compatible-chat.md), [internal model-client document](internal/llm.md), and owning package tests. |
|
||||
| Internal package implementation | The [architecture policy](policy/architecture.md), [internal component overview](internal/overview.md), and focused internal document listed for that package. |
|
||||
| Tests or test fixtures | The [testing policy](policy/testing.md), owning package, and focused internal document listed by the component overview. |
|
||||
| Maintained example | The [preparation](../examples/go-library/prepare/main.go), [execution](../examples/go-library/run/main.go), and [observation](../examples/go-library/observe/main.go) examples, [consumer guide](consumers/pkg-promptkit.md), [framework format reference](formats.md), and [documentation policy](policy/documentation.md). |
|
||||
| Documentation | The [documentation policy](policy/documentation.md) and canonical owner of every affected contract. |
|
||||
| Licensing, dependencies/catalog updates, copied material, or attribution | The [OSS policy](policy/oss.md), [project grant](../COPYRIGHT), [license texts](../LICENSE), and [attribution entries](oss/attribution.md); inspect the selected upstream source's notices. |
|
||||
| Licensing, dependencies, copied material, or attribution | The [OSS policy](policy/oss.md), [project grant](../COPYRIGHT), [license texts](../LICENSE), and [attribution entries](oss/attribution.md); inspect the selected upstream source's notices. |
|
||||
| Release preparation or publication, including release notes | The [release policy](policy/release.md), [release procedure](release.md), and [OSS discoverability rules](policy/oss.md#discoverability). |
|
||||
|
||||
For cross-cutting changes, follow every applicable row. Do not create
|
||||
@@ -52,37 +59,43 @@ exist.
|
||||
## Maintainer Validation
|
||||
|
||||
For ordinary work, select validation proportionate to the change. Do not repeat
|
||||
successful checks for unchanged content merely to create a commit. The full
|
||||
release-candidate checks have one executable entry point from the repository root:
|
||||
successful checks for unchanged content merely to create a commit. During
|
||||
development, run relevant Go tests and vet, format changed Go files, and execute
|
||||
the maintained examples when they are affected:
|
||||
|
||||
```sh
|
||||
./scripts/release-check.sh
|
||||
go test ./...
|
||||
go vet ./...
|
||||
gofmt -w path/to/changed.go
|
||||
python3 scripts/check-examples.py
|
||||
git diff --check
|
||||
```
|
||||
|
||||
The [script](../scripts/release-check.sh) owns the executable checks:
|
||||
race-enabled tests, vet, the three offline consumer examples with semantic output
|
||||
assertions, tracked Go formatting, local Markdown links, module/workspace
|
||||
hygiene, whitespace, and the credential-pattern scan. It does not publish,
|
||||
change files, validate release-note structure, or retire roadmaps. It requires Go,
|
||||
Git, a POSIX shell, and Python 3. Tests and examples require no provider
|
||||
credentials or live inference calls.
|
||||
Release candidates use `./scripts/release.sh VERSION`, which runs the selected
|
||||
profile checks and publishes the tag. The [release procedure](release.md)
|
||||
describes its requirements and recovery. For an optional diagnostic of a
|
||||
committed candidate without publication, run `./scripts/releasekit.sh check
|
||||
VERSION`; this is not a required pre-publication step.
|
||||
|
||||
The script can run on intended uncommitted changes during development.
|
||||
Formatting and documentation checks use tracked files; stage intended new
|
||||
files before final validation so those checks include them. The release
|
||||
procedure separately enforces a clean, committed candidate.
|
||||
The standalone `python3 scripts/check-examples.py` command checks semantic
|
||||
expectations for the three offline consumer examples. The focused
|
||||
`python3 scripts/test-release.py` command exercises the release tooling against
|
||||
disposable local repositories; use it when changing that tooling, not as a
|
||||
recurring release gate. Tests and examples require no provider credentials or
|
||||
live inference calls.
|
||||
|
||||
Ordinary validation can run on intended uncommitted work. Release validation
|
||||
uses committed candidate contents and the selected local Go installation.
|
||||
|
||||
### Change Review
|
||||
|
||||
Inspect `git status --short --untracked-files=all` and the complete diff before
|
||||
accepting a change. Reject unintended files, credentials, private data,
|
||||
generated binaries, test output, downloaded assets, and template residue.
|
||||
The automated credential scan does not replace review of the actual change.
|
||||
|
||||
For releases, follow the [release procedure](release.md). Its publication
|
||||
command runs the full local checks once on the final committed candidate and
|
||||
automates release-note structure and Git checks. Do not repeat ordinary change
|
||||
review as a release sign-off. Roadmap retirement remains preparation housekeeping.
|
||||
For releases, follow the [release procedure](release.md). Do not repeat its
|
||||
candidate checks as a separate sign-off. Roadmap retirement remains preparation
|
||||
housekeeping.
|
||||
|
||||
Follow the [agent commit policy](policy/agents.md#git-commit-policy) for local
|
||||
commits. Ordinary commits may leave unrelated work in the workspace; preserve
|
||||
|
||||
658
docs/formats.md
658
docs/formats.md
@@ -2,394 +2,390 @@
|
||||
|
||||
## Purpose
|
||||
|
||||
This document is the canonical contract for Promptkit prompt-definition,
|
||||
profile, and schema files. The [Go package consumer guide](consumers/pkg-promptkit.md)
|
||||
explains how to select these sources and invoke the engine. The
|
||||
This is the canonical reference for Promptkit's backend, profile, prompt, and
|
||||
schema definitions. The [Go consumer guide](consumers/pkg-promptkit.md) shows
|
||||
how to select sources and use an engine; the
|
||||
[OpenAI-compatible integration contract](integrations/openai-compatible-chat.md)
|
||||
owns the resulting outbound wire behavior.
|
||||
owns outbound wire behavior. [ConfigKit](https://gitea.maximumdirect.net/eric/ConfigKit)
|
||||
owns the generic YAML grammar, imports, layering, inheritance, and diagnostics.
|
||||
This reference describes what Promptkit does with the final effective values.
|
||||
|
||||
Prompt and profile sources recursively discover files ending in `.yaml` or
|
||||
`.yml`. Each prompt-definition and profile file contains exactly one YAML
|
||||
document; comments and trailing whitespace are allowed. YAML decoding is
|
||||
strict: unknown fields are errors for the selected definition. Definitions are
|
||||
selected by their YAML `id`, not their file name or directory.
|
||||
A configuration document may contain keyed top-level `backends`, `profiles`,
|
||||
and `prompts` mappings, plus a `collections` mapping of namespaced resource
|
||||
groups. Resource keys are literal IDs, not filename-derived identities. Each
|
||||
prompt ID maps directly to one definition; an optional `version` string is
|
||||
descriptive metadata, not a selection key. Keys and references must be nonblank
|
||||
and remain exact: Promptkit does not trim or re-key them. Old embedded `id` and
|
||||
`base_profile` fields are invalid. Unknown fields are errors. Promptkit installs
|
||||
no provider or profile catalog; callers select the definitions they need.
|
||||
|
||||
```yaml
|
||||
backends:
|
||||
local:
|
||||
endpoint: http://localhost:8000/v1
|
||||
concurrency_limit: 2
|
||||
profiles:
|
||||
local-summary:
|
||||
backend: local
|
||||
model: example-model
|
||||
prompts:
|
||||
meeting.summary:
|
||||
version: "1.0.0"
|
||||
default_profile: local-summary
|
||||
messages:
|
||||
- role: user
|
||||
content: Summarize this note.
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
collections:
|
||||
support:
|
||||
release_version: "1.4.0"
|
||||
backends:
|
||||
offline:
|
||||
endpoint: https://example.invalid/v1
|
||||
profiles:
|
||||
base:
|
||||
backend: support/offline
|
||||
model: offline-model
|
||||
fast:
|
||||
extends: base
|
||||
timeout_seconds: 30
|
||||
prompts:
|
||||
answer:
|
||||
default_profile: support/fast
|
||||
messages:
|
||||
- role: user
|
||||
content: Answer the selected caller input.
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
```
|
||||
|
||||
## Source Headers, Collections, And Resource IDs
|
||||
|
||||
An input may omit a source header. When present, the source-root `version` is
|
||||
reserved for ConfigKit's specification header and accepts only the exact string
|
||||
`"1"` or strict decimal integer `1`; null and other values are invalid. This
|
||||
header is distinct from a prompt's nested `version` label and a collection's
|
||||
`release_version`. ConfigKit checks and consumes accepted headers on non-main
|
||||
inputs; a declaration on the main input remains visible in explicit
|
||||
configuration inspection. Headerless inputs still undergo ordinary final-value
|
||||
validation. ConfigKit owns generic import and source-history behavior.
|
||||
|
||||
The root resource mappings contain unqualified IDs. Each `collections` entry
|
||||
has a namespace key and may contain `release_version` plus the same three
|
||||
resource mappings. Collection resource keys are local keys; their runtime IDs
|
||||
are `<namespace>/<local-key>`. Root and collection keys may be the same. A
|
||||
namespace and local key must be nonblank and slash-free, and references must
|
||||
name their intended exact ID; mounting a source does not rewrite references.
|
||||
For example, the `support/fast` profile above stores `support/offline` as its
|
||||
backend ID. In `collections.support.profiles.fast`, ConfigKit `extends: base`
|
||||
refers to a sibling profile in that same mapping. It is not a qualified or
|
||||
cross-collection reference.
|
||||
|
||||
`release_version` and prompt `version` are optional opaque labels. Absent or
|
||||
null means no label; an explicitly empty string is retained as a declared
|
||||
label. Neither label selects a resource or changes execution. Runtime lookups
|
||||
use the captured effective resource indexes. `InspectConfiguration` is an
|
||||
explicit operator view of effective settings and provenance; source history is
|
||||
not a second runtime resolver, and the retained effective shape is not flattened
|
||||
or rewritten into those indexes.
|
||||
|
||||
For examples of explicitly selecting native collection inputs and inspecting
|
||||
inventories, see the [consumer guide](consumers/pkg-promptkit.md) and its
|
||||
[maintained preparation example](../examples/go-library/prepare/main.go). For
|
||||
breaking identity and prompt-selection changes, see the
|
||||
[configuration migration guide](consumers/configuration-migration.md).
|
||||
|
||||
Every effective backend, profile, and prompt definition is validated when
|
||||
`NewEngine` constructs the engine, even if it is never selected. A profile may
|
||||
be partial: an endpoint and model need only be complete after request overrides
|
||||
are applied. A prompt's default profile similarly need not be runnable without
|
||||
an overriding request. Missing request data, artifacts, schemas, credentials,
|
||||
and provider work remain runtime concerns.
|
||||
|
||||
The engine captures effective definitions, referenced message-file contents,
|
||||
and static template syntax at construction. It does not reread configuration or
|
||||
message files on ordinary runs; construct a new engine to observe edits.
|
||||
Schema and request artifact reads still happen during preparation. Use
|
||||
`InspectConfiguration` for an explicit effective-configuration or provenance
|
||||
report, `InspectProfile` for backend-completed profile settings, and preparation
|
||||
for request-specific rendered details. For identity discovery before inspection,
|
||||
see the consumer guide's [resource and collection listing](consumers/pkg-promptkit.md#discover-available-resources-and-collections).
|
||||
|
||||
## Prompt Definitions
|
||||
|
||||
A prompt definition describes inputs, Go-template messages, an optional
|
||||
default profile, and an output contract.
|
||||
The `prompts` mapping selects a literal prompt ID whose value is the definition:
|
||||
|
||||
```yaml
|
||||
id: meeting.summary
|
||||
version: "1.0.0"
|
||||
default_profile: local-summary
|
||||
description: Summarize a synthetic meeting note.
|
||||
session_id: '{{.session}}'
|
||||
inputs:
|
||||
- name: note
|
||||
required: true
|
||||
content_type: text/plain
|
||||
description: Meeting note to summarize.
|
||||
messages:
|
||||
- role: system
|
||||
content: Return a concise summary.
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
ttl: 1h
|
||||
- role: user
|
||||
content_file: ./summary.user.md
|
||||
output:
|
||||
format: markdown
|
||||
validation_mode: basic
|
||||
repair_attempts: 0
|
||||
prompts:
|
||||
meeting.summary:
|
||||
version: "1.0.0"
|
||||
default_profile: local-summary
|
||||
description: Summarize a synthetic meeting note.
|
||||
session_id: '{{.session}}'
|
||||
inputs:
|
||||
- name: note
|
||||
required: true
|
||||
content_type: text/plain
|
||||
description: Meeting note to summarize.
|
||||
messages:
|
||||
- role: system
|
||||
content: Return a concise summary.
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
ttl: 1h
|
||||
- role: user
|
||||
content_file: ./summary.user.md
|
||||
output:
|
||||
format: markdown
|
||||
validation_mode: basic
|
||||
repair_attempts: 0
|
||||
```
|
||||
|
||||
| Field | Required | Meaning |
|
||||
| --- | --- | --- |
|
||||
| `id` | yes | Non-empty prompt identifier used by `RunRequest.PromptID`. |
|
||||
| `version` | yes | Non-empty version selected by an optional `RunRequest.PromptVersion`. |
|
||||
| `default_profile` | no | Non-empty profile ID used when the request omits `ProfileID`. |
|
||||
| `description` | no | Human-readable description. |
|
||||
| `session_id` | no | Go template rendered from request variables and input helpers. |
|
||||
| `inputs` | no | Declared input metadata. |
|
||||
| `messages` | yes | One or more chat-message templates. |
|
||||
| `output` | yes | Output format and validation settings. |
|
||||
|
||||
When a request omits a version, the selected prompt ID must identify exactly
|
||||
one definition. When it supplies a version, the ID and version pair must be
|
||||
unique.
|
||||
|
||||
Exact prompt inspection uses this same configured source, strict decoding,
|
||||
referenced content-file resolution, and ID/version selection. It reports the
|
||||
selected definition's declared metadata without changing the prompt format or
|
||||
executing the definition.
|
||||
`messages` must be nonempty and `output` must be a valid mapping. Optional
|
||||
`default_profile`, `description`, `session_id`, `inputs`, and cache-control
|
||||
fields may be absent or cleared with null where appropriate. A null whole
|
||||
definition, required role, output format, or validation mode is invalid.
|
||||
`required` and `repair_attempts` default to false and zero, respectively, and
|
||||
are not nullable. `InspectPrompt` selects the same captured definition without
|
||||
rendering or resolving a profile.
|
||||
|
||||
### Inputs
|
||||
|
||||
Each `inputs` item has these fields:
|
||||
|
||||
| Field | Required | Meaning |
|
||||
| --- | --- | --- |
|
||||
| `name` | yes | Non-empty name used by the request input map and `input` template helper. Names must be unique. |
|
||||
| `required` | no | When true, preparation fails if the request omits the input. The default is false. |
|
||||
| `content_type` | no | Expected media-type metadata. |
|
||||
| `description` | no | Human-readable input description. |
|
||||
|
||||
Requests supply inputs as inline or file-backed `ArtifactRef` values. Declared
|
||||
required inputs must be present. A template reference also requires the named
|
||||
input to exist, whether or not it was declared. Extra request inputs are
|
||||
allowed.
|
||||
Each `inputs` item may contain `name`, `required`, `content_type`, and
|
||||
`description`. `name` is required, nonblank, and unique within the prompt.
|
||||
Omitted `required` means false. Null `inputs` means no declarations. Requests
|
||||
supply inline or file-backed `ArtifactRef` values; declared required inputs
|
||||
must be present. A template reference also needs its named input, even when
|
||||
undeclared. Extra request inputs are allowed.
|
||||
|
||||
### Messages And Templates
|
||||
|
||||
Each message has a `role` that Promptkit trims and lowercases. It must then be
|
||||
exactly one of `developer`, `system`, `user`, or `assistant`; blank, custom,
|
||||
`tool`, and `function` roles are invalid. This intentionally tightens the
|
||||
previous nonblank-string rule. Consumers migrating to the next minor release
|
||||
must update any nonstandard prompt-definition roles before upgrading.
|
||||
Each message needs a role and exactly one nonnull `content` or `content_file`.
|
||||
An explicitly empty inline `content` is valid. Null clears either content
|
||||
selection. Roles are trimmed and lowercased, then must be `developer`,
|
||||
`system`, `user`, or `assistant`. Static template syntax is checked during
|
||||
construction; missing variables and named inputs fail when rendered.
|
||||
Request variables are template data (`{{.audience}}`), and
|
||||
`{{input "note"}}` renders an input body.
|
||||
|
||||
Each message also has exactly one of:
|
||||
`content_file` uses the effective field's own source context, including when
|
||||
inherited or imported. Its relative path is resolved against that source's
|
||||
base directory within its rooted filesystem. Absolute paths and lexical
|
||||
escapes from the selected root fail. Host-file sources also prevent symlink
|
||||
escapes. Promptkit preserves the path text, including meaningful whitespace,
|
||||
after checking that it is not blank. A source without a unique usable resource
|
||||
context must supply one explicitly or override the field. Content is captured
|
||||
at construction, not reopened for each request.
|
||||
|
||||
- `content`, containing an inline Go template; or
|
||||
- `content_file`, naming a file whose contents are the Go template.
|
||||
Optional `session_id` uses the same template variables and input helper. Its
|
||||
rendered value is trimmed, omitted if empty, and limited to 256 Unicode code
|
||||
points. A nonblank direct request session ID bypasses the template.
|
||||
|
||||
`content_file` must be a relative path. It resolves from the directory that
|
||||
contains the prompt file and must remain within the configured prompt source
|
||||
root; parent components are allowed only when the resolved target remains
|
||||
inside that root. Absolute paths and paths that escape the root are rejected.
|
||||
Operating-system directory and single-file sources also reject symlink targets
|
||||
outside the root, while injected `fs.FS` sources apply containment in that
|
||||
filesystem's relative path namespace. For `WithPromptFile`, the source root is
|
||||
the directory containing the selected prompt file. Promptkit uses the parsed
|
||||
path text exactly after checking separately that it is not blank, so leading
|
||||
and trailing whitespace can name real filesystem entries.
|
||||
|
||||
Request variables are the template data, so a variable named `audience` is
|
||||
referenced as `{{.audience}}`. The `{{input "note"}}` helper renders the body
|
||||
of a named input. Missing variables and input references are errors.
|
||||
|
||||
The optional `session_id` uses the same template data and input helper. Its
|
||||
rendered value is trimmed, omitted when empty, and limited to 256 Unicode code
|
||||
points. A nonblank direct request session ID bypasses this template completely;
|
||||
a blank direct value leaves the template behavior unchanged.
|
||||
`InspectRequirements` can enumerate recognizable dependencies from captured
|
||||
templates before artifact content is available. Its conservative syntax
|
||||
coverage does not change template validity or rendering; see the [consumer
|
||||
guide](consumers/pkg-promptkit.md#inspect-requirements-before-preparation) and
|
||||
the method's [GoDoc](../requirements.go) for that inspection boundary.
|
||||
|
||||
### Cache Control
|
||||
|
||||
`cache_control` is optional and has these fields:
|
||||
|
||||
| Field | Required | Values |
|
||||
| --- | --- | --- |
|
||||
| `type` | yes | `ephemeral` |
|
||||
| `ttl` | no | Empty or `1h` |
|
||||
|
||||
Promptkit preserves cache-control metadata on the rendered message. The
|
||||
outbound integration determines its wire representation.
|
||||
Optional `cache_control` has required `type: ephemeral` and optional `ttl`,
|
||||
which may be empty or `1h`. Promptkit retains this metadata on rendered
|
||||
messages; the integration contract defines its wire representation.
|
||||
|
||||
### Output Contract
|
||||
|
||||
| Field | Required | Values or behavior |
|
||||
| --- | --- | --- |
|
||||
| `format` | yes | `text`, `markdown`, or `json`. |
|
||||
| `validation_mode` | yes | `none`, `basic`, `json`, or `json_schema`. |
|
||||
| `schema_path` | for `json_schema` | Path to a schema in the configured schema source. |
|
||||
| `repair_attempts` | no | Integer from zero through three; omitted means zero. A positive value requires `basic`, `json`, or `json_schema` validation. |
|
||||
| `format` | yes | `text`, `markdown`, or `json` |
|
||||
| `validation_mode` | yes | `none`, `basic`, `json`, or `json_schema` |
|
||||
| `schema_path` | for `json_schema` | Path in the selected schema source |
|
||||
| `repair_attempts` | no | Integer 0–3; default 0. Positive values need `basic`, `json`, or `json_schema` |
|
||||
|
||||
The validation modes behave as follows:
|
||||
`none` skips content validation; `basic` requires nonempty content; `json`
|
||||
accepts any valid JSON value; `json_schema` checks the selected schema.
|
||||
`format` determines output artifact metadata. JSON Schema mode also supplies
|
||||
structured-output metadata to compatible clients. Plain `json` does not
|
||||
request a provider-native JSON-object constraint. A positive repair budget
|
||||
counts additional generations after failed validation; exhaustion returns the
|
||||
last candidate and its failed validation result. Operational errors remain
|
||||
errors. A request-level `OutputContract` replaces the whole configured one,
|
||||
not individual fields; an empty request format defaults to `text`.
|
||||
|
||||
- `none` skips content validation;
|
||||
- `basic` requires non-empty generated content;
|
||||
- `json` requires valid JSON; and
|
||||
- `json_schema` requires valid JSON that satisfies the selected schema.
|
||||
## Backend Definitions
|
||||
|
||||
`format` controls output artifact metadata. JSON Schema mode also supplies the
|
||||
schema to compatible model clients as structured-output metadata. Plain `json`
|
||||
validation accepts every valid JSON value and does not request a provider-native
|
||||
JSON-object constraint.
|
||||
Backend definitions are supplied through caller-selected configuration
|
||||
sources. The conventional backend IDs are names only; PromptKit does not
|
||||
install definitions for them.
|
||||
|
||||
`repair_attempts` counts additional generation calls after a failed validation.
|
||||
Zero is single-pass. With a positive eligible budget, Promptkit stops at the
|
||||
first valid candidate. If the budget is exhausted, it returns the final
|
||||
candidate and its complete failed validation result; generation and operational
|
||||
validation failures remain errors. `none` never permits repair.
|
||||
A backend definition may have `endpoint`, `api_key_env`,
|
||||
`concurrency_limit`, `queue_capacity`, `extra_params`, and `smoke_tests`. The
|
||||
final endpoint is required and must be an absolute HTTP(S) URL with a host, no
|
||||
user information, query, or fragment. An omitted or null API-key environment
|
||||
name means no lookup. Omitted/null concurrency is zero (unlimited); a supplied
|
||||
negative value is invalid. For positive concurrency, omitted/null queue
|
||||
capacity defaults to 1024. An explicit queue capacity, including zero,
|
||||
requires positive concurrency and must be nonnegative. Unlimited concurrency
|
||||
has no queue. Provider maps contain JSON-compatible values; null entries are
|
||||
ordinary JSON null. Unknown fields, including a raw `api_key`, fail.
|
||||
|
||||
A request-level `OutputContract` replaces the complete prompt output contract.
|
||||
It does not merge individual fields. If its format is empty, Promptkit uses
|
||||
`text`.
|
||||
`smoke_tests` is optional backend metadata for explicit diagnostic operations.
|
||||
It may contain `default_profile`, `reachability`, and `authentication`:
|
||||
|
||||
## Built-In Backends
|
||||
```yaml
|
||||
backends:
|
||||
openrouter:
|
||||
smoke_tests:
|
||||
default_profile: application-profile
|
||||
reachability:
|
||||
endpoint: https://status.example/ready?check=1
|
||||
authentication:
|
||||
kind: openrouter_key
|
||||
endpoint: https://openrouter.ai/api/v1/key
|
||||
```
|
||||
|
||||
Every engine provides these reserved OpenAI-compatible backend IDs. Consumers
|
||||
must not register either ID with `WithBackend`; exact registration and
|
||||
reservation behavior belongs to the [`Backend` GoDoc](../backends.go).
|
||||
An omitted or null `smoke_tests` block disables these metadata defaults.
|
||||
`default_profile` is preserved as an exact, nonblank profile ID; its existence
|
||||
is checked only if an inference diagnostic selects that default. Omitted/null
|
||||
`default_profile` means there is no inference default. Omitted/null
|
||||
`reachability` or its endpoint uses the effective backend endpoint. An explicit
|
||||
reachability endpoint is trimmed and must be an absolute HTTP(S) URL with a
|
||||
host; unlike the backend base endpoint, its path and query are allowed. Null
|
||||
means fallback, while an explicit blank value is invalid.
|
||||
|
||||
| ID | Base endpoint | API-key environment variable | Active generation limit | Default queue capacity |
|
||||
| --- | --- | --- | ---: | ---: |
|
||||
| `openrouter` | `https://openrouter.ai/api/v1` | `OPENROUTER_API_KEY` | 16 | 1024 |
|
||||
| `rakestrawhome` | `https://inference.ai.rakestrawhome.com/v1` | `RAKESTRAWHOME_INFERENCE_API_KEY` | 4 | 1024 |
|
||||
Omitted/null `authentication` means no authentication adapter is declared. A
|
||||
non-null object requires `kind`, which is trimmed and case-normalized; the
|
||||
recognized values are `none` and `openrouter_key`. Its omitted/null endpoint
|
||||
uses the adapter's backend-relative destination (`openrouter_key` appends
|
||||
`/key` to the effective backend base); an explicit endpoint follows the same
|
||||
full-URL rules as reachability. `none` declares authentication inapplicable.
|
||||
These values are configuration only: engine construction does not contact a
|
||||
backend.
|
||||
|
||||
These fields are resolved through the ordinary ConfigKit source layers before
|
||||
Promptkit validates the final effective values. A higher-priority mapping can
|
||||
override individual nested fields; an explicit null can clear the whole
|
||||
`smoke_tests` block or one of its nested objects. An endpoint set to null uses
|
||||
its documented fallback, while an explicit blank endpoint is invalid. The
|
||||
metadata does not change ordinary generation credential requirements or
|
||||
provider settings.
|
||||
|
||||
## Profile Definitions
|
||||
|
||||
A profile supplies model execution settings:
|
||||
Profiles are keyed by literal ID and may inherit a named parent with
|
||||
`extends`:
|
||||
|
||||
```yaml
|
||||
id: local-summary
|
||||
backend: openrouter
|
||||
model: example-model
|
||||
temperature: 0.2
|
||||
max_tokens: 500
|
||||
top_p: 0.95
|
||||
timeout_seconds: 90
|
||||
service_tier: flex
|
||||
reasoning_effort: medium
|
||||
extra_params:
|
||||
provider_option: enabled
|
||||
profiles:
|
||||
local-summary:
|
||||
backend: local
|
||||
model: example-model
|
||||
temperature: 0.2
|
||||
max_tokens: 500
|
||||
timeout_seconds: 90
|
||||
extra_params:
|
||||
provider_option: enabled
|
||||
local-summary-fast:
|
||||
extends: local-summary
|
||||
timeout_seconds: 30
|
||||
reasoning_effort: low
|
||||
```
|
||||
|
||||
A derived profile can use a named base and override only the settings it owns:
|
||||
|
||||
```yaml
|
||||
id: local-summary-fast
|
||||
base_profile: local-summary
|
||||
timeout_seconds: 30
|
||||
reasoning_effort: low
|
||||
```
|
||||
|
||||
| Field | Required | Meaning |
|
||||
| --- | --- | --- |
|
||||
| `id` | yes | Profile identifier, trimmed before selection and publication. It must be non-empty after trimming and unique within one source after normalization. |
|
||||
| `base_profile` | no | One optional parent profile ID. A derived profile may inherit target fields from it. |
|
||||
| `backend` | unless `endpoint` is present | Backend registry ID. It is trimmed and registry membership is checked when the profile is prepared or inspected. |
|
||||
| `endpoint` | unless `backend` is present | OpenAI-compatible base URL, including an API version path when required. A nonempty value is trimmed and must be absolute HTTP or HTTPS with a host and without user information, a query, or a fragment. When both connection fields are present, this overrides the backend endpoint without changing backend identity. |
|
||||
| `model` | yes | Non-empty provider model name. |
|
||||
| `temperature` | no | Number from 0 through 2. |
|
||||
| `max_tokens` | no | Integer zero or greater. |
|
||||
| `top_p` | no | Number from 0 through 1. |
|
||||
| `timeout_seconds` | no | Per-generation deadline in whole seconds; integer zero or greater. |
|
||||
| `service_tier` | no | Provider-specific request tier. |
|
||||
| `reasoning_effort` | no | Provider-specific reasoning setting. |
|
||||
| `api_key_env` | no | Optional environment-variable lookup source for an API key. |
|
||||
| `extra_params` | no | JSON-compatible provider-specific outbound fields. |
|
||||
|
||||
Raw `api_key` is prohibited in profile YAML. Store only an environment
|
||||
variable name in `api_key_env`.
|
||||
|
||||
A standalone profile must provide a model and at least one of `backend` or
|
||||
`endpoint`. A derived profile may omit those target fields because its selected
|
||||
base chain can provide them. Local parsing still validates a derived profile's
|
||||
own ID, supplied endpoint, execution-setting bounds, and `extra_params`.
|
||||
|
||||
Promptkit does not infer a backend from a model or endpoint. Endpoint-only
|
||||
profiles remain supported and have no effective backend ID.
|
||||
The engine always provides the built-in `openrouter` and `rakestrawhome` IDs.
|
||||
Consumers can add engine-scoped IDs with
|
||||
[`WithBackend`](../backends.go); exact registration validation belongs to its
|
||||
GoDoc.
|
||||
|
||||
`extra_params` accepts null, booleans, finite numbers, strings, arrays, and
|
||||
objects with string keys. Keys must be non-empty. With the built-in client,
|
||||
they also cannot collide with the standard fields listed in the
|
||||
[outbound request contract](integrations/openai-compatible-chat.md#request-body).
|
||||
Excessively deep or large JSON-shaped values are rejected for safety.
|
||||
Profiles may also supply `endpoint`, `top_p`, `service_tier`, `api_key_env`,
|
||||
and `api_key_required`. Promptkit does not infer a backend from an endpoint or
|
||||
model. A profile with only an endpoint has no backend capacity identity.
|
||||
Nonblank endpoints follow the backend URL rules; temperature is 0–2, top_p
|
||||
is 0–1, max_tokens and timeout_seconds are nonnegative. Provider maps accept
|
||||
JSON-compatible values with nonempty keys and cannot collide with the built-in
|
||||
client's [standard request fields](integrations/openai-compatible-chat.md#request-body).
|
||||
Credential values do not belong in profiles: raw `api_key` is invalid.
|
||||
|
||||
### Defaults And Overrides
|
||||
|
||||
Execution settings resolve in this order:
|
||||
After effective profile inheritance, Promptkit applies the selected backend's
|
||||
defaults once, then request `ExecutionTargetOverride` values. Backend-derived
|
||||
settings are not inheritance input: changing a child profile's backend does
|
||||
not carry the parent's derived endpoint or credential source into it. Changing
|
||||
only a model or backend does not clear unrelated explicitly inherited fields.
|
||||
|
||||
1. the framework timeout baseline;
|
||||
2. the selected backend, when the profile names one;
|
||||
3. the selected profile; and
|
||||
4. request `ExecutionTargetOverride` values.
|
||||
| Profile field | Absent | Explicit value | Explicit null |
|
||||
| --- | --- | --- | --- |
|
||||
| `backend` | No backend | Exact backend ID; blank means none | No backend |
|
||||
| `endpoint` | Selected backend endpoint | Valid nonblank URL; empty clears | Clear, without backend fallback |
|
||||
| `model` | Unspecified | Nonblank model; empty clears | Unspecified |
|
||||
| `temperature`, `top_p`, `max_tokens` | Unspecified | Valid value, including outbound zero | Suppress outbound control |
|
||||
| `service_tier`, `reasoning_effort` | Unspecified | Normalized value; blank suppresses | Suppress outbound control |
|
||||
| `timeout_seconds` | 600 seconds | Nonnegative; zero disables generation deadline | Reset to 600 seconds |
|
||||
| `api_key_env` | Backend lookup name, if any | Trimmed name; blank disables | Disable backend lookup |
|
||||
| `api_key_required` | False | Boolean, including false | False |
|
||||
| `extra_params` | Backend defaults | Recursively overlay profile map on backend map | Suppress configured provider map |
|
||||
|
||||
The framework baseline is:
|
||||
|
||||
| Setting | Default |
|
||||
| --- | --- |
|
||||
| `temperature` | Unspecified and omitted from compatible provider requests unless a profile or runtime override selects it. |
|
||||
| `max_tokens` | Unspecified and omitted from compatible provider requests unless a profile or runtime override selects it. |
|
||||
| `top_p` | Unspecified and omitted from compatible provider requests unless a profile or runtime override selects it. |
|
||||
| `timeout_seconds` | `600` |
|
||||
|
||||
Numeric zero in a file or in-memory profile does not select a numeric value.
|
||||
For `temperature`, `max_tokens`, and `top_p`, it leaves the provider control
|
||||
unspecified. For `timeout_seconds`, it retains the framework deadline. Numeric
|
||||
request overrides use pointers, so an explicit zero is retained and sent to
|
||||
compatible providers. In particular, an explicit request `timeout_seconds` of
|
||||
zero disables the per-generation deadline while leaving the caller context and
|
||||
transport timeout intact.
|
||||
|
||||
Non-empty profile strings replace backend defaults, and non-empty request
|
||||
strings replace both. Request reasoning is the exception: a nil
|
||||
`ReasoningEffort` pointer inherits the profile, a pointer to a nonblank string
|
||||
trims and replaces it, and a pointer to a blank string clears it. Backend
|
||||
identity is retained when either layer overrides the endpoint, so the override
|
||||
also retains any engine-local capacity policy configured for that backend.
|
||||
Capacity configuration belongs to the Go
|
||||
[`Backend` API](../backends.go), not prompt or profile YAML. A non-empty
|
||||
`extra_params` map at each layer replaces the entire lower-precedence map
|
||||
rather than merging keys.
|
||||
The [outbound integration contract](integrations/openai-compatible-chat.md)
|
||||
defines how the effective settings are serialized.
|
||||
An empty mapping does not clear inherited map entries; null clears the whole
|
||||
map. Nested maps merge recursively, while scalar and array values replace.
|
||||
Runtime nonempty `ExtraParams` overrides replace the entire configured map.
|
||||
Request overrides otherwise follow the [public value GoDoc](../types.go).
|
||||
For example, a runtime `extra_params["extends"]` key is ordinary provider
|
||||
data, not a configuration inheritance instruction. An explicit zero timeout
|
||||
does not disable the caller context or HTTP transport timeout.
|
||||
|
||||
### Source And Profile Precedence
|
||||
|
||||
An explicit request profile ID takes precedence over the prompt's
|
||||
`default_profile`. If neither is present, preparation fails. Exact profile
|
||||
inspection instead takes one explicit profile ID and does not use a prompt
|
||||
default.
|
||||
`NewEngine` composes caller underlays in their given order, followed by an
|
||||
optional main input. Later layers overlay earlier ones. Inputs within one
|
||||
underlay group are strictly composed: conflicting contributions are errors,
|
||||
not priority signals. The directory helpers discover `.yaml` and `.yml` files recursively
|
||||
in sorted order, but that order does not give one file higher precedence.
|
||||
An explicitly selected missing/unreadable root fails; an empty root contributes
|
||||
nothing. Imports are disabled by these helpers unless the caller enables them
|
||||
on an input. Single files can use ConfigKit inputs directly.
|
||||
|
||||
Profile sources resolve matching IDs in this order:
|
||||
|
||||
1. in-memory profiles supplied with `WithProfiles`;
|
||||
2. the ordinary configured source selected by a profile file, `fs.FS`, or
|
||||
configured profile directory;
|
||||
3. application fallback profiles supplied with `WithFallbackProfileFS`; and
|
||||
4. Promptkit-owned profiles; and
|
||||
5. maintained external catalog profiles.
|
||||
|
||||
A profile source supplies a complete definition; definitions and their fields
|
||||
are not merged across sources. A higher-precedence source falls back only when
|
||||
the requested profile ID is absent. An invalid matching profile is an error and
|
||||
does not fall back. In-memory `Profile` values follow the same ranges as YAML
|
||||
profiles. They use `APIKeyRequired` for request-scoped credentials instead of
|
||||
`api_key_env`. Preparation and exact profile inspection use this same source
|
||||
precedence.
|
||||
|
||||
When a selected definition names `base_profile`, every profile ID in that
|
||||
chain is looked up through this same precedence order. A higher-precedence
|
||||
definition therefore shadows a lower-precedence definition of the same base
|
||||
ID, including a built-in. References are not source-qualified.
|
||||
Root backends, profiles, and prompts, as well as collection declarations and
|
||||
their resource mappings, follow these same source rules. `CollectionInput`
|
||||
mounts one explicitly selected source at a collection destination and enables
|
||||
ConfigKit imports for that input; it does not discover or select sources.
|
||||
A later profile or backend can change individual fields of a lower definition.
|
||||
A request `ProfileID` takes precedence over a prompt `default_profile`; if
|
||||
neither is selected, preparation fails. Exact profile inspection selects one
|
||||
explicit ID and does not consult a prompt default.
|
||||
|
||||
### Profile Inheritance
|
||||
|
||||
Promptkit resolves one linear base chain of at most 32 profiles, including the
|
||||
selected profile. It merges settings from the root base to the selected leaf.
|
||||
The leaf's `id` remains the selected profile identity. Nonblank string fields
|
||||
(`backend`, `endpoint`, `model`, `service_tier`, `reasoning_effort`, and
|
||||
`api_key_env`) and nonzero numeric fields replace inherited values. A nonempty
|
||||
`extra_params` map replaces the complete inherited map rather than merging
|
||||
keys, and `APIKeyRequired: true` remains true through the chain. Backend and
|
||||
endpoint are independent: replacing one does not clear the other.
|
||||
|
||||
There is no profile-level clearing syntax. Blank strings, zero numbers, false,
|
||||
and empty maps remain unspecified and inherit from a base. Use existing
|
||||
presence-aware request overrides where an execution needs an explicit zero or
|
||||
empty reasoning setting.
|
||||
|
||||
An absent directly selected profile reports the ordinary not-found error. Once
|
||||
the selected profile exists, a missing base, cycle, overlong chain, or
|
||||
incomplete resolved target is a profile-load failure. Ordinary operations
|
||||
resolve chains afresh; prepared execution retains the fully resolved target.
|
||||
|
||||
## Built-In Profile Catalog
|
||||
|
||||
Every built-in profile selects one maintained built-in backend and inherits
|
||||
that backend's connection and credential metadata. Profile files do not repeat
|
||||
those values. A configured, application fallback, or in-memory profile with
|
||||
the same profile ID takes precedence.
|
||||
|
||||
Promptkit owns two opt-in profiles. The following definitions are current
|
||||
selections, not automatic defaults:
|
||||
|
||||
| ID | Current definition |
|
||||
| --- | --- |
|
||||
| `promptkit-openrouter-standard` | Inherits `gemini-flash-latest`. |
|
||||
| `promptkit-openrouter-auto` | Selects backend `openrouter` and model `openrouter/auto`. |
|
||||
|
||||
Select either name through an existing request `profile_id` or prompt
|
||||
`default_profile`. Consumers can replace either definition or its inherited
|
||||
base through the ordinary profile-source precedence above. The
|
||||
[release policy](policy/release.md#catalog-profile-compatibility) defines the
|
||||
cross-version guarantee for these names.
|
||||
|
||||
## External Profile Inventories
|
||||
|
||||
The external profile inventory is defined by the selected catalog modules, not
|
||||
duplicated here. Browse the exact data selected by this Promptkit version:
|
||||
|
||||
- [OpenRouter catalog v2.20260926.0 profiles](https://gitea.maximumdirect.net/eric/promptkit-backend-openrouter/src/tag/v2.20260926.0/catalog/profiles)
|
||||
- [Rakestrawhome catalog v2.20260926.0 profiles](https://gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/src/tag/v2.20260926.0/catalog/profiles)
|
||||
|
||||
External profile entries are volatile across catalog releases. The
|
||||
[release policy](policy/release.md#catalog-profile-compatibility) owns the precise
|
||||
compatibility boundary; a selected module version identifies tested local data,
|
||||
not live provider availability.
|
||||
ConfigKit resolves `extends` on effective profile definitions. One profile
|
||||
may inherit a sibling profile by local key in the same root or collection
|
||||
mapping; collection inheritance cannot cross namespaces. Presence, explicit
|
||||
null, and recursively merged maps determine the final fields. Other resource
|
||||
references are exact IDs and use their qualified form when they select a
|
||||
collection resource. There is no Promptkit-specific depth limit or second
|
||||
base-chain resolver. Missing references and cycles fail at construction. For
|
||||
generic inheritance and layer mechanics, consult
|
||||
[ConfigKit](https://gitea.maximumdirect.net/eric/ConfigKit).
|
||||
|
||||
## Schemas
|
||||
|
||||
Schemas are JSON documents selected by a prompt or request
|
||||
`schema_path`. For a directory or `fs.FS` source, paths resolve within the
|
||||
configured source root. Referenced nested schemas resolve relative to the
|
||||
owning schema document. `WithSchemaFile` exposes one schema, addressed by its
|
||||
base name.
|
||||
|
||||
An unreadable, invalid, or unresolvable schema produces an operational
|
||||
validation error. Generated content that is valid JSON but does not satisfy the
|
||||
schema produces a failed validation result.
|
||||
Schemas are JSON documents selected by a prompt or request `schema_path`.
|
||||
Directory and `fs.FS` schema paths resolve within the configured root;
|
||||
referenced nested schemas resolve relative to the owning schema document.
|
||||
`WithSchemaFile` exposes one schema under its base name. An unreadable,
|
||||
invalid, or unresolvable schema causes an operational validation error.
|
||||
Valid JSON that fails the schema instead yields a failed validation result.
|
||||
|
||||
## Credentials
|
||||
|
||||
Credential values belong at the request or environment boundary, never in
|
||||
prompt, profile, schema, or example files:
|
||||
Only environment-variable names belong in backend/profile definitions, never
|
||||
credential values. A request may supply a direct `APIKey` or an `APIKeyEnv`
|
||||
override. A direct key takes precedence, followed by a request environment
|
||||
source, an explicit profile source, and an inherited backend source. A true
|
||||
profile `api_key_required` suppresses the backend-derived environment source,
|
||||
but an explicit profile or request environment source remains usable. False
|
||||
restores ordinary optional-credential behavior. Missing or blank optional
|
||||
environment values leave the built-in client's Authorization header absent;
|
||||
required availability is checked at preparation and again during prepared
|
||||
execution. Injected clients own their credential-resolution behavior.
|
||||
|
||||
- a backend or file profile can name an optional environment lookup source
|
||||
with `APIKeyEnv` or `api_key_env`;
|
||||
- an in-memory profile may set `APIKeyRequired` as an explicit local
|
||||
requirement;
|
||||
- a request can provide a direct `APIKey` or override the optional `APIKeyEnv`
|
||||
source; and
|
||||
- a direct request key takes precedence over environment lookup.
|
||||
|
||||
After a direct request key, the credential-source precedence is request
|
||||
`APIKeyEnv`, profile `api_key_env`, then the backend default. An in-memory
|
||||
profile with `APIKeyRequired` clears an inherited backend environment name and
|
||||
requires a direct key unless the request explicitly supplies `APIKeyEnv`.
|
||||
Named environment sources are optional: when the selected source is absent,
|
||||
empty, or whitespace-only, the built-in client omits the `Authorization`
|
||||
header and handles the provider response normally. `APIKeyRequired` is the
|
||||
only explicit local availability requirement. Promptkit validates required
|
||||
credential availability during preparation and rechecks it when a prepared
|
||||
execution runs. Injected clients receive resolved source metadata but define
|
||||
their own credential-resolution behavior.
|
||||
Direct keys are excluded from JSON results and redacted by public string
|
||||
formatters. Environment-variable names may appear in prepared metadata, but
|
||||
their values do not.
|
||||
Credential values are excluded from JSON results and redacted by public
|
||||
string formatters. ConfigKit inspection redacts credential-shaped dynamic
|
||||
provider keys, but does not make arbitrary prompt/provider content safe to
|
||||
disclose. Applications own report disclosure and secret handling.
|
||||
|
||||
@@ -45,6 +45,45 @@ precedence over the backend default. Only the name reaches prepared metadata;
|
||||
the environment value is read just before the provider call and is never added
|
||||
to the JSON body.
|
||||
|
||||
## Explicit Smoke Diagnostics
|
||||
|
||||
Promptkit's explicit smoke diagnostics are separate consumer operations; they
|
||||
are not part of ordinary `Run`, the offline examples, or release validation.
|
||||
With the built-in client, inference diagnostics use the same configured client
|
||||
and `/chat/completions` request as generation. Their content cases are private
|
||||
to Promptkit and are not application prompt, schema, or provider protocol
|
||||
extensions. The public
|
||||
[`Engine.SmokeTest` GoDoc](../../smoke_diagnostics.go) owns request selection,
|
||||
report facts, and injected-client behavior.
|
||||
|
||||
HTTP-only diagnostics use the configured built-in HTTP client. Reachability is
|
||||
one unauthenticated `GET` to the selected diagnostic URL; an observed response
|
||||
counts as reachable even when its status is non-2xx. Authentication behavior
|
||||
is selected from backend metadata, not inferred from the host. The
|
||||
`openrouter_key` adapter makes one `GET` to `/key` relative to the effective
|
||||
backend base, unless configuration supplies a full diagnostic endpoint (see
|
||||
the [backend format reference](../formats.md#built-in-backends)). It may send
|
||||
the available bearer credential, but can dispatch without one. A 401 is
|
||||
recognized as rejection. A 200 is recognized as acceptance only when its JSON
|
||||
object has a `data` object containing string `label`, Boolean `is_free_tier`,
|
||||
and numeric `usage`; unrelated fields and account limits are ignored. This is
|
||||
only a minimal identifying subset of OpenRouter's documented
|
||||
[current-key response](https://github.com/OpenRouterTeam/docs/blob/main/api_reference/limits.mdx#checking-your-limits),
|
||||
not validation of the full account document. Other statuses and unrecognized
|
||||
200 responses do not establish acceptance.
|
||||
|
||||
Diagnostic HTTP evidence describes the last request observed at Promptkit's
|
||||
RoundTripper boundary and any response associated with it. It includes the
|
||||
selected destination without URL user information, response status, and
|
||||
credential presence when observed. Redirects can change the last observed
|
||||
destination or strip credentials. This evidence does not prove that bytes
|
||||
reached a peer or reveal dispatch hidden inside a custom transport. Diagnostic
|
||||
reports do not retain headers, credentials, provider
|
||||
bodies, or generated text. A consumer-injected `LLMClient` can support inference
|
||||
diagnostics, but does not establish HTTP status, destination, reachability, or
|
||||
upstream credential-supply facts; Promptkit does not fall back to a built-in
|
||||
transport for HTTP-only diagnostics.
|
||||
|
||||
## Request Body
|
||||
|
||||
The request body always contains `model` and `messages`. The execution
|
||||
@@ -78,9 +117,8 @@ The client conditionally includes:
|
||||
|
||||
The engine resolves backend, profile, and request extra-parameter maps by
|
||||
whole-map replacement rather than key merging. The resulting effective map is
|
||||
then merged directly into the top-level body after JSON serialization is
|
||||
verified. Empty keys and collisions with these reserved fields are rejected
|
||||
before any provider call:
|
||||
merged into the top-level body. Empty keys, values that cannot be encoded, and
|
||||
collisions with these reserved fields are rejected before any provider call:
|
||||
|
||||
- `model`
|
||||
- `session_id`
|
||||
|
||||
@@ -88,6 +88,17 @@ release, including canceled acquisition without dispatch. A retry wait holds
|
||||
the logical admission lease but no active permit. Observer callbacks must not
|
||||
block on work that requires the same run's admission slot.
|
||||
|
||||
## Smoke-Test Capacity
|
||||
|
||||
The engine's smoke tester shares this manager and generation coordinator with
|
||||
the ordinary runner. Inference diagnostics reserve the same logical admission
|
||||
slot and acquire the same backend-scoped active permits for actual model
|
||||
attempts, including retries. They therefore participate in the same configured
|
||||
backend limits as `Run`. Reachability and authentication HTTP probes do not
|
||||
enter generation admission or consume active permits. The public
|
||||
[`Engine.SmokeTest` GoDoc](../../smoke_diagnostics.go) owns the caller-visible
|
||||
operation and injected-client limits.
|
||||
|
||||
## Cancellation And Release
|
||||
|
||||
Admission checks the caller context before reserving a slot. A call canceled
|
||||
@@ -127,4 +138,7 @@ cover shared and independent backend pools during retry and repair. The
|
||||
own the public prepared-capacity boundary. The
|
||||
[root error-boundary tests](../../errors_internal_test.go) own preservation of
|
||||
the public generation category and context identity when generation is
|
||||
canceled.
|
||||
canceled. The assembled
|
||||
[smoke diagnostic contract tests](../../smoke_integration_contract_test.go)
|
||||
verify that inference diagnostics share the engine's Run admission and recover
|
||||
capacity after cancellation or failure.
|
||||
|
||||
@@ -41,9 +41,22 @@ original error chain. This keeps injected attempt classifications consistent
|
||||
with terminal observations without parsing diagnostic text or retrying injected
|
||||
clients.
|
||||
|
||||
The diagnostic path uses the same private attempt and retry implementation as
|
||||
ordinary generation. It retains invocation-local transport, response,
|
||||
generation, usage, capacity-wait, and retry-delay evidence before normal
|
||||
result-discard and cancellation checks. Built-in preparation also supplies its
|
||||
frozen request URL to the evidence path before attempts begin; injected
|
||||
generation has no equivalent HTTP destination. This evidence is returned to
|
||||
diagnostic orchestration; it is not added to public `ExecutionEvent` values or ordinary
|
||||
observer accounting. The [model-client document](llm.md) describes the
|
||||
transport-observation boundary, and the root
|
||||
[`SmokeTestReport` GoDoc](../../smoke_diagnostics.go) owns report semantics.
|
||||
|
||||
Focused coordinator tests use controlled transport and wait functions. They
|
||||
cover eligibility, exhaustion, cancellation, backoff, request freezing, permit
|
||||
release, and the injected-client path. The capacity manager tests own permit
|
||||
release, and the injected-client path. The
|
||||
[evidence tests](../../internal/generation/evidence_test.go) own diagnostic
|
||||
attempt retention and retry evidence. The capacity manager tests own permit
|
||||
acquisition and idempotent release. Public
|
||||
[integration tests](../../reliable_execution_contract_test.go) cover retry,
|
||||
repair, observation, frozen prepared execution, and shared-backend behavior
|
||||
|
||||
@@ -34,9 +34,13 @@ resolved request target may supply the endpoint. Generation preparation then:
|
||||
1. validates shared execution-setting invariants and the final selected base
|
||||
endpoint;
|
||||
2. maps the internal request into the OpenAI-compatible chat payload;
|
||||
3. validates and merges extra parameters;
|
||||
3. checks extra-parameter keys against empty and reserved names, then merges
|
||||
their values into the payload;
|
||||
4. composes `/chat/completions` through parsed URL path operations;
|
||||
5. resolves authentication and freezes the URL, body, and optional credential.
|
||||
5. encodes the complete body once on success. If encoding fails, extra values
|
||||
are checked individually to retain invalid-request classification; otherwise
|
||||
the complete-request failure remains a request failure;
|
||||
6. resolves authentication and freezes the URL, body, and optional credential.
|
||||
|
||||
Each execution of that prepared request creates a fresh HTTP request, body
|
||||
reader, and attempt timeout context. It closes the response body and returns
|
||||
@@ -55,6 +59,33 @@ This client does not schedule retries; the [generation coordinator](generation.m
|
||||
does so around its prepared attempts. The client has no tool-call support,
|
||||
provider catalog, inbound HTTP behavior, or durable session store.
|
||||
|
||||
## Diagnostic HTTP Evidence
|
||||
|
||||
Built-in diagnostics wrap the selected HTTP client's `RoundTripper` for one
|
||||
invocation using a shallow client copy, preserving its timeout, redirect policy,
|
||||
cookie jar, and underlying transport without mutating the configured client.
|
||||
Evidence distinguishes a diagnostic dispatch invocation from an observed
|
||||
request and response. A newly observed request replaces the previous
|
||||
destination-specific response evidence, so redirects retain only the last
|
||||
observed hop. A response is recorded before envelope decoding or later context
|
||||
checks can discard a generation result.
|
||||
|
||||
The collector keeps only the last observed destination (without URL user
|
||||
information), status, credential-presence, response-availability, decoded
|
||||
generation, and presence-aware usage facts needed by the diagnostic. It does
|
||||
not retain headers, credentials, raw request or response bodies, or a redirect
|
||||
trace. These are observations at Promptkit's RoundTripper boundary, not proof
|
||||
that bytes reached a peer or that a custom transport did no hidden dispatch.
|
||||
Reachability and named authentication probes use the same configured built-in
|
||||
client but discard their response bodies after local recognition. The
|
||||
[integration contract](../integrations/openai-compatible-chat.md#explicit-smoke-diagnostics)
|
||||
owns their observable HTTP semantics.
|
||||
|
||||
An injected `LLMClient` does not pass through this HTTP boundary. Its returned
|
||||
response can establish response and inference facts, but not an HTTP
|
||||
destination, status, reachability, or upstream credential-supply fact. The
|
||||
facade does not silently install a built-in transport for those diagnostics.
|
||||
|
||||
## Prepared Generation
|
||||
|
||||
For [`RunPrepared`](../../engine.go), the runner supplies the model client with
|
||||
@@ -133,7 +164,10 @@ own configuration, client cloning, deterministic deadline precedence,
|
||||
authentication, request and response mapping, malformed data, error identity,
|
||||
cancellation, endpoint selection and composition, pre-transport rejection, and
|
||||
bounded single-document successful-response framing, closure, and
|
||||
response-body suppression. The focused
|
||||
response-body suppression. The [HTTP evidence tests](../../internal/llm/http_evidence_test.go)
|
||||
own invocation-local last-hop request/response facts, while the
|
||||
[prepared-generation evidence tests](../../internal/llm/prepared_generation_evidence_test.go)
|
||||
own retention across decoding and cancellation. The focused
|
||||
[provider HTTP error tests](../../internal/llm/provider_http_error_test.go)
|
||||
own envelope parsing, normalization, and bounded-reader cases; their
|
||||
[transport tests](../../internal/llm/provider_http_error_transport_test.go)
|
||||
|
||||
@@ -23,6 +23,10 @@ callback. The caller supplies normalized failure metadata; the recorder never
|
||||
parses an error string or receives provider bodies, credentials, headers, or
|
||||
other content. Events are values with no mutable containers. Callback panics
|
||||
propagate to the caller.
|
||||
Resolved prompt and profile IDs are the exact runtime identities selected for
|
||||
the invocation, including namespace qualification when supplied by an engine
|
||||
view. A prompt's optional version label is descriptive metadata, not a
|
||||
selection key.
|
||||
Per-operation callbacks are ordered and synchronous; different operations may
|
||||
invoke the same callback concurrently. The recorder marks each measured phase's
|
||||
end before delivery, so callback work does not inflate provider duration.
|
||||
@@ -41,6 +45,11 @@ attempts above the first in a generation, and repair generations with at least
|
||||
one dispatch. A terminal event is an aggregate and must not be added to its
|
||||
attempt events.
|
||||
|
||||
Smoke diagnostics mask inherited execution-recorder state while invoking
|
||||
generation or HTTP collaborators. Their private attempt evidence is assembled
|
||||
into a `SmokeTestReport`; it does not create ordinary run events or enter an
|
||||
enclosing run's observation totals.
|
||||
|
||||
The [recorder tests](../../internal/observation/recorder_test.go) own usage
|
||||
presence, phase sums, event ordering, empty-run accounting, identity snapshots,
|
||||
and callback panic behavior. The [generation coordinator](generation.md) owns
|
||||
|
||||
@@ -11,27 +11,27 @@ contributor workflow and validation.
|
||||
|
||||
| Component | Implemented responsibility | References |
|
||||
| --- | --- | --- |
|
||||
| Root `promptkit` package | Provides the supported engine facade, source, backend-registration, and injection options, public request, result, prompt-inspection, profile-inspection, and ordinary-run observation values, opaque prepared-execution handles, profile construction, extension interfaces, value conversion, redacted formatting, and typed capacity and generation error mapping. | [Package GoDoc](../../doc.go), [observations](../../observations.go), [prepared execution](../../prepared_execution.go), [backend API](../../backends.go), [engine facade](../../engine.go) |
|
||||
| `examples/go-library/prepare` | Demonstrates an offline downstream consumer using a prompt file, in-memory profile, inline input, and `Prepare`. It is not a public library package. | [Example program](../../examples/go-library/prepare/main.go) |
|
||||
| `examples/go-library/run` | Demonstrates an offline downstream consumer using a prompt file, in-memory profile, inline input, an injected deterministic model client, and `Run`. It is not a public library package. | [Example program](../../examples/go-library/run/main.go) |
|
||||
| Root `promptkit` package | Provides the supported engine facade, ConfigKit source helpers, collection and resource mapping conveniences, injection options, public requests and results, prompt/profile/requirement inspection and identity inventories, concrete-request validation, explicit smoke diagnostics, observation values, opaque prepared-execution handles, value conversion, redacted formatting, and typed capacity and generation error mapping. | [Package GoDoc](../../doc.go), [observations](../../observations.go), [prepared execution](../../prepared_execution.go), [backend API](../../backends.go), [engine facade](../../engine.go), [smoke diagnostics](../../smoke_diagnostics.go), [identity listing](../../listing.go), [requirement facade](../../requirements.go) |
|
||||
| `examples/go-library/prepare` | Demonstrates offline preparation from two explicitly selected synthetic collections, source-relative imported content, a scoped view, filtered listings, and operator inventory. It is not a public library package. | [Example program](../../examples/go-library/prepare/main.go), [collection inputs](../../examples/go-library/prepare/collections/support/collection.yaml) |
|
||||
| `examples/go-library/run` | Demonstrates an offline downstream consumer using a prompt file, programmatic profile, inline input, an injected deterministic model client, and `Run`. It is not a public library package. | [Example program](../../examples/go-library/run/main.go) |
|
||||
| `examples/go-library/observe` | Demonstrates offline built-in HTTP retry and synchronous metadata observation across structured proposal and assessment runs using embedded sources, two profiles on one finite backend, and an application decision. It is not a public library package. | [Example program](../../examples/go-library/observe/main.go) |
|
||||
| `internal/backend` | Constructs each engine's immutable registry from maintained definitions and consumer additions, validates and defensively copies definitions through the shared JSON-value package, and consumes the LLM-owned OpenAI-compatible reserved request-field rule. | [Backend registry](../../internal/backend/registry.go) |
|
||||
| `internal/catalog` | Strictly validates imported immutable maintained backend and profile catalog assets before engine assembly uses them. | [Catalog adapter](../../internal/catalog/catalog.go), [internal sources](sources.md#profiles-and-built-ins) |
|
||||
| `internal/backend` | Holds a defensive, immutable lookup of final backend definitions, including diagnostic metadata, plus capacity policies validated by the configuration adapter. | [Backend registry](../../internal/backend/registry.go) |
|
||||
| `internal/configuration` | Loads root and namespaced collection resources through ConfigKit; records effective field presence, validates final values including exact references and optional diagnostic metadata, captures indexed prompt definitions and source-relative message content, compiled templates, input metadata snapshots, sorted backend/profile/prompt identities, resource namespace sets, effective collection declarations, backend and presence-aware profile snapshots, and the unchanged read-only inspection view. | [Typed loader](../../internal/configuration/load.go), [identity listing](../../internal/configuration/listing.go), [internal sources](sources.md#typed-configuration-loading) |
|
||||
| `internal/capacity` | Owns engine-local bounded execution admission and FIFO model-generation permits for limited backend IDs, including cancellation-safe waiter removal and client wrapping. | [Internal capacity management](capacity.md) |
|
||||
| `internal/domain` | Defines internal framework values for requests, artifacts, prompt definitions, profiles, execution targets, rendering, generation, and validation, and owns source-neutral invariants for shared execution settings, OpenAI-compatible base endpoints, session identifiers, and output contracts. Source parsing, required fields, other source-specific normalization, defaulting, and boundary-specific error classification remain with their callers. | [Domain declarations](../../internal/domain/domain.go), [endpoint invariant](../../internal/domain/endpoint.go) |
|
||||
| `internal/engine` | Assembles external catalogs, Promptkit-owned profiles, repositories, registry, capacity, collaborators, repairer, and the one use-case runner for each engine instance. | [Engine assembly](../../internal/engine/engine.go), [internal sources](sources.md#profiles-and-built-ins), [internal runner](runner.md) |
|
||||
| `internal/generation` | Coordinates frozen built-in HTTP attempts and one-shot injected calls through active backend permits, including bounded status retries, cancellable backoff, and attempt observations. | [Internal generation coordination](generation.md) |
|
||||
| `internal/domain` | Defines internal framework values for requests, artifacts, prompt definitions, requirement inspection, profiles, execution targets, rendering, generation, validation, and source-neutral smoke diagnostics, and owns source-neutral invariants for shared execution settings, OpenAI-compatible base endpoints, session identifiers, output contracts, and resource-identity syntax and qualification. Source parsing, required fields, other source-specific normalization, defaulting, and boundary-specific error classification remain with their callers. | [Domain declarations](../../internal/domain/domain.go), [requirement values](../../internal/domain/requirements.go), [endpoint invariant](../../internal/domain/endpoint.go) |
|
||||
| `internal/engine` | Assembles caller-selected configuration into one snapshot, then constructs shared capacity, rendering, validation, generation, repair, the ordinary runner, and the smoke tester for each engine. Immutable namespace views share those collaborators and qualify caller-supplied execution, inspection, and diagnostic selections without rewriting captured references. Built-in engines provide the diagnostic HTTP collaborator; injected-client engines do not. It delegates identity listing directly to the captured configuration inventory rather than the runner. | [Engine assembly](../../internal/engine/engine.go), [internal sources](sources.md#typed-configuration-loading), [internal runner](runner.md) |
|
||||
| `internal/generation` | Coordinates frozen built-in HTTP attempts and one-shot injected calls through active backend permits, including bounded status retries, cancellable backoff, ordinary attempt observations, and invocation-local evidence for diagnostics. | [Internal generation coordination](generation.md) |
|
||||
| `internal/defaults` | Defines application-neutral framework constants and constructs the default execution target. It contains no CLI, server, or inbound HTTP limits. | [Framework defaults](../../internal/defaults/defaults.go) |
|
||||
| `internal/filecatalog` | Provides deterministic YAML discovery and path helpers for operating-system filesystems and `fs.FS` sources. | [File catalog](../../internal/filecatalog/catalog.go) |
|
||||
| `internal/jsonvalue` | Validates and deeply copies bounded JSON-compatible extra-parameter and prepared-schema trees while preserving supported concrete value types and rejecting cycles or excessive depth and work. | [JSON values](../../internal/jsonvalue/jsonvalue.go) |
|
||||
| `internal/promptdef` | Loads strictly decoded, validated prompt definitions from filesystem and `fs.FS` sources, including version selection and contained file-backed message content. | [Framework formats](../formats.md), [prompt-definition repository](../../internal/promptdef/filesystem_repository.go) |
|
||||
| `internal/profile` | Owns immutable in-memory and filesystem profile repositories, local profile validation, raw-source overlays with error-preserving fallback, and inherited-profile resolution. | [Framework formats](../formats.md), [profile repositories](../../internal/profile/filesystem_repository.go), [internal sources](sources.md#profiles-and-built-ins) |
|
||||
| `internal/prompt` | Renders prompt messages from Go templates with artifact, variable, session, and cache-control data. | [Go-template renderer](../../internal/prompt/go_renderer.go) |
|
||||
| `internal/promptdef` | Provides a narrow captured-prompt lookup contract, not-found identity, and execution-ready definition hash. | [Prompt lookup](../../internal/promptdef/repository.go), [framework formats](../formats.md) |
|
||||
| `internal/profile` | Provides a narrow captured-profile lookup contract and not-found identity. | [Profile lookup](../../internal/profile/repository.go), [framework formats](../formats.md) |
|
||||
| `internal/prompt` | Renders prompt messages from Go templates with artifact, variable, session, and cache-control data, and conservatively enumerates dependencies from captured template syntax. | [Go-template renderer](../../internal/prompt/go_renderer.go), [requirement discovery](../../internal/prompt/requirements.go) |
|
||||
| `internal/artifact` | Resolves ordinary inline and unrestricted caller-selected file references into copied artifacts with metadata and hashes. | [Internal sources and validation](sources.md) |
|
||||
| `internal/validate` | Validates basic, JSON, and JSON Schema output using operating-system filesystem or `fs.FS` schema sources and creates operation-local validation plans with canonical contained schema resources. | [Framework formats](../formats.md#schemas), [internal sources and validation](sources.md) |
|
||||
| `internal/llm` | Defines the internal generation boundary and implements outbound OpenAI-compatible chat requests from resolved execution targets, including bounded structured non-success response decoding, successful-response decoding, authentication, deadline handling, and ownership of the OpenAI-compatible reserved request-field policy. | [Internal model client](llm.md) |
|
||||
| `internal/observation` | Owns invocation-local run IDs, presence-aware usage and phase accounting, and synchronous metadata event delivery for ordinary and claimed prepared runs and generation attempts. | [Internal execution observation](observation.md) |
|
||||
| `internal/usecase` | Resolves prompt definitions and hashes, profiles, backends, and targets for exact inspection and request settings for preparation, and coordinates ordinary and single-claim prepared execution across internal sources, rendering, artifact loading, operation-local validation plans, generation, capacity, and bounded repair. | [Internal runner](runner.md), [prepared-execution implementation](../../internal/usecase/prepared_execution.go) |
|
||||
| `internal/llm` | Defines the internal generation boundary and implements outbound OpenAI-compatible chat requests from resolved execution targets, including bounded response decoding, authentication, deadline handling, per-invocation HTTP evidence and built-in diagnostic probes, and ownership of the OpenAI-compatible reserved request-field policy. | [Internal model client](llm.md) |
|
||||
| `internal/observation` | Owns invocation-local run IDs, presence-aware usage and phase accounting, and synchronous metadata event delivery for ordinary and claimed prepared runs and generation attempts; smoke diagnostics keep their private evidence outside ordinary observer events. | [Internal execution observation](observation.md) |
|
||||
| `internal/usecase` | Selects captured prompts and profiles, enumerates requirements from captured prompt syntax, applies request overrides and completion checks, coordinates ordinary and single-claim prepared execution across rendering, artifact loading, operation-local validation plans, generation, capacity, and bounded repair, and runs explicit smoke diagnostics with private cases, shared generation/capacity, and optional built-in HTTP probes. | [Internal runner](runner.md), [requirement inspection](../../internal/usecase/requirements_inspection.go), [prepared-execution implementation](../../internal/usecase/prepared_execution.go) |
|
||||
|
||||
The internal engine assembles these internal components without exposing their
|
||||
representations. Consumers depend only on the root facade.
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
|
||||
This document describes Promptkit's implemented internal orchestration. The
|
||||
[architecture policy](../policy/architecture.md) owns dependency and consumer
|
||||
boundaries. The [source and validation document](sources.md) owns repository,
|
||||
boundaries. The [source and validation document](sources.md) owns configuration,
|
||||
artifact, rendering, and validation behavior, while the
|
||||
[model-client document](llm.md) owns generation behavior and failure
|
||||
categories.
|
||||
@@ -16,11 +16,10 @@ and override semantics consumed by the runner.
|
||||
|
||||
## Collaborators
|
||||
|
||||
`Runner` coordinates narrow internal interfaces for prompt definitions,
|
||||
profiles, backend resolution, artifacts, rendering, model generation, and
|
||||
validation. The internal engine supplies one immutable registry containing the
|
||||
built-in backend and validated consumer additions, one engine-local run
|
||||
admitter, and a generation coordinator using the same capacity manager. Validation
|
||||
`Runner` coordinates narrow internal interfaces for captured prompt definitions
|
||||
and resolved profiles, artifacts, rendering, model generation, and validation.
|
||||
The internal engine supplies one engine-local run admitter and a generation
|
||||
coordinator using the same capacity manager. Validation
|
||||
plans and provider-facing schema metadata come from the validator's preparation
|
||||
interface.
|
||||
The internal engine supplies one default output repairer through the explicit
|
||||
@@ -33,32 +32,82 @@ The runner has no durable run or session store.
|
||||
|
||||
## Shared Prompt Selection
|
||||
|
||||
The runner uses one prompt-selection and hashing boundary for ordinary
|
||||
preparation and exact prompt inspection. Preparation retains its early
|
||||
request-ID check before direct-session normalization; both operations then use
|
||||
the configured prompt repository to select one definition, load referenced
|
||||
message content, and calculate the same prompt hash.
|
||||
The internal engine applies each namespace view to slashless caller selections
|
||||
before they reach the runner. The runner therefore selects captured prompts and
|
||||
profiles by the resulting exact runtime IDs. Already-qualified selections pass
|
||||
through, while configured defaults and stored references are used as authored
|
||||
and are not requalified. Prompt version labels are returned as optional
|
||||
descriptive metadata; they are not prompt selectors.
|
||||
|
||||
Inspection stops after that structural lookup. It does not parse templates or
|
||||
touch profile, artifact, schema, renderer, validator, admission, or model
|
||||
collaborators. The root [`Engine.InspectPrompt`](../../engine.go) GoDoc owns
|
||||
the public operation's exact contract.
|
||||
Ordinary preparation selects the full prompt definition and construction-time
|
||||
hash. It retains its early request-ID check before direct-session
|
||||
normalization. Referenced message content and static template syntax were
|
||||
captured when the engine was built.
|
||||
|
||||
`InspectPrompt` selects captured metadata through an optional repository
|
||||
capability, falling back to the full definition and existing hash resolution
|
||||
for internal repositories without it. It returns owned metadata without
|
||||
rendering templates or touching profile, artifact, schema, renderer, validator,
|
||||
admission, or model collaborators. The root
|
||||
[`Engine.InspectPrompt`](../../engine.go) GoDoc owns the public operation's
|
||||
exact contract.
|
||||
|
||||
`InspectRequirements` uses the same metadata selection to resolve the
|
||||
request-profile/default identity, then obtains the captured requirement
|
||||
snapshot by the selected prompt ID. The analyzer receives only
|
||||
those identity fields rather than an execution definition. It analyzes the
|
||||
snapshot with the supplied caller variables and normalized direct-session
|
||||
bypass; it does not resolve target readiness or enter the preparation
|
||||
pipeline. Missing snapshot capability is a prompt-load failure rather than a
|
||||
source fallback. The root
|
||||
[`Engine.InspectRequirements`](../../requirements.go) GoDoc owns the public
|
||||
operation's exact contract.
|
||||
|
||||
## Shared Profile Selection
|
||||
|
||||
The runner uses one profile-selection and target-resolution boundary for
|
||||
ordinary preparation and exact profile inspection. Preparation first selects a
|
||||
request profile or a prompt default; inspection begins with its required
|
||||
explicit profile ID. Both then apply the ordinary source precedence, resolve a
|
||||
named backend, and construct the effective target from framework, backend, and
|
||||
profile values.
|
||||
|
||||
Inspection stops after the resulting endpoint and model are structurally
|
||||
validated. It does not check credential availability or perform prompt,
|
||||
explicit profile ID. Both use the captured, presence-aware target constructed
|
||||
from the final effective profile and backend configuration. Request overrides
|
||||
apply afterward; preparation checks endpoint and model completeness only after
|
||||
those overrides. Inspection can report a partial profile without requiring a
|
||||
runnable target. Neither path checks credential availability or performs prompt,
|
||||
artifact, schema, rendering, admission, or model-client work. The root
|
||||
[`Engine.InspectProfile`](../../engine.go) GoDoc owns the public operation's
|
||||
exact contract.
|
||||
|
||||
## Smoke-Test Diagnostics
|
||||
|
||||
The separate `SmokeTester` in `internal/usecase` selects from the same captured
|
||||
configuration snapshot and backend/profile collaborators as the runner. The
|
||||
internal engine applies namespace-view qualification to slashless caller IDs
|
||||
before diagnostic selection; captured defaults and references remain exact.
|
||||
Explicit profile selection takes precedence over prompt-default selection and a
|
||||
backend diagnostic default; HTTP-only selection does not resolve a default
|
||||
inference profile. Exact public selection rules belong to the root
|
||||
[`Engine.SmokeTest` GoDoc](../../smoke_diagnostics.go).
|
||||
|
||||
Diagnostics do not call `Run`, `Prepare`, or the prompt preparation pipeline.
|
||||
They do not render application messages, load application artifacts, use
|
||||
application schemas, or inspect session templates. Private text and structured
|
||||
cases and the structured case's schema remain internal to the diagnostic
|
||||
operation; they are not entries in prompt/profile listings or inspection.
|
||||
`InspectConfiguration` remains a separate, explicit operator-oriented view of
|
||||
effective settings and source history. A diagnostic report may include selected
|
||||
effective settings and identifiers, so consumers still apply their disclosure
|
||||
policy.
|
||||
|
||||
The engine assembles the smoke tester with the runner's generation coordinator
|
||||
and capacity manager. Inference holds the ordinary logical admission lease and
|
||||
uses the same active permits, retries, timeout, and request evidence path as
|
||||
generation. Reachability and authentication probes bypass generation capacity
|
||||
and do not require a model. An engine with an injected `LLMClient` can run
|
||||
inference diagnostics but has no built-in HTTP probe collaborator; HTTP-only
|
||||
diagnostics report that limitation instead of falling back to another client.
|
||||
The public GoDoc owns the caller-visible timeout, cancellation, report, and
|
||||
transport semantics.
|
||||
|
||||
## Shared Preparation Pipeline
|
||||
|
||||
`Prepare` and `Run` share one private preparation pipeline split at the point
|
||||
@@ -66,11 +115,10 @@ where a run can be assigned to its selected backend pool. The resolution phase
|
||||
performs only the work needed to validate routing and admission:
|
||||
|
||||
1. validate the required prompt selection and normalize any direct session ID;
|
||||
2. load the prompt definition and hash the original definition;
|
||||
2. select the captured prompt definition and hash;
|
||||
3. select the request profile or the prompt's default profile;
|
||||
4. resolve the profile's backend ID, when present;
|
||||
5. resolve application-neutral defaults, backend defaults, profile values,
|
||||
and explicit request overrides in that order;
|
||||
4. retain the profile's selected backend identity, when present;
|
||||
5. apply explicit request overrides to its captured backend-completed target;
|
||||
6. validate endpoint, model, numeric overrides, and credential requirements;
|
||||
7. resolve and validate the effective output contract without loading its
|
||||
schema; and
|
||||
@@ -95,33 +143,42 @@ one execution-precedence and error-ordering implementation while allowing a
|
||||
full backend pool to reject work before expensive schema, artifact, and
|
||||
rendering operations. `Prepare` discards the plan after returning its public
|
||||
metadata. `Run` retains the plan through initial and repaired-output validation
|
||||
and discards it when the operation ends. Prepared execution stores the same
|
||||
kind of plan only in its private payload.
|
||||
and discards it when the operation ends. Prepared execution transfers the
|
||||
completed preparation's owned message, input-hash, and copied-schema values
|
||||
into its private payload, while deep-copying mutable provider extras and
|
||||
creating an independent details snapshot. Claim completion or discard clears
|
||||
the execution payload without removing the details snapshot.
|
||||
|
||||
The root facade's `Engine.ValidateRequest` uses the same complete preparation
|
||||
path as `Prepare` and discards the resulting internal prepared value without
|
||||
converting it into a public snapshot. It does not add a runner operation or
|
||||
validator component; request conversion and public error mapping remain at the
|
||||
facade boundary.
|
||||
|
||||
Pointer-based numeric overrides preserve an explicit zero. Invalid negative or
|
||||
out-of-range values fail as invalid requests. Endpoint overrides do not change
|
||||
the selected backend identity. Non-empty extra-parameter maps replace whole
|
||||
lower-precedence maps. A direct API key takes precedence over environment
|
||||
lookup; otherwise request, profile, and backend environment-variable names
|
||||
apply in that order. A profile requiring a direct key clears an inherited
|
||||
lookup; otherwise request, explicit profile, and backend environment-variable
|
||||
names apply in that order. A profile requiring a direct key suppresses an inherited
|
||||
backend environment name unless the request supplies its own name. Secret
|
||||
values remain excluded from serialized metadata.
|
||||
|
||||
Reasoning overrides are tri-state: nil inherits the profile, a pointer to a
|
||||
nonblank string trims and replaces it, and a pointer to a blank string clears
|
||||
it. A nonblank direct session is normalized before source loading, bypasses
|
||||
it. A nonblank direct session is normalized before prompt selection, bypasses
|
||||
the prompt session template, and is applied after ordinary message rendering.
|
||||
A blank direct value retains prompt-template behavior. The runner clears the
|
||||
template only on a value copy of the definition, so the definition hash always
|
||||
describes the original source while the rendered-prompt hash includes the
|
||||
describes captured effective content while the rendered-prompt hash includes the
|
||||
effective direct or rendered session and the complete effective message
|
||||
sequence. The rendered-prompt hash uses a versioned, length-framed SHA-256
|
||||
encoding of that session, every role and content value, and cache-control
|
||||
presence and values; its hexadecimal value is opaque.
|
||||
|
||||
The registry is read-only after engine construction. Concurrent `Prepare` and
|
||||
`Run` calls resolve independent defensive backend values and keep all
|
||||
invocation state local.
|
||||
The configuration snapshot is read-only after engine construction. Concurrent
|
||||
`Prepare` and `Run` calls copy their resolved targets and keep all invocation
|
||||
state local.
|
||||
|
||||
## Run Flow
|
||||
|
||||
@@ -215,7 +272,7 @@ generation, validation, repair, and cancellation failures.
|
||||
Other context cancellation propagates through the invoked collaborator and is
|
||||
classified by the owning operation. In particular, cancellation observed by
|
||||
validation retains the context identity through the validation error category.
|
||||
An overlong direct session is an invalid request before source loading, while
|
||||
An overlong direct session is an invalid request before prompt selection, while
|
||||
an invalid or overlong prompt session template remains a prompt-render failure.
|
||||
An unknown selected backend, or a selected backend with no configured resolver,
|
||||
is classified as a profile-load failure.
|
||||
@@ -237,4 +294,8 @@ Changes to orchestration should continue to use the existing package
|
||||
interfaces, keep request state local to an invocation, and preserve the shared
|
||||
resolution and completion pipeline. Source, renderer, validator, or
|
||||
model-client contract changes belong first in their owning package and
|
||||
document.
|
||||
document. The
|
||||
[smoke selection tests](../../internal/usecase/smoke_selection_test.go),
|
||||
[diagnostic orchestration tests](../../internal/usecase/smoke_tester_test.go),
|
||||
and [private inference-case tests](../../internal/usecase/smoke_inference_test.go)
|
||||
own diagnostic selection, report assembly, and inference findings.
|
||||
|
||||
@@ -2,117 +2,161 @@
|
||||
|
||||
## Purpose
|
||||
|
||||
This document describes Promptkit's implemented internal source, artifact,
|
||||
This document describes Promptkit's implemented configuration, artifact,
|
||||
rendering, and output-validation behavior. The
|
||||
[architecture policy](../policy/architecture.md) owns the library boundary and
|
||||
dependency rules. None of these internal packages is a supported consumer API,
|
||||
and the internal engine assembles them behind the public facade's source options and values.
|
||||
The [framework format reference](../formats.md) owns the exact file fields,
|
||||
validation modes, built-in catalog, and source precedence.
|
||||
[architecture policy](../policy/architecture.md) owns dependency rules, while
|
||||
the [framework format reference](../formats.md) owns fields, defaults, and
|
||||
source semantics. Internal packages are not consumer APIs.
|
||||
|
||||
## Typed Configuration Loading
|
||||
|
||||
`internal/configuration` is the single adapter from ConfigKit's final
|
||||
effective configuration to execution-owned definitions. It performs one
|
||||
layered load, captures effective presence (including explicit null), records
|
||||
normalization and defaults through resolution, and validates all effective
|
||||
root and collection backend, profile, and prompt values. Replaced lower-layer
|
||||
domain values are not separately validated. It indexes root resources under
|
||||
their unqualified IDs and collection resources under their full
|
||||
`namespace/local-key` IDs while retaining ConfigKit's effective tree in its
|
||||
authored shape. It builds an immutable backend index and presence-aware profile
|
||||
targets, applying final backend defaults only after profile inheritance. Runtime
|
||||
lookups use those final indexes and exact stored references; they do not rewrite
|
||||
or resolve resources from ConfigKit source history. The runtime/domain packages
|
||||
do not depend on ConfigKit.
|
||||
|
||||
ConfigKit owns source headers, imports, composition, and source evidence.
|
||||
Promptkit does not preparse contributions: it validates the final effective
|
||||
values once, captures the indexed identities and their original resource
|
||||
contexts, and retains ConfigKit's result for explicit operator inspection.
|
||||
Accepted non-main source headers are consumed during composition; the main
|
||||
source declaration remains available in that inspection view. Prompt and
|
||||
collection release labels are descriptive metadata, not selection inputs. The
|
||||
[format reference](../formats.md#source-headers-collections-and-resource-ids)
|
||||
owns the exact source-header and collection contracts.
|
||||
|
||||
The `ConfigurationInspection.Diff` facade method compares two retained results
|
||||
through a private wrapper and a narrow `internal/configuration.Diff` bridge.
|
||||
The wrapper keeps `Result[document]` inside the adapter; the bridge delegates
|
||||
once to ConfigKit v0.8.0 and returns its public difference records. Comparison
|
||||
uses only frozen effective values and does not reopen sources, resolve runtime
|
||||
credentials, or enter the execution path. ConfigKit applies sensitivity from
|
||||
both results; reports may still contain ordinary application data. See the
|
||||
[consumer comparison guide](../consumers/pkg-promptkit.md#inspect-the-captured-configuration)
|
||||
and ConfigKit's [v0.8.0 comparison contract](https://gitea.maximumdirect.net/eric/ConfigKit/src/tag/v0.8.0/docs/config.md#effective-configuration-diffs).
|
||||
|
||||
After resource materialization and effective-profile snapshot construction, the
|
||||
adapter captures lexically ordered identity inventories for backends, profiles,
|
||||
and prompts, per-kind resource namespace sets, and metadata for effective
|
||||
collection declarations. Listing methods return caller-owned copies from those
|
||||
captured values without consulting ConfigKit or copying resource definitions.
|
||||
|
||||
The root `DirectoryInputs` and `FilesystemInputs` helpers discover sorted YAML
|
||||
files under an explicit root, provide resource context, and leave imports
|
||||
disabled. Each returned slice belongs to one strict-composition group; file
|
||||
order does not confer priority. Caller underlays are composed in their given
|
||||
order, followed by the optional main input. The
|
||||
[configuration tests](../../internal/configuration/load_test.go) exercise the
|
||||
real loader and final-value handoff.
|
||||
|
||||
The public `CollectionInput` helper mounts one caller-selected source at
|
||||
`collections/<namespace>` and enables ConfigKit imports for that input. The
|
||||
helper does not read the source; ConfigKit retains responsibility for loading,
|
||||
composition, imports, and source-local resource context. Imported or inherited
|
||||
`content_file` values continue resolving relative to the source that contributed
|
||||
the effective field, not to a flattened runtime identity. Explicit
|
||||
`InspectConfiguration` views expose the retained effective configuration and
|
||||
provenance for operator review; runtime selection and inventories do not derive
|
||||
from source history.
|
||||
|
||||
## Prompt Definitions
|
||||
|
||||
`internal/promptdef` uses one source-neutral flow for prompt selection and
|
||||
normalization. That flow scans normalized YAML ID and version metadata,
|
||||
requires one strictly decoded document per file, classifies errors for the
|
||||
selected definition, detects duplicates, and normalizes the exact match.
|
||||
Small operating-system and `fs.FS` adapters own discovery, byte reads, display
|
||||
paths, content opening, and root containment. Each lookup remains a
|
||||
point-in-time scan: definitions and catalogs are not cached, and file-backed
|
||||
message content is opened only for the exact selected candidate.
|
||||
The configuration adapter indexes each prompt by exact ID,
|
||||
materializes file-backed message content using the exact effective field's
|
||||
origin and resource context, and checks static message/session template syntax
|
||||
at construction. It stores captured content, compiled templates, declared
|
||||
input metadata, and a definition hash. Alongside the unchanged hashed
|
||||
definition, it retains content-type and description presence for each declared
|
||||
input and provides an internal requirement snapshot with those declarations
|
||||
and the captured parses. Metadata values come from the final normalized
|
||||
declaration; absent and null fields remain distinguishable from explicit
|
||||
strings, including empty strings. It closes content-file handles before
|
||||
returning and retains no source capability for public inspection.
|
||||
|
||||
Message roles are normalized through the shared domain owner by trimming
|
||||
Unicode whitespace and lowercasing. Only `developer`, `system`, `user`, and
|
||||
`assistant` are published; invalid roles remain selected prompt-definition
|
||||
failures rather than becoming request errors. Cache-control metadata uses the
|
||||
same shared domain normalization and defensive-copy rule.
|
||||
Prompt materialization shares a lazy metadata snapshot across root and collection
|
||||
resources during construction, avoiding a full metadata copy per collection.
|
||||
|
||||
Operating-system sources enforce containment against canonical roots and
|
||||
targets so symlinks cannot escape. Injected `fs.FS` sources enforce containment
|
||||
in their clean relative path namespace. A single-file source uses the selected
|
||||
prompt file's containing directory as its root. Every content path must be
|
||||
relative and is opened from its exact parsed text after a separate blank check;
|
||||
contained parent components and whitespace-bearing names remain valid.
|
||||
Role and cache-control normalization is applied to execution-owned message
|
||||
copies. ConfigKit v0.8.0 retains effective input message metadata and its
|
||||
resource origins; recording a sequence-wide normalization in the retained
|
||||
configuration would replace origins of unchanged content-file fields.
|
||||
|
||||
Exact prompt inspection performs one point-in-time lookup through that same
|
||||
repository and validates referenced message content before returning declared
|
||||
metadata. It does not parse templates or read profile, input, or schema
|
||||
sources, and it does not retain the definition for a later execution.
|
||||
The runner selects full definitions from this index for preparation and uses
|
||||
captured prompt metadata for `InspectPrompt` and `InspectRequirements`; the
|
||||
internal requirement snapshot lookup uses the selected prompt ID alone.
|
||||
These paths do not rescan configuration or reopen message files. `InspectPrompt`
|
||||
returns declared metadata without rendering or resolving profile, input, or
|
||||
schema resources, and its public shape and prompt hashes do not include the
|
||||
captured presence state. Runtime rendering still checks request variables and
|
||||
input helpers. The [prompt adapter tests](../../internal/configuration/prompts_test.go)
|
||||
cover resource origins, context containment, capture, and selection.
|
||||
|
||||
Its package tests own prompt selection, strict decoding, definition validation,
|
||||
duplicate detection, and source containment:
|
||||
[prompt-definition repository tests](../../internal/promptdef/repository_test.go).
|
||||
The configuration adapter shares ID-only selection across its internal prompt
|
||||
lookups. The execution snapshot lookup returns an owned prompt
|
||||
definition and cached hash; metadata inspection returns owned input metadata
|
||||
without copying execution templates. Profile selection likewise has an
|
||||
identity-only lookup for consumers that do not need the execution target,
|
||||
while the full profile lookup continues to return an owned target and deep copy
|
||||
of provider extras. These lookups use captured indexes and do not query
|
||||
ConfigKit again.
|
||||
|
||||
## Profiles And Built-Ins
|
||||
The `internal/prompt` requirement walker uses the captured parse trees and
|
||||
declaration metadata without executing templates or reading artifacts. It
|
||||
collects literal `input` names, direct root caller keys, and literal-key
|
||||
`index` lookups. For `if` and `with`, it classifies branches when their
|
||||
conditions are simple supported values: direct values, literals, negation, or
|
||||
two-string equality/inequality. It uses Go template truthiness, so a supplied
|
||||
nonempty string such as `false` is true. Missing decision values and
|
||||
content-dependent conditions leave dependency-bearing branches conditional
|
||||
and report incomplete discovery; unresolved conditions with no branch
|
||||
dependencies do not by themselves make discovery incomplete. Recognizable
|
||||
references on known skipped paths remain optional inventory entries with an
|
||||
inactive-use explanation, unless another applicable use establishes a
|
||||
stronger classification. Skipped expressions are scanned only for
|
||||
syntactically recognizable names and are not evaluated. A direct caller value
|
||||
or literal-key lookup can also supply an input name, including through simple
|
||||
parentheses or a direct pipeline to `input`; names are used exactly as
|
||||
supplied. Unresolved names and unsupported dependency-bearing expressions
|
||||
produce structured limitations while retaining recognizable references.
|
||||
Where an unresolved name or scope could affect existing optional entries, those
|
||||
entries are reported as conditional rather than inactive-only or definitively
|
||||
optional. Direct root-map `with` forms retain caller-map identity. Known scalar
|
||||
values remain separate from unknown scopes; accesses through untracked local
|
||||
maps report incomplete discovery rather than silently losing caller keys.
|
||||
Unsupported name expressions reuse value analysis's conservative syntax scan
|
||||
instead of recursively rescanning the same subtree.
|
||||
Invoked named templates are scanned conservatively, and
|
||||
dependency-bearing range scopes make discovery incomplete. Uncalled definitions
|
||||
and bypassed session templates are not traversed. This internal walker does not
|
||||
change rendering behavior.
|
||||
|
||||
`internal/profile` owns immutable in-memory and filesystem repositories, local
|
||||
validation, overlays, and resolution for execution profiles. In-memory
|
||||
definitions are normalized and validated once at that repository boundary and
|
||||
are defensively copied on storage and lookup. A file contains
|
||||
exactly one YAML document and its trimmed YAML `id` is its only selection
|
||||
identity; filenames do not confer authority. Each point lookup reads discovered
|
||||
files once for their metadata and reuses the selected file's bytes for strict
|
||||
decoding; unrelated profiles are not fully decoded. Strict selected decoding
|
||||
recognizes `base_profile` and the optional `backend` field, trims their values,
|
||||
and permits inherited target fields only when a base is named. File-backed
|
||||
`extra_params` values are validated and defensively copied through the shared
|
||||
bounded JSON-value owner before a profile is published. OpenAI-compatible
|
||||
reserved-field policy remains with the model-client and backend-registry owners.
|
||||
## Profiles
|
||||
|
||||
`LoadFSRepository` is the eager immutable loading boundary for internal
|
||||
catalog consumers. It discovers and strictly validates every raw profile once,
|
||||
preserves safe source metadata including explicitly present YAML fields, and
|
||||
publishes independently copied values from memory. It does not resolve profile
|
||||
inheritance. Configured consumer sources continue to use the lazy point lookup
|
||||
repositories described above.
|
||||
ConfigKit resolves profile inheritance before the handoff. Within a collection,
|
||||
`extends` names a sibling local key; it is not a cross-collection reference.
|
||||
The configuration adapter validates final effective resource references and
|
||||
field values, then builds target
|
||||
snapshots with backend defaults and presence/suppression state. A profile can
|
||||
remain partial until runtime overrides complete it. `InspectProfile` selects
|
||||
that captured target without prompt, artifact, schema, credential lookup, or
|
||||
provider work. Preparation applies runtime overrides and then checks target
|
||||
completeness. Identity-only profile selection can identify a partial captured
|
||||
profile without copying its target or requiring readiness. The
|
||||
[target tests](../../internal/configuration/targets_test.go) cover effective
|
||||
target interpretation.
|
||||
|
||||
`internal/catalog` validates the imported immutable OpenRouter and
|
||||
Rakestrawhome asset modules as one private adapter boundary. It enforces their
|
||||
manifest, layout, profile ownership, inheritance, and secret-safety rules
|
||||
before returning raw catalog sources. Internal engine assembly uses those validated
|
||||
catalogs as the lowest-precedence profile source.
|
||||
|
||||
Normalized external catalog profile IDs with the exact `promptkit-` prefix are
|
||||
rejected at this adapter boundary. That reservation applies only to imported
|
||||
catalogs; configured consumer sources retain the ordinary profile-ID rules.
|
||||
|
||||
The overlay repository consults the next repository only when the
|
||||
higher-precedence repository reports that a profile is absent. A reliably
|
||||
selected malformed profile stops fallback, while an unrelated malformed file
|
||||
does not become authoritative through its filename. Loading does not check
|
||||
backend registry membership because the available registry belongs to the
|
||||
assembled engine; the runner checks membership during preparation and exact
|
||||
profile inspection.
|
||||
|
||||
The internal engine assembles one raw composite catalog in precedence order:
|
||||
in-memory profiles, one ordinary configured source, an application fallback
|
||||
source, Promptkit-owned profiles, then the maintained external catalog. It
|
||||
constructs the owned definitions through the in-memory repository and validates
|
||||
their maintained-only resolution and backend membership before consumer sources
|
||||
are overlaid. An explicit file or `fs.FS` profile source replaces
|
||||
`Config.ProfileDir` within the ordinary configured-source category. One outer
|
||||
resolving repository wraps that complete raw catalog, so each base lookup
|
||||
observes the same precedence and shadowing rules.
|
||||
|
||||
The resolving repository traverses every selected chain afresh, retains no
|
||||
cache, detects cycles, limits a chain to 32 profiles, merges root-to-leaf into a
|
||||
new caller-owned value, and validates the final target before publishing it. It
|
||||
does not check backend registry membership. Exact `base_profile` syntax, merge
|
||||
rules, and consumer-visible failure behavior belong to the [framework format
|
||||
reference](../formats.md#profile-inheritance).
|
||||
|
||||
Exact profile inspection performs one point-in-time resolved lookup through
|
||||
those profile sources and checks the final target without reading prompt, input,
|
||||
or schema sources. It does not retain that lookup for a later execution.
|
||||
Prepared execution instead freezes the fully resolved target; a later ordinary
|
||||
operation performs a fresh traversal.
|
||||
|
||||
The imported external catalog provides external profiles and matching backend
|
||||
definitions. The internal engine owns the Promptkit profiles documented in the
|
||||
[framework format reference](../formats.md#built-in-profile-catalog). Profile
|
||||
loading and overlay behavior are owned by the [profile repository tests](../../internal/profile/repository_test.go), while
|
||||
catalog completeness, backend selection, and duplicate IDs are owned by the
|
||||
[catalog adapter tests](../../internal/catalog/catalog_test.go).
|
||||
The internal engine delegates listing through a narrow identity-inventory
|
||||
interface directly to the configuration snapshot. The execution runner has no
|
||||
listing responsibility.
|
||||
|
||||
## Ordinary Artifacts
|
||||
|
||||
@@ -123,10 +167,15 @@ and after opening the file. It copies content into an artifact, records
|
||||
metadata and an opaque content-equality value, and applies a content-type
|
||||
fallback.
|
||||
|
||||
Regular files are read synchronously in bounded chunks. Cancellation is
|
||||
checked before opening, before and after every read, and before publishing the
|
||||
artifact, so a canceled read never publishes partial content. The ordinary
|
||||
reader does not detach file reads into background goroutines.
|
||||
Regular files are read synchronously in bounded chunks. The opened file's size
|
||||
is only an initial-capacity hint, capped at 1 MiB; reads continue to EOF, and
|
||||
the returned body determines artifact size and hash. If an overreported hint
|
||||
leaves unused capacity greater than the actual body length, the completed body
|
||||
is copied to release the excess backing storage, including when the body is
|
||||
empty. Cancellation is checked before opening, before and after every read,
|
||||
and before publishing the artifact, so a
|
||||
canceled read never publishes partial content. The ordinary reader does not
|
||||
detach file reads into background goroutines.
|
||||
|
||||
This ordinary reader does not implement an inbound HTTP security boundary. In
|
||||
particular, it does not constrain files to an application root or impose an
|
||||
@@ -137,19 +186,22 @@ implemented reader behavior and failures.
|
||||
|
||||
## Rendering
|
||||
|
||||
`internal/prompt` renders definition messages as Go templates using named
|
||||
artifacts and variables. Within one render, each referenced artifact body is
|
||||
converted to text lazily and cached by input name for reuse across the session
|
||||
and every message; the cache is not shared across renders. Conversion uses
|
||||
bounded chunks and preserves the artifact bytes exactly.
|
||||
`internal/prompt` renders captured, pre-parsed definition templates using named
|
||||
artifacts and variables. It clones templates to bind request-specific input
|
||||
helpers without mutating shared state. Within one render, each referenced
|
||||
artifact body is converted to text lazily and cached by input name for reuse
|
||||
across the session and every message; the cache is not shared across renders.
|
||||
Conversion uses bounded chunks and preserves the artifact bytes exactly.
|
||||
|
||||
Session and message parsing and execution remain synchronous. The renderer
|
||||
checks cancellation before and after each parse and execution boundary,
|
||||
Template parsing occurs during engine construction. Runtime execution remains
|
||||
synchronous. The renderer checks cancellation around execution boundaries,
|
||||
between artifact conversion chunks, around each message, and before publishing
|
||||
the complete prompt. It cannot interrupt template work already in progress and
|
||||
never publishes a partial prompt after observing cancellation. It validates and
|
||||
canonicalizes message roles before carrying roles, session IDs, and cache
|
||||
control into the rendered prompt. The
|
||||
control into the rendered prompt. Message and session templates execute into
|
||||
per-call string builders, so their rendered strings do not require a final
|
||||
buffer-to-string copy. The
|
||||
[renderer tests](../../internal/prompt/renderer_test.go) own rendering behavior.
|
||||
|
||||
## Schemas And Output Validation
|
||||
@@ -157,7 +209,8 @@ control into the rendered prompt. The
|
||||
`internal/validate` provides validators backed by an operating-system
|
||||
filesystem or an `fs.FS`. Invalid generated content is returned as a validation
|
||||
result; inability to load, register, or compile a schema is an operational
|
||||
error.
|
||||
error. Basic validation checks body emptiness on the byte slice directly
|
||||
instead of first making a response-sized string copy.
|
||||
|
||||
Every preparation operation creates one operation-local validation plan. None,
|
||||
basic, and JSON modes retain the effective output contract without source
|
||||
@@ -181,9 +234,10 @@ not detach dependency work into background goroutines.
|
||||
`Prepare` discards its validation plan after returning metadata. `Run` retains
|
||||
its plan for initial and repaired-output validation, then discards it with the
|
||||
operation. `PrepareExecution` retains the plan in its private frozen payload;
|
||||
`RunPrepared` uses that plan without reopening prompt, profile, input, or
|
||||
schema sources or rerendering the request. A later ordinary `Run` always
|
||||
performs fresh source resolution and preparation.
|
||||
`RunPrepared` uses that plan without reopening configuration, message, input,
|
||||
or schema resources or rerendering the request. A later ordinary `Run` uses
|
||||
the same captured configuration but prepares request artifacts and schemas
|
||||
afresh.
|
||||
|
||||
The [validator tests](../../internal/validate/standard_validator_test.go) own
|
||||
basic, JSON, JSON Schema, source resolution, schema loading, compilation,
|
||||
|
||||
@@ -1,60 +1,53 @@
|
||||
# Open Source Attribution Notices
|
||||
|
||||
These entries identify upstream material used by Promptkit. The
|
||||
[OSS policy](../policy/oss.md) owns the format and maintenance rules; the
|
||||
[project grant](../../COPYRIGHT) does not replace third-party terms.
|
||||
## Standard Repository Policies
|
||||
|
||||
The module entries cover the five external modules used by `go list -deps -test
|
||||
./...` at adoption, including the transitive `golang.org/x/text` dependency.
|
||||
Other modules present only in dependency module graphs are not thereby
|
||||
incorporated into Promptkit. No separately copied third-party code or asset
|
||||
notices were identified in Promptkit's source; adapted policy documents are
|
||||
attributed separately below. Standard license texts in `licenses/` are shared
|
||||
reference texts, not additional project grants.
|
||||
Material: policy documents under `docs/policy/` and the Policy Reading Guide in
|
||||
`docs/development.md`, adapted for this repository; root `AGENTS.md`, copied
|
||||
without modification from upstream `agents/AGENTS.md`.
|
||||
|
||||
## Promptkit OpenRouter Catalog
|
||||
|
||||
Selected module: `gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v2`
|
||||
at `v2.20260926.0`.
|
||||
The release policy uses upstream commit
|
||||
`e833faddf120c789a9b45348edaf75720de16348`; the remaining material uses
|
||||
`d84a38ebab26f048cafbe59a2246700c35d29843`.
|
||||
|
||||
Copyright (C) 2026 Eric Rakestraw
|
||||
|
||||
SPDX-License-Identifier: MIT
|
||||
SPDX-License-Identifier: CC-BY-SA-4.0
|
||||
|
||||
License and notices: [MIT](licenses/MIT.txt) |
|
||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit-backend-openrouter/src/commit/4f731d217e0bc39151c010f50fb5912706e7b6b3/COPYRIGHT) |
|
||||
[upstream LICENSE](https://gitea.maximumdirect.net/eric/promptkit-backend-openrouter/src/commit/4f731d217e0bc39151c010f50fb5912706e7b6b3/LICENSE).
|
||||
License and notices: [CC BY-SA 4.0](licenses/CC-BY-SA-4.0.txt) |
|
||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/COPYRIGHT) |
|
||||
[LICENSE](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/LICENSE) |
|
||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/policy/src/commit/e833faddf120c789a9b45348edaf75720de16348/COPYRIGHT) |
|
||||
[LICENSE](https://gitea.maximumdirect.net/eric/policy/src/commit/e833faddf120c789a9b45348edaf75720de16348/LICENSE).
|
||||
|
||||
Source: [OpenRouter catalog at the imported commit](https://gitea.maximumdirect.net/eric/promptkit-backend-openrouter/src/commit/4f731d217e0bc39151c010f50fb5912706e7b6b3).
|
||||
[Source](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843).
|
||||
|
||||
## Promptkit Rakestrawhome Catalog
|
||||
[Source](https://gitea.maximumdirect.net/eric/policy/src/commit/e833faddf120c789a9b45348edaf75720de16348).
|
||||
|
||||
Selected module: `gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v2`
|
||||
at `v2.20260926.0`.
|
||||
## ConfigKit
|
||||
|
||||
Selected module: `gitea.maximumdirect.net/eric/ConfigKit` at `v0.8.0`.
|
||||
|
||||
Copyright (C) 2026 Eric Rakestraw
|
||||
|
||||
SPDX-License-Identifier: MIT
|
||||
SPDX-License-Identifier: LGPL-3.0-or-later
|
||||
|
||||
License and notices: [MIT](licenses/MIT.txt) |
|
||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/src/commit/48918a3ad39fd99843cda94a86133b1b3881d596/COPYRIGHT) |
|
||||
[upstream LICENSE](https://gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/src/commit/48918a3ad39fd99843cda94a86133b1b3881d596/LICENSE).
|
||||
License and notices: [LGPL 3.0](../../LICENSE) |
|
||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/ConfigKit/src/commit/137769f3f18a370c68444242fb5d88e620e307e5/COPYRIGHT) |
|
||||
[LICENSE](https://gitea.maximumdirect.net/eric/ConfigKit/src/commit/137769f3f18a370c68444242fb5d88e620e307e5/LICENSE).
|
||||
|
||||
Source: [Rakestrawhome catalog at the imported commit](https://gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/src/commit/48918a3ad39fd99843cda94a86133b1b3881d596).
|
||||
[Source](https://gitea.maximumdirect.net/eric/ConfigKit/src/commit/137769f3f18a370c68444242fb5d88e620e307e5).
|
||||
|
||||
## jsonschema
|
||||
|
||||
Selected module: `github.com/santhosh-tekuri/jsonschema/v6` at `v6.0.2`.
|
||||
|
||||
No project-specific copyright statement was supplied in the selected source's
|
||||
LICENSE, README, or Go source files; none is invented here.
|
||||
|
||||
SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
License and notices: [Apache 2.0](licenses/Apache-2.0.txt) |
|
||||
[upstream LICENSE](https://github.com/santhosh-tekuri/jsonschema/blob/29cbed948d24a04700eb94436416b18a07953b71/LICENSE).
|
||||
[LICENSE](https://github.com/santhosh-tekuri/jsonschema/blob/29cbed948d24a04700eb94436416b18a07953b71/LICENSE).
|
||||
|
||||
Source: [jsonschema at the imported commit](https://github.com/santhosh-tekuri/jsonschema/tree/29cbed948d24a04700eb94436416b18a07953b71).
|
||||
[Source](https://github.com/santhosh-tekuri/jsonschema/tree/29cbed948d24a04700eb94436416b18a07953b71).
|
||||
|
||||
## Go Text
|
||||
|
||||
@@ -62,20 +55,17 @@ Selected module: `golang.org/x/text` at `v0.14.0` (transitive dependency).
|
||||
|
||||
Copyright (c) 2009 The Go Authors. All rights reserved.
|
||||
|
||||
Individual source files retain their additional year-specific Go Authors
|
||||
notices in the linked source tree.
|
||||
|
||||
SPDX-License-Identifier: BSD-3-Clause
|
||||
|
||||
License and notices: [BSD 3-Clause](licenses/BSD-3-Clause.txt) |
|
||||
[upstream LICENSE](https://github.com/golang/text/blob/6c97a165dd661335ff7bce6104a008558123c353/LICENSE) |
|
||||
[additional patent grant](https://github.com/golang/text/blob/6c97a165dd661335ff7bce6104a008558123c353/PATENTS).
|
||||
[LICENSE](https://github.com/golang/text/blob/6c97a165dd661335ff7bce6104a008558123c353/LICENSE) |
|
||||
[PATENTS](https://github.com/golang/text/blob/6c97a165dd661335ff7bce6104a008558123c353/PATENTS).
|
||||
|
||||
Source: [Go Text at the imported commit](https://github.com/golang/text/tree/6c97a165dd661335ff7bce6104a008558123c353).
|
||||
[Source](https://github.com/golang/text/tree/6c97a165dd661335ff7bce6104a008558123c353).
|
||||
|
||||
## Go YAML v3
|
||||
|
||||
Selected module: `gopkg.in/yaml.v3` at `v3.0.1`.
|
||||
Selected module: `gopkg.in/yaml.v3` at `v3.0.1` (transitive dependency).
|
||||
|
||||
Copyright (c) 2006-2010 Kirill Simonov
|
||||
|
||||
@@ -87,32 +77,28 @@ Copyright 2011-2016 Canonical Ltd.
|
||||
|
||||
SPDX-License-Identifier: MIT AND Apache-2.0
|
||||
|
||||
The licenses apply to different files, not an either/or choice. The upstream
|
||||
LICENSE identifies the libyaml-derived MIT files and the Apache-licensed
|
||||
remainder; NOTICE supplies the final copyright statement above.
|
||||
License and notices: [MIT](licenses/MIT.txt) |
|
||||
[Apache 2.0](licenses/Apache-2.0.txt) |
|
||||
[LICENSE](https://github.com/go-yaml/yaml/blob/f6f7691b1fdeb513f56608cd2c32c51f8194bf51/LICENSE) |
|
||||
[NOTICE](https://github.com/go-yaml/yaml/blob/f6f7691b1fdeb513f56608cd2c32c51f8194bf51/NOTICE).
|
||||
|
||||
[Source](https://github.com/go-yaml/yaml/tree/f6f7691b1fdeb513f56608cd2c32c51f8194bf51).
|
||||
|
||||
## Go YAML v4
|
||||
|
||||
Selected module: `go.yaml.in/yaml/v4` at `v4.0.0-rc.6` (transitive dependency).
|
||||
|
||||
Copyright 2006-2010 Kirill Simonov
|
||||
|
||||
Copyright 2011-2019 Canonical Ltd
|
||||
|
||||
Copyright 2025 The go-yaml Project Contributors
|
||||
|
||||
SPDX-License-Identifier: MIT AND Apache-2.0
|
||||
|
||||
License and notices: [MIT](licenses/MIT.txt) |
|
||||
[Apache 2.0](licenses/Apache-2.0.txt) |
|
||||
[upstream LICENSE and file scope](https://github.com/go-yaml/yaml/blob/f6f7691b1fdeb513f56608cd2c32c51f8194bf51/LICENSE) |
|
||||
[NOTICE](https://github.com/go-yaml/yaml/blob/f6f7691b1fdeb513f56608cd2c32c51f8194bf51/NOTICE).
|
||||
[LICENSE](https://github.com/yaml/go-yaml/blob/1c17b9cee72bf81ae73a8be50fbd7dd3a9985125/LICENSE) |
|
||||
[NOTICE](https://github.com/yaml/go-yaml/blob/1c17b9cee72bf81ae73a8be50fbd7dd3a9985125/NOTICE).
|
||||
|
||||
Source: [Go YAML at the imported commit](https://github.com/go-yaml/yaml/tree/f6f7691b1fdeb513f56608cd2c32c51f8194bf51).
|
||||
|
||||
## Standard Repository Policies
|
||||
|
||||
Material: adapted standard policy documents under `docs/policy/`, with
|
||||
Promptkit-specific ownership, library boundaries, contributor guidance, and
|
||||
release/OSS conventions. These documents have been modified from the templates.
|
||||
|
||||
No project-specific copyright statement is supplied by the standard policy
|
||||
repository's README or LICENSE; none is invented here.
|
||||
|
||||
SPDX-License-Identifier: CC-BY-SA-4.0
|
||||
|
||||
The adapted policy documentation retains these upstream terms; this does not
|
||||
change Promptkit's library grant.
|
||||
|
||||
License and notices: [CC BY-SA 4.0](licenses/CC-BY-SA-4.0.txt).
|
||||
|
||||
Source: maintainer-provided internal standard policy templates. The source
|
||||
repository is non-public; no public source or notice URL is available.
|
||||
[Source](https://github.com/yaml/go-yaml/tree/1c17b9cee72bf81ae73a8be50fbd7dd3a9985125).
|
||||
|
||||
@@ -1,19 +1,20 @@
|
||||
# Agent Policy
|
||||
|
||||
**Base Policy Version:** 1.1
|
||||
**Base Policy Version:** 1.3
|
||||
|
||||
**Last Updated:** 2026-09-23
|
||||
**Last Updated**: 2026-09-27
|
||||
|
||||
**Upstream Source:** [agents.md](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/docs/policy/agents.md)
|
||||
|
||||
## Purpose
|
||||
|
||||
Agents must follow these house rules before recommending, planning, or
|
||||
implementing repository changes.
|
||||
Agents must read and follow these rules.
|
||||
|
||||
## Implementation Discipline
|
||||
|
||||
Write the simplest idiomatic code that satisfies the user's request.
|
||||
Simplify the mechanism, not the user's choices. Do not implicitly strengthen
|
||||
requirements, restrictions, or guarantees.
|
||||
**KEY PRINCIPLES:** Write the simplest idiomatic code that satisfies the user's
|
||||
request. Do not implicitly strengthen requirements, restrictions, or guarantees
|
||||
beyond what the user actually requested.
|
||||
|
||||
**Keep implementation machinery proportionate.** Use straightforward,
|
||||
idiomatic code and introduce only the machinery needed to support the requested
|
||||
@@ -28,41 +29,48 @@ operator and caller choice where policy has not been established. If an
|
||||
additional restriction appears advisable, present it as a decision rather than
|
||||
silently implementing it.
|
||||
|
||||
**Do not invent guarantees.** Do not strengthen public contracts, guarantees,
|
||||
or invariants beyond those expressly requested by the user.
|
||||
|
||||
**Use the standard library.** For example: in almost every case, the best
|
||||
approach to copying files is to use the standard library. Writing custom code to
|
||||
implement power-loss protection at the application level, or to guard against
|
||||
bad actors with privileged and concurrent access to the host's filesystem,
|
||||
is wasteful, unnecessary, and not idiomatic. Stick to the simple, standard, and
|
||||
idiomatic approach unless another path is expressly requested by the user.
|
||||
|
||||
**An implementation detail is not repository policy.** Tests and documentation
|
||||
must not elevate incidental behavior into a durable guarantee or restriction
|
||||
unless that behavior was expressly selected as part of the contract.
|
||||
|
||||
## Promptkit Contract Authority
|
||||
|
||||
The following clarification preserves Promptkit's established distinction
|
||||
between implementation choices and authorized contracts.
|
||||
|
||||
A correctness constraint must follow from the requested behavior or an
|
||||
established invariant, not from the convenience of the chosen implementation.
|
||||
Explain the specific failure it prevents and why a less restrictive
|
||||
implementation cannot satisfy the contract.
|
||||
|
||||
**An implementation detail is not repository policy.** Tests and documentation
|
||||
must not elevate incidental behavior into a durable guarantee or restriction
|
||||
unless that behavior was expressly selected as part of the contract. Existing
|
||||
explicit contracts remain authoritative; changing or weakening one requires a
|
||||
deliberate decision rather than being inferred from this principle.
|
||||
|
||||
Agent-authored plans, tests, documentation, and prior implementations are not
|
||||
evidence that an additional restriction was deliberately approved. Identify the
|
||||
authorizing requirement or decision; when that authority is unclear, surface
|
||||
the ambiguity rather than treating repetition as approval.
|
||||
|
||||
**Do not invent guarantees.** Do not strengthen public contracts, guarantees,
|
||||
or invariants beyond those expressly requested by the user.
|
||||
|
||||
**Use the standard library.** Prefer simple, idiomatic standard-library
|
||||
implementations when they satisfy the requested behavior. Do not add custom
|
||||
power-loss protection or defenses against privileged concurrent filesystem
|
||||
actors unless expressly required. Preserve existing repository contracts.
|
||||
Existing explicit contracts remain authoritative; changing or weakening one
|
||||
requires a deliberate decision. Agent-authored plans, tests, documentation,
|
||||
and prior implementations are not evidence that an additional restriction was
|
||||
deliberately approved. Identify the authorizing requirement or decision; when
|
||||
that authority is unclear, surface the ambiguity rather than treating repetition
|
||||
as approval.
|
||||
|
||||
## Git Commit Policy
|
||||
|
||||
By default, create a commit after completing each coherent unit of work, using a
|
||||
By default, commit each completed, nonempty unit of work using a
|
||||
concise, plain-English message describing what changed. This policy is standing
|
||||
authorization to create those local commits; a separate user request is not
|
||||
required. An explicit user instruction to leave work uncommitted takes precedence.
|
||||
|
||||
A unit of work is a focused, reviewable outcome, including its relevant tests and
|
||||
documentation. Do not commit after every intermediate edit or create empty commits.
|
||||
documentation.
|
||||
Complete the appropriate validation before committing. If work is incomplete or
|
||||
validation is blocked, report that state rather than presenting it as completed.
|
||||
validation is blocked, report that state accurately.
|
||||
|
||||
Before committing:
|
||||
|
||||
@@ -70,10 +78,10 @@ Before committing:
|
||||
changes. Identify changes that predate the current unit of work.
|
||||
2. Exclude secrets, local environment files, generated junk, and unrelated changes.
|
||||
3. Stage only the files or hunks belonging to the completed unit. Use `git add .`
|
||||
only when every included change belongs to that unit. Do not disturb unrelated
|
||||
staged or unstaged work.
|
||||
4. Review `git diff --staged` and run appropriate checks for the change. Do not
|
||||
repeat checks already passed for unchanged content merely to create a commit.
|
||||
only when every included change belongs to that unit. Preserve unrelated
|
||||
staged and unstaged work.
|
||||
4. Review `git diff --staged` and run appropriate checks for the change. Reuse
|
||||
successful checks for unchanged content.
|
||||
5. Review `git log --oneline -n 10` and use
|
||||
`git commit -m "<concise plain-English commit message>"`.
|
||||
6. Report the commit identifier and any remaining uncommitted work.
|
||||
|
||||
@@ -1,15 +1,41 @@
|
||||
# Architecture Policy
|
||||
|
||||
**Base Policy Version:** 1.0
|
||||
**Base Policy Version:** 1.3
|
||||
|
||||
**Last Updated:** 2026-09-23
|
||||
**Last Updated**: 2026-09-27
|
||||
|
||||
## Purpose
|
||||
**Upstream Source:** [architecture.md](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/docs/policy/architecture.md)
|
||||
|
||||
This document defines Promptkit's current high-level architecture and the
|
||||
durable boundaries that implementation changes must preserve. The
|
||||
[internal component overview](../internal/overview.md) inventories concrete
|
||||
implemented packages without redefining these rules.
|
||||
## Purpose And Ownership
|
||||
|
||||
The adopted policy defines Promptkit's purpose and the architectural
|
||||
constraints contributors must follow. It owns system boundaries, responsibility
|
||||
and dependency rules, established invariants, and explicit non-goals.
|
||||
|
||||
Describe the concrete package inventory and implementation mechanics in internal
|
||||
documentation. Link to canonical public contracts for their exact behavior and
|
||||
to ADRs for decision rationale.
|
||||
|
||||
## Architectural Approach
|
||||
|
||||
Hexagonal architecture is strongly recommended, including encapsulating external
|
||||
integrations and APIs within internal adapters. Keep domain and application
|
||||
logic independent of external integration details, and make dependency direction
|
||||
clear in the adopted policy.
|
||||
|
||||
Implement that separation with proportionate, idiomatic code. Choose interfaces
|
||||
and package layout to serve the application's actual boundaries. Use adapters
|
||||
for collaborators with external integration responsibilities.
|
||||
|
||||
## Adopted Constraints And Implemented Structure
|
||||
|
||||
Architectural constraints govern current and future development. Record
|
||||
implemented structure accurately, proposals and implementation status in the
|
||||
roadmap, and significant decisions and rationale in ADRs. Describe a target
|
||||
structure as implemented when its changes land.
|
||||
|
||||
Constraints must reflect deliberately selected requirements and decisions.
|
||||
Do not promote incidental implementation choices into architectural invariants.
|
||||
|
||||
## System Shape
|
||||
|
||||
@@ -25,23 +51,22 @@ The [internal component overview](../internal/overview.md) owns the inventory
|
||||
of implemented packages and their concrete responsibilities. This policy owns
|
||||
the boundaries and dependency rules those components must preserve.
|
||||
|
||||
The internal engine assembles one immutable backend registry, one capacity manager,
|
||||
the internal repositories, renderer, validator, outbound client, and use-case
|
||||
runner. The registry contains validated maintained definitions plus engine-scoped
|
||||
consumer additions. The root facade translates public values and errors at the
|
||||
library boundary.
|
||||
The internal engine constructs the capacity manager from the registry's immutable
|
||||
policy snapshot, wraps the selected built-in or injected model client, and
|
||||
supplies bounded admission to the runner. The defaults and renderer depend on
|
||||
the domain model. Prompt-definition and profile repositories use the domain
|
||||
model, file catalog, and YAML decoder. The catalog adapter validates imported
|
||||
immutable maintained data, including its reserved namespace, before internal
|
||||
engine assembly supplies it to the profile package. Artifact reading uses the
|
||||
domain model and application-neutral defaults. Validation uses the domain model,
|
||||
file catalog, and JSON Schema implementation. The model client uses the domain
|
||||
model and application-neutral defaults, and an injected or standard-library HTTP
|
||||
client. The use-case runner depends on the narrow interfaces owned by each
|
||||
internal component, including backend lookup and run admission.
|
||||
The internal engine composes caller-selected ConfigKit inputs once, then
|
||||
assembles the captured definitions, capacity manager, renderer, validator,
|
||||
outbound client, and use-case runner. The configuration adapter interprets and
|
||||
validates only the final effective values, including presence, null, and
|
||||
resource context. It retains source history for explicit operator inspection,
|
||||
not for validation or a second resolution pass. The root facade translates
|
||||
public values and errors at the library boundary.
|
||||
|
||||
The engine constructs capacity policy from final backend definitions, wraps
|
||||
the selected built-in or injected model client, and supplies bounded admission
|
||||
to the runner. Runtime/domain components consume execution-owned values, not
|
||||
ConfigKit types. Artifact reading and validation use the domain model and
|
||||
application-neutral defaults; validation also uses file-catalog helpers and
|
||||
JSON Schema. The model client uses the domain model, defaults, and an injected
|
||||
or standard-library HTTP client. The runner coordinates narrow lookup and
|
||||
collaborator interfaces without rereading configuration.
|
||||
|
||||
The current implementation follows this dependency direction:
|
||||
|
||||
@@ -58,9 +83,10 @@ downstream consumers, including Scriptorium
|
||||
narrow injected abstractions
|
||||
```
|
||||
|
||||
The backend registry depends on the domain model and shared JSON-value
|
||||
validation, has no mutation API after construction, and consumes the
|
||||
OpenAI-compatible reserved request-field rule owned by the model client. The
|
||||
The backend index depends on the domain model and shared JSON-value ownership
|
||||
and has no mutation API after construction. The configuration adapter applies
|
||||
the OpenAI-compatible reserved request-field rule owned by the model client
|
||||
when it validates final provider maps. The
|
||||
capacity component depends on the domain model and the narrow internal
|
||||
model-client boundary, not on provider transport implementation. The model
|
||||
client does not depend on registry or capacity configuration. The facade
|
||||
@@ -71,7 +97,8 @@ on consumers or on Scriptorium.
|
||||
`internal/domain` owns source-neutral invariants for values shared across
|
||||
multiple input and execution boundaries, including execution-setting bounds,
|
||||
OpenAI-compatible base endpoints, session identifiers, and output-contract
|
||||
legality. Callers retain source parsing, required-field rules, other
|
||||
legality, plus shared resource-identity syntax and qualification. Callers retain
|
||||
source parsing, required-field rules, other
|
||||
source-specific normalization, defaulting, error classification, and policy
|
||||
specific to their own boundary.
|
||||
|
||||
@@ -155,3 +182,21 @@ state.
|
||||
Do not list planned packages as implemented components. When implementation
|
||||
introduces a package, update the internal inventory and the owning contract or
|
||||
subsystem document in the same change.
|
||||
|
||||
## Trust Boundaries And Threat Scope
|
||||
|
||||
Where useful, state the trust boundaries and threat scenarios relevant to the
|
||||
application's adopted requirements, along with scenarios it explicitly excludes.
|
||||
For example, distinguish handling untrusted application input from defending
|
||||
against a privileged actor concurrently modifying the host filesystem.
|
||||
|
||||
Threat exclusions define the scope of the adopted requirements. Base defenses
|
||||
and guarantees on those requirements and raise unsettled scope for a deliberate
|
||||
decision under the [agent policy](agents.md).
|
||||
|
||||
Keep this discussion proportionate to the application. A separate threat
|
||||
assessment or security document is optional when adopting this template.
|
||||
For this library, the framework format and integration documents own credential
|
||||
supply and external contracts; internal documents own implementation mechanisms.
|
||||
Consumer-specific filesystem and security policy remains outside Promptkit's
|
||||
boundary as described above.
|
||||
|
||||
@@ -1,157 +1,139 @@
|
||||
# Documentation Policy
|
||||
|
||||
**Base Policy Version:** 1.5
|
||||
**Base Policy Version:** 1.11
|
||||
|
||||
**Last Updated:** 2026-09-26
|
||||
**Last Updated**: 2026-09-27
|
||||
|
||||
**Upstream Source:** [documentation.md](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/docs/policy/documentation.md)
|
||||
|
||||
## Purpose
|
||||
|
||||
This policy assigns each Promptkit documentation topic to one canonical owner.
|
||||
Its goal is to keep documentation for this reusable Go library accurate,
|
||||
concise, discoverable, and resistant to drift for consumers, contributors,
|
||||
maintainers, integrators, and coding agents.
|
||||
This policy assigns each documentation topic to one canonical owner. Its goal is
|
||||
to keep this library's documentation accurate, concise, discoverable, and resistant
|
||||
to drift for users, operators, developers, integrators, and LLM coding agents.
|
||||
|
||||
## Core Rules
|
||||
|
||||
### One Canonical Owner
|
||||
|
||||
Each authoritative fact belongs in one document or source form. A non-owning
|
||||
document may give a short, stable summary for orientation, but it must link to
|
||||
the canonical owner instead of repeating exact contracts.
|
||||
Each authoritative fact belongs in one document. A non-owning document may give
|
||||
a short, stable summary for orientation, but it must link to the canonical owner
|
||||
for volatile details.
|
||||
|
||||
Volatile details include exported declarations, accepted inputs, defaults,
|
||||
schemas, file names, paths, error identities, retry behavior, and runtime
|
||||
guarantees. If readers could reasonably treat a statement as a contract,
|
||||
maintain its exact definition only in the owning source.
|
||||
Volatile details include commands, flags, configuration fields and defaults,
|
||||
module keys, schemas, file names, paths, status codes, retry behavior, and
|
||||
runtime guarantees. If readers could reasonably treat a statement as a
|
||||
contract, maintain it only in the owning document.
|
||||
|
||||
### Current State, Decisions, And Future Work
|
||||
### Current And Future Behavior
|
||||
|
||||
Outside `docs/roadmap/`, documentation describes implemented behavior only.
|
||||
Partial features may be described only to their implemented boundary.
|
||||
|
||||
An accepted architecture decision may describe an approved direction before it
|
||||
is implemented, but acceptance is not evidence that the behavior exists.
|
||||
Current-state documents change when the implementation lands. Temporary
|
||||
roadmaps own future work, sequencing, and implementation status; they do not
|
||||
replace durable policies or current contracts.
|
||||
Architectural constraints may govern future changes. Describe current structure
|
||||
from implemented behavior.
|
||||
|
||||
ADRs may record an accepted architectural decision before implementation. The
|
||||
roadmap owns implementation status and sequencing until the decision is
|
||||
implemented. Current architecture, user, operator, integration, and internal
|
||||
documentation are updated when the behavior lands.
|
||||
|
||||
### Audience And Detail
|
||||
|
||||
Write for the document's stated audience and include only the detail needed for
|
||||
its owned topic. Consumer guidance should not expose incidental implementation
|
||||
detail. Contributor documentation should link to public contracts and durable
|
||||
policies instead of restating them.
|
||||
its owned topic. Keep user and operator docs focused on using and operating the
|
||||
application. Developer docs should link to user-facing and external contracts
|
||||
for their definitions.
|
||||
|
||||
### Links
|
||||
### Examples
|
||||
|
||||
Use descriptive link text and repository-relative links for repository
|
||||
documents. Link to the canonical owner rather than to a duplicate summary.
|
||||
Check every added or changed link and repair or remove links when their target
|
||||
moves or is retired.
|
||||
Complete copyable files belong in `examples/`. Documentation may use the
|
||||
smallest illustrative snippet needed to explain its owned topic, but should link
|
||||
to maintained examples for complete copyable files.
|
||||
|
||||
### Examples And Code Fences
|
||||
|
||||
Complete copyable files belong in `examples/` when maintained examples exist.
|
||||
Documentation may use the smallest illustrative snippet needed for its owned
|
||||
topic, but should link to a maintained example instead of embedding a second
|
||||
complete copy.
|
||||
|
||||
Examples must be valid, secret-free, and tested where practical. Commands,
|
||||
imports, and Go snippets must match the implemented library. Use a language tag
|
||||
on fenced code blocks and make clear when a fragment is illustrative rather
|
||||
than directly runnable.
|
||||
Examples must be valid, secret-free, and tested where practical. Commands and
|
||||
configuration used in documentation should match the application.
|
||||
|
||||
### Security And Privacy
|
||||
|
||||
Documentation and examples must not contain real credentials, private keys,
|
||||
private environment dumps, sensitive source material, or private
|
||||
infrastructure details unless intentionally public. Document secret-handling
|
||||
mechanisms, not secret values.
|
||||
private environment dumps, sensitive source material, or private infrastructure
|
||||
details unless intentionally public. Document secret-handling mechanisms, not
|
||||
secret values.
|
||||
|
||||
## Canonical Ownership
|
||||
|
||||
| Topic | Canonical owner | Owned content | Content owned elsewhere |
|
||||
| --- | --- | --- | --- |
|
||||
| Project orientation | `README.md` | What Promptkit is, its current usability, module identity, license summary, a minimal successful quickstart, and links onward. | Exact API contracts, contributor procedures, architecture detail, and release steps. |
|
||||
| Contributor workflow | `docs/development.md` | Task-oriented reading guide, local workflow, validation commands, and repository hygiene. | Architecture rules, API semantics, subsystem behavior, and release procedure. |
|
||||
| Current architecture | `docs/policy/architecture.md` | System shape, normative ownership, dependency direction, package boundaries, invariants, and non-goals. | Concrete component inventory, implementation mechanics, contributor procedures, decision history, and future work. |
|
||||
| Agent policy | `docs/policy/agents.md` | Agent implementation discipline, proportionality, idiomatic simplicity, local commit authorization and discipline, and rules against inventing restrictions, guarantees, or repository policy. | Public contracts, architecture, testing, and contributor procedures. |
|
||||
| Documentation organization | `docs/policy/documentation.md` | Documentation ownership, audience boundaries, maintenance rules, and document lifecycle. | Library architecture or runtime behavior. |
|
||||
| Testing policy | `docs/policy/testing.md` | Test philosophy, risk-based sufficiency, test boundaries, doubles, coverage guidance, regression policy, and test maintenance. | Subsystem behavior, exact public contracts, subsystem-specific test inventories, and implementation plans. |
|
||||
| Release policy | `docs/policy/release.md` | Versioning, compatibility, release-note requirements, validation obligations, immutability, recovery, and adoption scope. | Executable commands, publication mechanics, and historical release summaries. |
|
||||
| Licensing and attribution policy | `docs/policy/oss.md` | Project licensing conventions, attribution format, text and notice-link rules, discoverability, maintenance, and concrete exceptions. | Component entries, license texts, release commands, and dependency security review. |
|
||||
| Project grant | `COPYRIGHT` | Project identity, holders and years, SPDX expression, grant, warranty disclaimer, and third-party statement. | Complete license texts and component attribution. |
|
||||
| Project license texts | `LICENSE` | Complete project license and incorporated companion texts. | General licensing policy and dependency attribution. |
|
||||
| Upstream attribution | `docs/oss/attribution.md` | Component names, copyright statements, license identification, and license, notice, and source links. | General policy and complete standard texts. |
|
||||
| Standard dependency license texts | `docs/oss/licenses/` | Complete, verbatim texts shared by attribution entries. | Project-specific notices, attribution prose, and policy interpretation. |
|
||||
| Release procedure | `docs/release.md` | Maintainer prerequisites, candidate preparation, concrete publication guards and commands, verification, and recovery steps. | Release-policy requirements, public API semantics, and historical summaries. |
|
||||
| Product orientation and minimal end-to-end quickstart | `README.md` | Purpose, value, shortest successful invocation, and navigation, including licensing and attribution. | Complete command reference, configuration reference, operational procedures, implementation detail. |
|
||||
| Contributor entry point | `docs/development.md` | Task-oriented reading guide, contributor orientation, baseline validation commands, and canonical links. | Package inventory, architecture rules, subsystem behavior, detailed change recipes. |
|
||||
| Application architecture policy | `docs/policy/architecture.md` | System shape, normative ownership, dependency direction, boundaries, invariants, safety properties, and non-goals. | Concrete package inventory, implementation mechanics, contributor procedures, decision history, future work. |
|
||||
| Policy-set provenance and upstream navigation | `docs/policy/meta.md` | Policy-set origin, local authority, adoption baselines, and upstream adoption and update links. | Task-specific reading triggers, detailed adoption procedures, and formal attribution notices. |
|
||||
| Agent policy | `docs/policy/agents.md` | Proportionate, idiomatic implementation; rules against inventing requirements, restrictions, guarantees, or policy. | Application contracts, architecture, testing policy, and contributor procedures. |
|
||||
| Documentation policy | `docs/policy/documentation.md` | Documentation ownership, audience boundaries, maintenance rules, and ADR/document lifecycle. | Application architecture or product behavior. |
|
||||
| Testing policy | `docs/policy/testing.md` | Test philosophy, risk sufficiency, boundaries, doubles, coverage, regressions, and test maintenance. | Subsystem behavior, application contracts, subsystem-specific test inventories, and implementation plans. |
|
||||
| Release policy | `docs/policy/release.md` | Versioning, compatibility, release integrity and immutability, release documentation, proportionate gates, publication controls, completion, and required versus optional checks. | Repository-specific commands, product installation guidance, CLI version semantics, historical release summaries, and CI implementation detail. |
|
||||
| Open source licensing and attribution policy | `docs/policy/oss.md` | License defaults and author choices, COPYRIGHT and attribution formats, license/notice links, discoverability, maintenance, and concrete exceptions. | Component entries, license texts, release commands, and dependency security review. |
|
||||
| Project copyright and license grant | `COPYRIGHT` | Project identity, copyright holders and years, SPDX expression, grant, warranty disclaimer, and third-party statement. | Complete license texts and upstream component attribution. |
|
||||
| Project license texts | `LICENSE` | Complete project and incorporated companion license texts, with required project-specific fields. | General licensing policy and upstream component attribution. |
|
||||
| Upstream component attribution | `docs/oss/attribution.md` | House Format component entries: identity, material scope and modifications where applicable, copyright, license, required notices, and license, notice, and source links. | General attribution policy, exception reasoning, research narratives, compliance commentary, and complete standard license texts. |
|
||||
| Standard dependency license texts | `docs/oss/licenses/` | Complete, verbatim standard license texts shared by applicable entries. | Project-specific upstream notices, attribution prose, and policy interpretation. |
|
||||
| Repository-specific release procedure | `docs/release.md` | Maintainer prerequisites, candidate preparation, validation and publication commands and automated guards, asynchronous deliverables, delivery status, optional consumer verification, and recovery. | General release requirements, product installation guidance, historical release summaries, and current compatibility contracts. |
|
||||
| Consumer guidance for the public Go API (if applicable) | `docs/consumers/` | Task-oriented public Go API usage, minimal examples, and consumer responsibilities. | HTTP wire semantics, external protocol contracts, internal implementation detail. |
|
||||
| Integration contracts for external components (if applicable) | `docs/integrations/` | External formats and protocols, upstream and downstream contracts, logical output paths and schemas, media types, and compatibility. | Physical runtime placement and lifecycle, internal transformations, CLI syntax, configuration defaults. |
|
||||
| Inventory of implemented internal packages | `docs/internal/overview.md` | Implemented packages and components, responsibilities, and links to focused internal docs. | Normative architecture, contributor reading policy, external contracts. |
|
||||
| Behavior and architecture of internal packages | Other files under `docs/internal/` | Implementation flow, collaborators, state transitions, package-local guarantees and failures, and relevant tests. | Global architecture invariants, configuration definitions and defaults, external schemas, operator procedures. |
|
||||
| Architectural decision records | `docs/adr/` | Significant decisions, context, alternatives, rationale, consequences, and supersession history. | Current behavior reference, implementation status, task sequencing. |
|
||||
| Future work and implementation status | `docs/roadmap/` | Proposed, accepted, deferred, or rejected work; implementation status, sequencing, and task breakdowns. | Implemented behavior reference and architectural decision rationale. |
|
||||
| Release-note history | `docs/releases/` | One checked-in summary per source release under the procedure; the note at its immutable tag is the release record. | Current commands, behavior, contracts, and compatibility definitions. |
|
||||
| Complete copyable artifacts | `examples/` | Maintained configuration, inputs, and other copyable or runnable files. | Field-by-field reference, command reference, prose explanation. |
|
||||
| Exact exported Go API | Go declarations and GoDoc, as APIs are implemented | Exported names, signatures, types, values, errors, and exact behavioral contracts. | Task-oriented consumer walkthroughs, implementation details, and future API proposals. |
|
||||
| Framework file formats | `docs/formats.md` | Prompt-definition and profile YAML fields, schema references, defaults, validation modes, built-in profiles, credentials, and file-to-request precedence. | Exported Go declarations, outbound wire behavior, internal parsing mechanics, and application configuration. |
|
||||
| Consumer guidance | `docs/consumers/`, when consumer workflows require dedicated guidance | Task-oriented use of implemented public APIs, minimal examples, and consumer responsibilities. | Exact exported declarations and internal mechanics. |
|
||||
| Durable integration contracts | `docs/integrations/`, when integrations exist | External formats and protocols, compatibility behavior, and upstream or downstream responsibilities. | Internal transformations and public Go declarations. |
|
||||
| Release-note history | `docs/releases/` | Mandatory version-matched historical summaries for releases made under the adopted release policy; the note at the immutable tag records that release. | Current APIs, behavior, formats, compatibility contracts, and release procedures. |
|
||||
| Implemented component inventory | `docs/internal/overview.md` | Current packages and components, their implemented responsibilities, and links to focused internal documents. | Normative architecture, contributor workflow, external contracts, and proposed components. |
|
||||
| Internal subsystem behavior | Other files under `docs/internal/`, when a subsystem needs durable detail | Implementation flow, internal collaborators and state transitions, package-local guarantees and failures, and relevant tests. | Global architecture invariants, public API definitions, and future package plans. |
|
||||
| Architectural decision history | `docs/adr/`, when repository-local decisions require records | Significant decisions, context, alternatives, rationale, consequences, and supersession history. | Current behavior reference, implementation status, and task sequencing. |
|
||||
| Temporary feature roadmaps | `docs/roadmap/`, while planned work needs coordination | Proposed or accepted scope, sequencing, gates, and implementation status. | Implemented behavior reference and durable decision rationale. |
|
||||
| Complete copyable artifacts | `examples/` | Valid inputs, Go programs, and other files intended to be copied or run. | Field-by-field reference, exact API declarations, and prose explanation. |
|
||||
| Framework file formats | `docs/formats.md` | Prompt-definition and profile YAML fields, schema references, defaults, validation modes, credentials, and file-to-request precedence. | Exported Go declarations, outbound wire behavior, internal parsing mechanics, and application configuration. |
|
||||
|
||||
Conditional owners do not require placeholder files or directories. Create a
|
||||
consumer, integration, release, subsystem, ADR, roadmap, or example document
|
||||
only when the corresponding implemented interface, release, decision, planned
|
||||
effort, or maintained artifact exists.
|
||||
Create supporting documents for existing interfaces and responsibilities when
|
||||
substantive content is available.
|
||||
|
||||
## Boundary Rules
|
||||
|
||||
### Orientation
|
||||
|
||||
The README owns project orientation. The development guide routes
|
||||
contributors. Architecture owns normative structure. The internal overview
|
||||
owns the current concrete component map. These documents may link to one
|
||||
another but must not maintain parallel package or behavior descriptions.
|
||||
Link among orientation, architecture, and internal documents; maintain each
|
||||
package or behavior description in its canonical owner.
|
||||
|
||||
### Public Contracts And Implementation
|
||||
|
||||
Go declarations and GoDoc own exact exported API contracts once those APIs
|
||||
exist. Consumer and integration documents explain how to use those contracts
|
||||
for a task. Internal documents explain how Promptkit implements them. Internal
|
||||
documentation may identify a public type or external format as a dependency,
|
||||
but must link to its canonical definition rather than restate it.
|
||||
Go declarations and GoDoc own exact exported Go API contracts. The
|
||||
[framework format reference](../formats.md) owns prompt, profile, and schema
|
||||
formats; integration documents own external wire behavior. Consumer and
|
||||
internal documents link to those owners for exact definitions.
|
||||
|
||||
The [framework format reference](../formats.md) owns exact prompt, profile, and
|
||||
schema-file contracts. Integration documents own external wire formats.
|
||||
### Releases
|
||||
|
||||
### Release Documentation
|
||||
The release policy defines requirements and invariants that every release must
|
||||
satisfy. The repository-specific release procedure defines the commands and
|
||||
workflow maintainers use to satisfy that policy. Release notes record history
|
||||
at immutable tags. Maintain current behavior in its canonical documentation,
|
||||
update it with the corresponding behavior changes, and keep release procedures
|
||||
consistent with adopted policy.
|
||||
|
||||
The [release policy](release.md) owns release requirements; the
|
||||
[repository release procedure](../release.md) owns their concrete execution.
|
||||
Version-matched checked-in notes are required for future releases under that
|
||||
policy. The source tag is the release identity, and the note in its immutable
|
||||
tree is the historical summary; annotated tag messages complement that record.
|
||||
### Project Licensing And Upstream Attribution
|
||||
|
||||
Release notes do not own current API or behavior contracts. Link to the
|
||||
canonical owners for details instead of reproducing those contracts. Published
|
||||
notes are not disposable guidance and must not be replaced under the same
|
||||
version. Corrections belong in a subsequent release.
|
||||
README and release documents link to the canonical licensing and attribution
|
||||
materials under the [OSS policy](oss.md). The release procedure owns URL
|
||||
construction and publication mechanics.
|
||||
|
||||
Adoption is prospective. Previously published tags and historical notes retain
|
||||
their original records and are not reformatted or rewritten to satisfy the new
|
||||
requirements.
|
||||
### Contracts And Implementation
|
||||
|
||||
### Project Licensing And Attribution
|
||||
|
||||
The [OSS policy](oss.md) owns licensing conventions, the attribution House
|
||||
Format, and README/release navigation. COPYRIGHT owns the project grant;
|
||||
LICENSE supplies complete texts. Component-specific information belongs in
|
||||
attribution entries, not parallel README or release-note notice lists. The
|
||||
release procedure owns concrete footer URLs and publication mechanics.
|
||||
Integration and API documents define externally observable shapes and
|
||||
semantics. Internal documents explain how this application implements or consumes those
|
||||
contracts. Internal docs may name a field, file, or protocol to identify a
|
||||
dependency, but must link to its canonical contract for the definition.
|
||||
|
||||
### Security Topics
|
||||
|
||||
This policy owns what documentation and examples may contain. Architecture owns
|
||||
library security boundaries and invariants. Public declarations and integration
|
||||
documents own consumer-visible security contracts. Internal documents own
|
||||
implementation mechanisms only.
|
||||
library security boundaries. Public declarations and integration documents own
|
||||
consumer-visible contracts; internal documents own implementation mechanisms.
|
||||
|
||||
## Architecture Decision Records
|
||||
|
||||
@@ -166,31 +148,35 @@ Use sequentially numbered ADR filenames such as
|
||||
6. alternatives considered;
|
||||
7. consequences.
|
||||
|
||||
Use one of these statuses:
|
||||
Treat the decision content of an accepted ADR as immutable. When a decision
|
||||
changes, create a new ADR and update the earlier ADR's status to superseded.
|
||||
Rejected architectural alternatives belong in the ADR; rejected product ideas
|
||||
belong in the roadmap.
|
||||
|
||||
- **Proposed:** the decision is under consideration and may change;
|
||||
- **Accepted:** the decision is approved, whether or not implementation is
|
||||
complete;
|
||||
- **Rejected:** the proposed decision was considered and not adopted;
|
||||
- **Superseded:** a later accepted ADR replaces the accepted decision.
|
||||
## Maintenance
|
||||
|
||||
A proposed ADR transitions to Accepted or Rejected. An Accepted ADR transitions
|
||||
to Superseded only when a later Accepted ADR replaces it. An ADR may be created
|
||||
as Accepted when the decision has already been made.
|
||||
When behavior changes, update its canonical owner in the same change. If
|
||||
ownership moves, remove the old definition and replace it with a link where
|
||||
navigation remains useful.
|
||||
|
||||
Treat the decision content of an Accepted ADR as immutable. A changed decision
|
||||
requires a later ADR rather than a rewrite of the accepted record. A Superseded
|
||||
ADR must link to its replacement, and the replacement must link back. Rejected
|
||||
architectural alternatives belong in the ADR; rejected feature ideas belong in
|
||||
a roadmap when they need to be retained.
|
||||
Before completing documentation work:
|
||||
|
||||
- verify affected behavior and examples;
|
||||
- check commands, flags, fields, defaults, schemas, and paths against their
|
||||
implementation;
|
||||
- keep unimplemented behavior in the roadmap, subject to the ADR exception;
|
||||
- remove stale references and validate links;
|
||||
- confirm that non-owning documents summarize and link rather than redefine;
|
||||
- confirm that no secrets or sensitive private data were added.
|
||||
|
||||
## Promptkit Documentation Scope
|
||||
|
||||
Promptkit is a Go library, so exported GoDoc and framework formats replace
|
||||
application CLI, configuration, operations, and HTTP-service owners. Reading
|
||||
triggers belong in the development guide, not this topic map.
|
||||
|
||||
## Document Lifecycle
|
||||
|
||||
Create durable current-state documentation with the implementation it
|
||||
describes. Update its canonical owner in the same change when behavior changes.
|
||||
If ownership moves, remove the old definition and leave a link where navigation
|
||||
remains useful.
|
||||
|
||||
Roadmaps are temporary coordination documents. When their work is complete,
|
||||
record completion, move any still-useful decisions or contracts to their
|
||||
durable owners, update incoming links, and archive or remove the roadmap
|
||||
@@ -200,13 +186,9 @@ second current-state reference.
|
||||
Retain release notes as historical records under the release policy. Do not
|
||||
apply temporary-roadmap retirement rules to published release notes.
|
||||
|
||||
Before completing documentation work:
|
||||
|
||||
- verify affected behavior and examples;
|
||||
- check commands, imports, declarations, defaults, schemas, and paths against
|
||||
their implementation;
|
||||
- keep unimplemented behavior in a roadmap, subject to the ADR exception;
|
||||
- validate links and fenced examples;
|
||||
- confirm non-owning documents summarize and link rather than redefine;
|
||||
- remove stale or unsupported claims; and
|
||||
- confirm that no secrets or sensitive private data were added.
|
||||
For Promptkit ADRs, retain the existing Proposed, Accepted, Rejected, and
|
||||
Superseded statuses. Proposed records may become Accepted or Rejected;
|
||||
Accepted records may become Superseded, with links between the earlier record
|
||||
and its replacement. Records may be created as Accepted when the decision has
|
||||
already been made. These lifecycle rules preserve existing local document
|
||||
semantics.
|
||||
|
||||
27
docs/policy/meta.md
Normal file
27
docs/policy/meta.md
Normal file
@@ -0,0 +1,27 @@
|
||||
# Policy Set Provenance
|
||||
|
||||
**Base Policy Version:** 1.0
|
||||
|
||||
**Last Updated**: 2026-09-27
|
||||
|
||||
**Upstream Source:** [meta.md](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/docs/policy/meta.md)
|
||||
|
||||
## Origin And Local Authority
|
||||
|
||||
This repository's policy set is adapted from
|
||||
[Standard Repository Policies](https://gitea.maximumdirect.net/eric/policy).
|
||||
The adopted local policies govern work in this repository. Upstream revisions
|
||||
become local policy through deliberate adoption, preserving local decisions.
|
||||
Each policy's `Upstream Source` identifies its adopted baseline.
|
||||
|
||||
## Adoption And Updates
|
||||
|
||||
For policy adoption, upstream revision reviews, and updates, consult the
|
||||
[upstream adoption and update guide](https://gitea.maximumdirect.net/eric/policy/src/branch/main/docs/adoption.md).
|
||||
That guide and the reference example collection are maintained upstream and
|
||||
accessed through links. Keep discovery links pointed at the current upstream
|
||||
project and guide; keep adopted-source and example links pinned to commits.
|
||||
|
||||
The local [developer guide](../development.md) routes tasks to applicable
|
||||
policies. Formal credit and licensing notices for the copied policy material
|
||||
belong in the local [attribution document](../oss/attribution.md).
|
||||
@@ -1,135 +1,227 @@
|
||||
# Open Source Licensing And Attribution Policy
|
||||
|
||||
**Base Policy Version:** 1.2
|
||||
**Base Policy Version:** 1.8
|
||||
|
||||
**Last Updated:** 2026-09-26
|
||||
**Last Updated**: 2026-09-27
|
||||
|
||||
Adapted from the standard policy; see [provenance and licensing](../oss/attribution.md#standard-repository-policies).
|
||||
**Upstream Source:** [oss.md](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/docs/policy/oss.md)
|
||||
|
||||
## Purpose And Scope
|
||||
|
||||
This policy defines Promptkit's project licensing conventions and House Format
|
||||
for upstream attribution. It covers relevant incorporated or distributed
|
||||
components, including transitive dependencies and copied code, documentation,
|
||||
or assets. Treatment follows the component's terms and actual use, not merely
|
||||
its direct-dependency status or common ownership. Development tools do not
|
||||
automatically require attribution.
|
||||
This policy defines project licensing conventions and a repeatable House Format
|
||||
for upstream attribution and license notices. It covers the project's own code
|
||||
and upstream components incorporated into or distributed with the application
|
||||
or library, including relevant transitive dependencies and copied code,
|
||||
documentation, or assets.
|
||||
Determine treatment from the component's terms and actual use.
|
||||
|
||||
Promptkit is a Go library with an author-selected LGPL-3.0-or-later grant in
|
||||
[COPYRIGHT](../../COPYRIGHT). Its public source repository is
|
||||
`https://gitea.maximumdirect.net/eric/promptkit`; it publishes source-only Go
|
||||
module releases through annotated tags, not binaries or separate hosted release
|
||||
pages. The [release procedure](../release.md) owns publication mechanics.
|
||||
Consulting development guidance or using tooling alone does not require an
|
||||
attribution entry. Copying or adapting documentation for inclusion in the
|
||||
repository is different: preserve its applicable attribution and license terms.
|
||||
A material's role as development guidance does not exempt copied documents.
|
||||
Apply any express permissions for attribution-free reuse of example formatting.
|
||||
|
||||
The House Format does not establish license compatibility or resolve every
|
||||
obligation arising from use or distribution. Adoption does not relicense
|
||||
existing material or override third-party terms.
|
||||
The baseline is a publicly available source repository with releases published
|
||||
alongside it. The format is independent of language, forge, and build system.
|
||||
It does not require attribution for every development tool, establish license
|
||||
compatibility, or resolve every obligation arising from use or distribution.
|
||||
|
||||
## Canonical Locations
|
||||
|
||||
The [documentation policy](documentation.md) assigns ownership:
|
||||
|
||||
- `COPYRIGHT`: project name, holders and years, SPDX expression, grant,
|
||||
warranty disclaimer, and statement about third-party material.
|
||||
- `LICENSE`: complete, unmodified project license texts, including incorporated
|
||||
companion text. Promptkit includes LGPLv3 followed by GPLv3.
|
||||
- `COPYRIGHT`: project name, copyright statements, SPDX expression, grant,
|
||||
warranty disclaimer, and third-party statement.
|
||||
- `LICENSE` contains the complete, unmodified project license text, including
|
||||
incorporated companion text. Dependency attribution does not relicense the project.
|
||||
- `docs/oss/attribution.md`: canonical component entries.
|
||||
- `docs/oss/licenses/`: complete, verbatim standard dependency texts, including
|
||||
companions where applicable. Shared texts can serve multiple entries.
|
||||
- `docs/oss/licenses/`: complete, verbatim standard and applicable companion texts.
|
||||
|
||||
Keep explanations and project-specific notices outside standard texts. Preserve
|
||||
distinct versions and exceptions, using recognizable filenames.
|
||||
Use recognizable license filenames, such as `Apache-2.0.txt` or `LGPL-3.0.txt`.
|
||||
A shared standard text may serve multiple entries. Keep explanations outside
|
||||
license texts and preserve distinct versions and exceptions.
|
||||
|
||||
## Upstream Reference Examples
|
||||
|
||||
Use the commit-pinned upstream examples to create actual project notices.
|
||||
Retain their links and keep the example collection upstream; keep applicable
|
||||
license texts locally as specified above.
|
||||
|
||||
## Project Licensing
|
||||
|
||||
The house defaults for new projects are GPL-3.0-or-later for applications and
|
||||
LGPL-3.0-or-later for libraries. An author's explicit choice takes precedence
|
||||
without an exception rationale, extra approval, or compensating controls.
|
||||
Promptkit retains its selected library grant; do not infer different licenses
|
||||
for individual packages or relicense existing code by adopting this policy.
|
||||
Unless the author specifies otherwise, use **GPL version 3 or later**
|
||||
(`GPL-3.0-or-later`) for applications and **LGPL version 3 or later**
|
||||
(`LGPL-3.0-or-later`) for libraries. An author's explicit choice takes precedence:
|
||||
for example, a one-off script may use MIT. Such a choice requires no exception
|
||||
rationale, additional approval, or compensating controls under this policy.
|
||||
|
||||
Keep COPYRIGHT, LICENSE, SPDX expressions, and README descriptions consistent.
|
||||
Use accurate holders and years; do not assume the maintainer owns every
|
||||
contribution or advance years automatically. Update grant, disclaimer, and text
|
||||
references together when the author deliberately changes licensing. Preserve
|
||||
third-party terms and required existing notices. No blanket per-file header
|
||||
requirement is introduced.
|
||||
These defaults guide new license selection. Adopting this policy does not
|
||||
relicense existing code or override third-party terms. Preserve an existing
|
||||
license unless the author deliberately changes it. For mixed-purpose projects,
|
||||
follow the author's selected scope rather than inferring different licenses for
|
||||
individual packages.
|
||||
|
||||
Use the upstream [application COPYRIGHT example](https://gitea.maximumdirect.net/eric/policy/src/commit/a469273adda336625f96b9d769ea27b33999f146/examples/licensing/application/COPYRIGHT)
|
||||
or [library COPYRIGHT example](https://gitea.maximumdirect.net/eric/policy/src/commit/a469273adda336625f96b9d769ea27b33999f146/examples/licensing/library/COPYRIGHT)
|
||||
to create the project’s root `COPYRIGHT` file.
|
||||
Use verified project names, contribution years, and copyright holders.
|
||||
For another selected license, adapt the grant, SPDX expression, disclaimer, and
|
||||
text references together to reflect the author's selected grant.
|
||||
|
||||
For GPLv3 projects, copy the complete GPLv3 text into `LICENSE`. For LGPLv3
|
||||
projects, place the complete LGPLv3 text followed by the complete GPLv3 text in
|
||||
`LICENSE`; the LGPL incorporates the GPL terms. Keep the texts unmodified and
|
||||
place project-specific statements in `COPYRIGHT`. For the default grants, the
|
||||
grant and SPDX expression in `COPYRIGHT` must expressly identify `or-later`;
|
||||
follow the author’s choice if a different grant is selected.
|
||||
|
||||
For other licenses, provide their complete text and any required project-specific
|
||||
fields; the standard reference collection remains unchanged. Keep `COPYRIGHT`,
|
||||
`LICENSE`, SPDX expressions, and README descriptions consistent with the actual
|
||||
grant. This convention introduces no blanket requirement for per-file headers;
|
||||
preserve notices or add them where the applicable terms require them.
|
||||
|
||||
Identify separately licensed documentation or assets in the project grant, with
|
||||
links to their attribution and license information. An application's default
|
||||
code license does not override the terms of incorporated documentation.
|
||||
|
||||
## Upstream Attribution House Format
|
||||
|
||||
Each component entry contains five fields:
|
||||
Begin `docs/oss/attribution.md` with `# Open Source Attribution Notices`,
|
||||
followed directly by component entries. These comprise the complete document.
|
||||
|
||||
1. Project name, normally as the heading.
|
||||
2. Upstream copyright statements, preserving their wording.
|
||||
3. Applicable SPDX license identifier or expression.
|
||||
4. License and notice links.
|
||||
5. Source repository link at the exact imported commit.
|
||||
Each entry presents these five fields in order, with blank lines between them:
|
||||
|
||||
These are fields, not a line limit. Include multiple statements or links where
|
||||
needed. Do not invent absent copyright statements or SPDX identifiers. Describe
|
||||
nonstandard terms through their linked text, preserve alternatives and
|
||||
exceptions, and record a selected option when relevant. Do not silently resolve
|
||||
an unsettled licensing choice.
|
||||
1. **Project name** as a level-two heading. An optional identification paragraph
|
||||
may follow, such as `Selected module: MODULE at VERSION.` with the module and
|
||||
version in backticks. For copied documentation, a concise `Material:` sentence
|
||||
may identify the affected files and whether they were adapted or copied
|
||||
without modification. A factual qualifier such as `(transitive dependency)`
|
||||
is permitted. Limit optional identification prose to factual component and
|
||||
material scope.
|
||||
2. **Upstream copyright statement or statements**, preserving their wording.
|
||||
Put multiple statements in separate paragraphs. Include this field when the
|
||||
selected upstream source supplies an applicable statement; otherwise, omit it.
|
||||
3. **SPDX license identifier or expression**, when applicable, using
|
||||
`SPDX-License-Identifier: EXPRESSION`.
|
||||
4. **License and notice links**, beginning `License and notices:`, separated
|
||||
by ` | `, and ending with a period. Label local standard texts by license
|
||||
name, such as `MIT` or `Apache 2.0`. Label upstream files by their actual
|
||||
filenames, such as `COPYRIGHT`, `LICENSE`, `NOTICE`, or `PATENTS`.
|
||||
5. **Source repository link**, rendered exactly as:
|
||||
|
||||
```markdown
|
||||
[Source](COMMIT_URL).
|
||||
```
|
||||
|
||||
Use the upstream [attribution example](https://gitea.maximumdirect.net/eric/policy/src/commit/a469273adda336625f96b9d769ea27b33999f146/examples/licensing/attribution.md)
|
||||
to create the project’s `docs/oss/attribution.md`.
|
||||
The example illustrates omitted fields and multiple copyrights and licenses.
|
||||
Replace the uppercase placeholders and illustrative URLs with actual component
|
||||
information. Local license links are relative to the destination
|
||||
`docs/oss/attribution.md`; copy the applicable texts into its `licenses/`
|
||||
directory. Do not copy example entries for components the project does not use.
|
||||
|
||||
These are five fields, not a line limit. Omit the SPDX field when no applicable
|
||||
identifier exists; identify nonstandard terms through the linked text. Preserve
|
||||
licensing alternatives and exceptions, and record a selected option when
|
||||
relevant through the factual license identification. Do not invent an identifier
|
||||
or silently resolve an unsettled licensing choice.
|
||||
|
||||
Keep entries factual. Do not add agent-authored explanations of license
|
||||
expressions, file-level license scope, research performed, retained source
|
||||
notices, or compliance conclusions. Avoid filler such as "none is invented
|
||||
here" and embellished link labels such as "upstream LICENSE", "additional
|
||||
patent grant", or "at the imported commit". Preserve actual required upstream
|
||||
content; this rule excludes editorial additions, not required notices. Policy
|
||||
interpretation and exception reasoning belong in the adopted OSS policy or an
|
||||
appropriate decision record, not the attribution document.
|
||||
|
||||
## License Texts And Upstream Links
|
||||
|
||||
Link standard texts to local copies. Link project-specific notices to the
|
||||
upstream copies at the exact commit corresponding to the imported version or
|
||||
copied material. Moving branches and version tags are not substitutes for
|
||||
commit-specific component links.
|
||||
Link standard license texts to their local copies. Link project-specific
|
||||
notices to their upstream copies. Every upstream repository or notice link must
|
||||
identify the exact commit corresponding to the
|
||||
imported version, including the selected source revision for copied material.
|
||||
|
||||
Classify material by content, not filename: an upstream LICENSE or COPYING can
|
||||
mix standard text with project-specific notices or exceptions. Link both when
|
||||
needed. Make incorporated companion texts directly reachable from each entry.
|
||||
Classify material according to its contents. An upstream `LICENSE`,
|
||||
`COPYING`, or similar file may contain project-specific notices or exceptions
|
||||
in addition to standard wording. Link both the local standard text and the
|
||||
commit-specific upstream file when needed. Make companion license texts directly
|
||||
reachable from the entry.
|
||||
|
||||
For project-specific license and attribution material, direct upstream links
|
||||
are accepted as satisfying reproduction, inclusion, or copy requirements under
|
||||
the adopted house interpretation. Do not add local-copy requirements or request
|
||||
repeated approval solely because a license uses those terms. This concerns
|
||||
notice reproduction, not other concrete obligations such as marking modified
|
||||
files or supplying corresponding source. Central entries do not authorize
|
||||
stripping required notices from incorporated source.
|
||||
For project-specific license and attribution material, this policy accepts
|
||||
direct links to the applicable upstream texts as satisfying reproduction,
|
||||
inclusion, or copy requirements. This is the adopted house interpretation.
|
||||
Agents must apply it without adding local-copy requirements or seeking repeated
|
||||
approval solely because a license uses those terms.
|
||||
|
||||
This interpretation concerns notice reproduction. Concrete obligations such as
|
||||
marking modified files or supplying corresponding source receive their own
|
||||
applicable treatment. Preserve required notices already present in incorporated
|
||||
source files; the central attribution entry does not authorize stripping them.
|
||||
|
||||
## Discoverability
|
||||
|
||||
End README with friendly statements linking to the actual grant in COPYRIGHT
|
||||
and to the attribution document. Each new checked-in release note ends with
|
||||
links whose visible text is `COPYRIGHT | LICENSE | ATTRIBUTION`, using absolute
|
||||
URLs at the release's immutable tag or corresponding immutable source revision.
|
||||
These release links differ from imported-component links, which use commits.
|
||||
End the project's README with two friendly statements linking to its actual
|
||||
grant in `COPYRIGHT` and the attribution document. For example, for a project
|
||||
with the stated license:
|
||||
|
||||
The release procedure owns concrete URL construction. If a hosted download page
|
||||
is deliberately introduced, reuse the checked-in note and preserve its footer
|
||||
there. This convention does not require a new hosted page, downloadable license
|
||||
assets, source bundles, or another packaging format. Do not maintain duplicate
|
||||
notice lists.
|
||||
```markdown
|
||||
PROJECT_NAME is licensed under the [LGPL v3.0 or later](COPYRIGHT).
|
||||
|
||||
[Open Source attribution notices](docs/oss/attribution.md) are available.
|
||||
```
|
||||
|
||||
Each new checked-in release document and its hosted download page must end
|
||||
with links whose visible text is `COPYRIGHT | LICENSE | ATTRIBUTION`. Use absolute
|
||||
URLs to those files at the release's immutable tag or corresponding immutable source
|
||||
revision so the links work in both locations. This release navigation is distinct
|
||||
from upstream component links, which must use the imported commit.
|
||||
|
||||
Reuse the checked-in release document as the hosted release body where supported.
|
||||
The release procedure owns concrete URLs and publication mechanics for delivering
|
||||
the links through the selected channel. Do not maintain duplicate notice lists.
|
||||
These conventions do not by themselves require separate downloadable license
|
||||
assets, source bundles, or a different binary package format.
|
||||
|
||||
## Maintenance And Exceptions
|
||||
|
||||
Update attribution in the change that adds, removes, upgrades, copies, or changes
|
||||
distribution of a relevant component. Inspect the selected source's actual
|
||||
licenses/notices rather than assuming they are unchanged. Update entries,
|
||||
commit links, and texts together; retain history through Git.
|
||||
Update project licensing statements together when the author changes the grant
|
||||
or its scope. Update upstream notices in the change that adds, removes, upgrades,
|
||||
copies, or changes the distribution of a relevant component. Check the selected source's license
|
||||
and notices rather than assuming they are unchanged. Update affected entries,
|
||||
commit links, and texts; retain historical versions through Git.
|
||||
|
||||
Review affected statements, standard-text fidelity, and link destinations during
|
||||
ordinary change review. Future-release URLs can be checked against candidate
|
||||
paths before the tag exists remotely. Do not add publication-time network checks,
|
||||
CI polling, recurring legal sign-offs, mandatory scanners, or an SBOM just to
|
||||
implement this policy. Attribution maintenance is preparation, not a release gate.
|
||||
Check affected statements, faithful standard-text copies, and link destinations
|
||||
as part of ordinary change review. For a future release tag, check intended file
|
||||
paths against the candidate source; the tag need not resolve remotely before
|
||||
publication. Do not add publication-time network checks, CI polling, recurring
|
||||
legal sign-offs, mandatory scanners, or a full software bill of materials merely
|
||||
to implement this policy.
|
||||
|
||||
Raise exceptions only for concrete obligations or unresolved rights questions
|
||||
the House Format does not address, including distribution-specific issues.
|
||||
Record their selected treatment here or in the affected entry and resolve them
|
||||
before the affected use/distribution, without reopening settled choices.
|
||||
Common ownership is not a blanket exception. Adapted standard policy documents
|
||||
retain their upstream documentation license and identify modifications in their
|
||||
provenance entry; this does not change Promptkit's library grant.
|
||||
Use the House Format routinely. Raise an exception only for a concrete obligation
|
||||
or unresolved rights question it does not address, including any arising from a
|
||||
particular distribution channel. Record the issue and selected treatment in the
|
||||
adopted OSS policy or an appropriate decision record. Attribution entries
|
||||
contain the resulting factual identification and required material, without
|
||||
repeating that reasoning. Resolve the issue before the affected use or
|
||||
distribution without reopening settled choices.
|
||||
Common ownership of projects is neither required nor a blanket exception.
|
||||
|
||||
The maintainer-provided standard policy templates are internal and non-public.
|
||||
Their attribution entry identifies that source in prose and retains the
|
||||
applicable license locally; do not link to or require publication of the
|
||||
internal repository.
|
||||
Apply the formatting convention prospectively. Do not rewrite published tags,
|
||||
notes, binaries, or checksums to retrofit it. Address actual omissions deliberately
|
||||
under the applicable terms and the [release correction policy](release.md#immutability-and-failure-recovery).
|
||||
Ordinary attribution maintenance and release links are preparation work, not new
|
||||
release approval gates.
|
||||
|
||||
Apply conventions prospectively. Do not rewrite published tags, notes, binaries,
|
||||
or checksums to retrofit formatting. Handle actual omissions deliberately under
|
||||
applicable terms and the [release correction policy](release.md#immutability-and-failure-recovery).
|
||||
## Promptkit Adoption
|
||||
|
||||
Promptkit retains its LGPL-3.0-or-later library grant in
|
||||
[COPYRIGHT](../../COPYRIGHT). Adapted policy documentation has the separate
|
||||
scope identified there and in [attribution](../oss/attribution.md).
|
||||
|
||||
Promptkit publishes source-only Go module tags on Gitea, with no binaries or
|
||||
separate hosted download page. The [release procedure](../release.md) owns
|
||||
publication and concrete footer URLs. The hosted-page guidance applies if such
|
||||
a channel is deliberately introduced; it does not require creating one.
|
||||
|
||||
@@ -1,151 +1,280 @@
|
||||
# Release Policy
|
||||
|
||||
**Base Policy Version:** 1.2
|
||||
**Base Policy Version:** 1.6
|
||||
|
||||
**Last Updated:** 2026-09-26
|
||||
**Last Updated**: 2026-10-08
|
||||
|
||||
## Purpose And Adoption Scope
|
||||
**Upstream Source:** [release.md](https://gitea.maximumdirect.net/eric/policy/src/commit/e833faddf120c789a9b45348edaf75720de16348/docs/policy/release.md)
|
||||
|
||||
This policy owns Promptkit's release requirements and compatibility rules.
|
||||
The [release procedure](../release.md) owns the commands used to satisfy them.
|
||||
This scoped adoption applies to releases prepared after its adoption on
|
||||
2026-09-23. Published tags and historical notes are not rewritten to retrofit
|
||||
the new note format.
|
||||
## Purpose
|
||||
|
||||
## Release Model And Compatibility
|
||||
This policy defines a proportionate workflow for validating and publishing an
|
||||
intended source revision under a fresh version. Releases should require little
|
||||
routine intervention, avoid repeated work, and preserve published contents.
|
||||
|
||||
Promptkit publishes source-only Go module releases from committed, pushed
|
||||
`main` candidates through annotated `vMAJOR.MINOR.PATCH` tags. It publishes
|
||||
no binaries, hosted releases, or other artifacts and uses maintainer-run
|
||||
validation rather than hosted CI. Versions must be unused locally and on
|
||||
`origin`; the existing procedure accepts stable version tags only.
|
||||
Adopt stronger verification or reproducibility requirements through an explicit
|
||||
project-specific decision justified by the intended audience and support needs.
|
||||
|
||||
Before v1, minor releases may change the public API and patches preserve
|
||||
compatibility within their minor line, subject to the catalog-data provision
|
||||
below. Release notes must identify compatibility impact, public API changes,
|
||||
and required consumer action. The source tag identifies the exact release;
|
||||
the checked-in note at that tag records its historical summary.
|
||||
The repository-specific commands belong in `docs/release.md`. Historical release
|
||||
summaries belong in `docs/releases/`. The procedure must agree with the adopted
|
||||
policy and link to it rather than introduce additional release requirements.
|
||||
|
||||
## Catalog Profile Compatibility
|
||||
## Release Model And Completion
|
||||
|
||||
External catalog profile data may be added, removed, renamed, or modified
|
||||
without deprecation or advance notice. This exception is limited to profile
|
||||
data imported from the maintained catalog modules. It does not weaken ordinary
|
||||
compatibility for Promptkit's public Go API, configuration and format semantics,
|
||||
runtime behavior, catalog-module APIs, manifest schema, backend definitions, or
|
||||
consumer-owned definitions.
|
||||
A source release identifies one exact committed source revision through an
|
||||
immutable version tag and a version-matched, checked-in release note. The
|
||||
primary release branch is `main` unless the adopted policy specifies otherwise.
|
||||
The procedure must name the authoritative publication remote.
|
||||
|
||||
Within the catalog versions selected and tested by a Promptkit release, and
|
||||
unless a consumer overrides the name or its base, Promptkit keeps the literal
|
||||
profile IDs `promptkit-openrouter-standard` and
|
||||
`promptkit-openrouter-auto` available. The names do not promise a permanent
|
||||
model, backend, routing behavior, settings, cost, output, or upstream
|
||||
availability. The [framework format reference](../formats.md#built-in-profile-catalog)
|
||||
owns their current definitions.
|
||||
A successful push of the intended tag, with its source commit available on the
|
||||
authoritative primary branch, completes the release. The source commit may be
|
||||
pushed before the tag or published together with it through an atomic push.
|
||||
The procedure must choose and document the supported approach.
|
||||
|
||||
`v0.10.0` is the initial minor release for this compatibility transition.
|
||||
Afterward, catalog-only updates and adjustments to the current selections may
|
||||
ship in a patch release when all other compatibility obligations remain intact.
|
||||
External profile removals can require consumer action; they are not automatic
|
||||
migrations.
|
||||
Release commands must finish at successful publication. They must not wait for
|
||||
or poll CI, require successful CI status, inspect downloadable artifacts, or
|
||||
check consumer installation or Go module availability as completion gates.
|
||||
A failed or ambiguous push requires resolving the authoritative remote state
|
||||
before declaring success or retrying; successful pushes do not require a
|
||||
separate remote read-back gate.
|
||||
|
||||
Catalog adoption and release validation remain local and offline. They do not
|
||||
probe provider endpoints, request model lists, or make inference calls.
|
||||
Repositories may publish source only. Additional binaries, images, packages,
|
||||
checksums, or other declared deliverables may be published asynchronously from
|
||||
the tagged source. The procedure must identify those deliverables and the
|
||||
responsibility of their automation. Published deliverables must be traceable
|
||||
to that source and identify the same version.
|
||||
|
||||
## Release Documentation
|
||||
Track source-release completion and artifact delivery separately. Report delivery
|
||||
failures and optional investigations under artifact-delivery status.
|
||||
|
||||
## Versioning And Compatibility
|
||||
|
||||
Stable releases use Semantic Versioning in the form `vMAJOR.MINOR.PATCH`, with
|
||||
no leading zeroes. Prereleases require an explicit adopted policy defining
|
||||
their format and use.
|
||||
|
||||
The selected version must be unused locally and on the authoritative remote.
|
||||
Do not require queries to every configured remote or publication service.
|
||||
A version known to have already been published must not be reused, even if the
|
||||
normal guards do not inspect that channel. Adding a distribution channel
|
||||
requires a deliberate update to the policy and procedure where needed.
|
||||
|
||||
For releases at or after `v1.0.0`:
|
||||
|
||||
- a major release may make intentional incompatible changes;
|
||||
- a minor release may add backward-compatible behavior or raise the Go consumer
|
||||
minimum as described below; and
|
||||
- a patch release may make backward-compatible fixes.
|
||||
|
||||
Before `v1.0.0`, a minor release may change a documented public, operational,
|
||||
configuration, data, integration, or compatibility contract when the release
|
||||
note explains the impact and required consumer action. A patch release must
|
||||
not intentionally break those contracts within its minor line.
|
||||
|
||||
Additional compatibility promises and deliberate exceptions belong in the
|
||||
adopted policy or canonical contract documentation. Release notes explain
|
||||
changes; they do not redefine current compatibility contracts.
|
||||
|
||||
## Consumer Minimum And Release Toolchain
|
||||
|
||||
Distinguish the minimum toolchain supported for consumers from the toolchain
|
||||
maintainers use to build and validate a release. Record the consumer minimum in
|
||||
the language's standard metadata and explain its support meaning in canonical
|
||||
documentation. Define the selected release toolchain and its setup commands in
|
||||
`docs/release.md`. Toolchain selection and upgrade cadence are repository-specific
|
||||
choices.
|
||||
|
||||
For Go repositories, normally declare the initial stable release of the selected
|
||||
series as the consumer minimum, such as `go 1.26.0`. Use a higher patch minimum
|
||||
when a dependency or an identified correctness fix requires it, and document
|
||||
the reason.
|
||||
|
||||
For release-policy purposes, successful build and release validation on the
|
||||
selected Go patch release are sufficient to presume support across that Go
|
||||
series, subject to the declared consumer minimum and concrete evidence of
|
||||
incompatibility. Investigate specific compatibility concerns as they arise.
|
||||
|
||||
Use the selected release toolchain for candidate validation and release builds,
|
||||
including CI jobs performing those tasks. A newer validation patch version alone
|
||||
leaves the consumer minimum unchanged.
|
||||
|
||||
A minor release may raise the Go consumer minimum, including from one Go series
|
||||
to the next, both before and after `v1.0.0`. This is an explicit exception to the
|
||||
backward-compatibility requirement for minor releases. Patch releases preserve
|
||||
the consumer minimum. Explain an increase's impact and required consumer action
|
||||
in the release note. Other incompatible changes remain subject to the versioning
|
||||
rules above.
|
||||
|
||||
## Release Documentation And Preparation
|
||||
|
||||
Every new release must include `docs/releases/<version>.md` in the tagged
|
||||
source, using the complete tag as the filename. Its title contains Promptkit
|
||||
and that exact version. Use this structure:
|
||||
source tree. Use the complete version tag as the filename and begin with a
|
||||
title containing the product name and that exact version.
|
||||
|
||||
```markdown
|
||||
# Promptkit vMAJOR.MINOR.PATCH
|
||||
Include a short summary and any relevant compatibility impact or upgrade
|
||||
instructions. Choose sections suited to the release and keep the note
|
||||
proportionate to the change.
|
||||
|
||||
This release ...
|
||||
Keep exact current contracts in their canonical documents and update affected
|
||||
documentation with the behavior change. Link to those documents from the note
|
||||
where useful. If annotated tags are used, derive their summary from the
|
||||
checked-in note.
|
||||
|
||||
## Summary
|
||||
Include the release footer defined by the [OSS policy](oss.md#discoverability)
|
||||
in each new release document and preserve it on the hosted download page. The
|
||||
release procedure defines the concrete URLs and publication mechanics. Prepare
|
||||
these links with the release note; they do not add a network check or approval
|
||||
gate.
|
||||
|
||||
## Compatibility
|
||||
Handle version selection, release-note content, and retirement of completed
|
||||
roadmaps during preparation. Preserve planned and deferred work and repair
|
||||
affected links when retiring roadmaps. Reuse ordinary change review for
|
||||
unintended files, credentials, and private data. Scope release-content review
|
||||
to the tagged source.
|
||||
|
||||
## Upgrade
|
||||
## Candidate Validation
|
||||
|
||||
## Changes
|
||||
```
|
||||
Provide one publication command that validates the final committed candidate,
|
||||
checks publication state, and publishes the intended refs. A validation-only
|
||||
mode or command may be provided for diagnosis, but must not be a required step
|
||||
before publication repeats the same checks.
|
||||
|
||||
State “No action required” or “None” where appropriate rather than omitting a
|
||||
section. Keep exact current contracts in their canonical documents and link
|
||||
to them. Update affected canonical documentation in the same candidate.
|
||||
Run the complete deterministic, offline default test suite once for the final
|
||||
candidate, following the [testing policy](testing.md). For repositories that
|
||||
publish builds, verify the required build targets. Select those targets for
|
||||
the application's actual support commitments; native testing on each platform
|
||||
and access to particular hardware are not default release prerequisites.
|
||||
|
||||
These notes are historical records, not optional disposable guidance. Published
|
||||
release-note contents must not be replaced under the same version. Corrections
|
||||
belong in a subsequent release. Annotated-tag messages complement the checked-in
|
||||
note: generate the annotation from that note and the validated commit rather
|
||||
than maintaining a second summary.
|
||||
Additional gates must address a concrete risk that the existing checks do not
|
||||
adequately cover. Do not automatically require linting, race testing, module
|
||||
hygiene audits, documentation audits, credential scans, example validation, or
|
||||
expanded platform matrices solely because they are possible. Retain such
|
||||
checks when their distinct protection justifies their cost. A one-time migration
|
||||
check must not silently become a permanent release requirement.
|
||||
|
||||
## Validation And Publication
|
||||
Release tests must not require real credentials, mutable application services,
|
||||
paid APIs, provider probes, or inference calls. The procedure may describe dependency
|
||||
preparation separately from offline validation.
|
||||
|
||||
New release notes prepared after the 2026-09-26 OSS-policy adoption end with
|
||||
the [licensing footer](oss.md#discoverability). The release procedure owns
|
||||
absolute, release-pinned URLs. This is prospective preparation work, not a new
|
||||
approval or network-validation gate; historical notes and tags are not reformatted.
|
||||
Avoid duplicate validation across release phases. Successful checks for the
|
||||
same final candidate need not be repeated solely to satisfy another phase;
|
||||
any reuse must clearly apply to that exact candidate. Persistent validation
|
||||
caches are not required. If the candidate changes, validate the new candidate.
|
||||
Cheap publication-state checks may be repeated immediately before pushing
|
||||
because local or remote state may have changed during validation.
|
||||
|
||||
The [publication script](../../scripts/release.sh) runs the
|
||||
[local checks](../../scripts/release-check.sh) once for the final candidate.
|
||||
Retain clean-candidate, synchronized-main, unused-version, module-hygiene,
|
||||
release-note structure, and tag-target checks. Publish only the selected tag
|
||||
ref. Changing the candidate requires fresh validation. No provider endpoint
|
||||
probes or inference calls are release checks.
|
||||
Automate mechanical guards. Human review gates are strongly disfavored: add one
|
||||
only through an explicit project-specific policy decision explaining the
|
||||
concrete risk and why ordinary change review or automation is insufficient.
|
||||
Do not add routine confirmations for checks the command can perform itself.
|
||||
|
||||
Release-note content and retirement of completed roadmaps are preparation
|
||||
responsibilities, not separate human sign-off gates. Preserve planned and
|
||||
deferred work and repair links. Ordinary change review covers unintended files,
|
||||
secrets, and private data; do not repeat it as a release-specific approval.
|
||||
## Publication Guards
|
||||
|
||||
A successful tag push to origin completes the release. Never wait for CI to
|
||||
finish or succeed. CI results, remote read-back, and Go module availability
|
||||
are informational follow-ups, not completion gates. An ambiguous push result
|
||||
requires confirming publication on origin, not waiting for CI.
|
||||
Automate the checks needed to publish the intended source under a fresh version:
|
||||
|
||||
- the version has the adopted format and is unused locally and on the
|
||||
authoritative remote;
|
||||
- the worktree and index are clean on the primary release branch;
|
||||
- the source being published is the exact candidate that passed validation,
|
||||
including its matching release note;
|
||||
- the remote primary branch already identifies that candidate, or an atomic
|
||||
source/tag push will advance it to that candidate without discarding remote
|
||||
history; and
|
||||
- the tag identifies the validated candidate and only the intended refs are
|
||||
published.
|
||||
|
||||
Fetch the authoritative branch and tag state as needed for these checks.
|
||||
Do not force branch or tag updates to bypass a failed guard. Publish explicit
|
||||
refs rather than all local tags. Do not require CI success, publication-service
|
||||
availability checks, or a separate manual confirmation for each guard.
|
||||
|
||||
## Immutability And Failure Recovery
|
||||
|
||||
Never move, replace, recreate, or delete a published tag or replace published
|
||||
release contents. If validation fails before publication, correct the candidate,
|
||||
rerun the complete checks, and repeat the publication guards. Remove an
|
||||
unpublished local tag only after confirming it was not published.
|
||||
Published version tags and published contents are immutable. Do not move,
|
||||
overwrite, recreate, or delete a published tag, or replace a published artifact,
|
||||
checksum, signature, provenance statement, or release note under that version.
|
||||
The checked-in note at the immutable tag is that release's historical record.
|
||||
|
||||
If publication or subsequent investigation reveals a defect, leave the
|
||||
published release intact. Correct the defect on main, select a new patch or
|
||||
other semantically appropriate version, write its matching release note, and
|
||||
repeat the complete procedure. If publication status is uncertain, inspect
|
||||
the authoritative remote before deciding whether a tag is unpublished; do not
|
||||
overwrite it to retry.
|
||||
If validation fails before publication, correct and commit the candidate and
|
||||
rerun the publication command. If a push fails or its outcome is ambiguous,
|
||||
inspect the authoritative remote before retrying. Remove an unpublished local
|
||||
tag only after confirming it was never published or used by a release channel.
|
||||
An unavailable remote leaves publication status unknown; do not overwrite a tag
|
||||
to resolve uncertainty.
|
||||
|
||||
Transient network, authentication, CI, or module-availability failures do not
|
||||
by themselves warrant a new version. Investigate separately and publish a
|
||||
corrective release only when published contents actually need changing.
|
||||
Transient network, authentication, CI, or consumer-availability failures do not
|
||||
by themselves require a new version. Asynchronous delivery may be retried from
|
||||
the same immutable tag. Missing artifacts may be published later, provided
|
||||
existing published contents are not replaced and the additions remain consistent
|
||||
with the tagged source and any already-published checksum manifest. If that
|
||||
cannot be achieved, publish a new version.
|
||||
|
||||
## Policy Exceptions And Deliberate Scope
|
||||
A defect requiring changes to published source or contents requires a corrected
|
||||
commit, a new semantically appropriate version, a matching release note, and
|
||||
the release procedure again. Leave the original release intact. A channel's
|
||||
yank, deprecation, or advisory mechanism may warn consumers away from a defective
|
||||
release without replacing its contents or reusing its version.
|
||||
|
||||
This adoption follows the maintainer-approved lean workflow rather than every
|
||||
base-policy guard:
|
||||
## Repository Release Procedure
|
||||
|
||||
- Collision checks concern origin, the authoritative publication destination,
|
||||
rather than every configured remote. Local and origin tag checks, synchronized
|
||||
main, and an explicit selected-ref push protect the actual release channel.
|
||||
- Automated local validation and publication-state checks replace manual
|
||||
mechanical inspections and separate human sign-offs. Ordinary change review,
|
||||
editorial responsibility for release notes, and roadmap housekeeping remain;
|
||||
repeating them as approvals would add friction without distinct protection.
|
||||
- Successful tag publication completes this source-only release. CI can be slow
|
||||
or unreliable, and module availability depends on external services. Neither
|
||||
is a completion gate. Local candidate validation and explicit tag-target
|
||||
checks protect publication; ambiguous push results are resolved against origin.
|
||||
Actual defects are corrected in a new release, never by replacing a tag.
|
||||
Maintain `docs/release.md` as a short, executable maintainer runbook. It must
|
||||
identify:
|
||||
|
||||
Ignored local files are not prohibited merely because they exist: they are not
|
||||
included in the tagged commit. The clean-candidate guard and ordinary review of
|
||||
tracked changes protect the published contents.
|
||||
1. prerequisites, selected release toolchain, primary branch, and authoritative
|
||||
remote;
|
||||
2. version and release-note preparation;
|
||||
3. one publication command with its selected local checks and automated guards;
|
||||
4. declared deliverables, required build targets, and automation responsibilities;
|
||||
5. the successful-push completion point and separately reported delivery status;
|
||||
and
|
||||
6. failure handling, ambiguous-push recovery, and asynchronous delivery retries.
|
||||
|
||||
The catalog profile provision preserves an already approved compatibility
|
||||
boundary; it does not extend volatility to backend definitions or runtime
|
||||
contracts. Further release-governance changes require deliberate approval.
|
||||
Use copyable commands where practical. Keep optional diagnostics and consumer
|
||||
verification clearly optional. The procedure must not turn preparation work,
|
||||
ordinary change review, or post-publication investigation into additional
|
||||
release gates.
|
||||
|
||||
## Promptkit Release Configuration
|
||||
|
||||
The release-note requirements were adopted on 2026-09-23 and licensing
|
||||
navigation on 2026-09-26. Apply them prospectively; previously published notes
|
||||
and tags retain their original records.
|
||||
|
||||
Promptkit publishes source-only Go modules from main to origin on Gitea, using
|
||||
annotated stable SemVer tags. It has no binaries, hosted download pages, or
|
||||
asynchronous deliverables. The source commit is pushed through main before the
|
||||
tag. The [release procedure](../release.md) owns commands and recovery.
|
||||
|
||||
Release preparation uses the latest Go installation available in the development
|
||||
environment to build the maintainer release executable and validate the
|
||||
candidate, including the consumer examples. Keep that same local toolchain
|
||||
active throughout those operations.
|
||||
|
||||
For a minor release, advance Promptkit's consumer minimum to the initial stable
|
||||
release of that Go series, retaining a higher patch minimum when required by a
|
||||
dependency or an identified correctness fix. Patch releases preserve the
|
||||
existing consumer minimum. Commit any module update before validating the final
|
||||
candidate. The [release procedure](../release.md#prepare-the-candidate) owns
|
||||
setup commands, and the consumer guide explains
|
||||
[Go compatibility](../consumers/pkg-promptkit.md#go-compatibility).
|
||||
|
||||
Use a title containing `Promptkit` and the exact version, followed by a short
|
||||
summary and any relevant compatibility impact or upgrade instructions. The
|
||||
note is committed with the candidate and supplies the annotated tag message.
|
||||
No fixed set of section headings is required.
|
||||
|
||||
The [publication command](../../scripts/release.sh) delegates to the released
|
||||
ReleaseKit tool selected in `go.mod`. The committed
|
||||
[profile](../../releasekit.json) selects the race-enabled Go suite, full vet,
|
||||
committed Go formatting, local Markdown links, workspace and filesystem
|
||||
replacement inspection, and the offline semantic checks for the three
|
||||
maintained consumer examples. These are the selected checks for the final
|
||||
candidate; a separate ordinary test run or standalone build is not an
|
||||
additional release gate. The upstream tool owns candidate, remote, and tag
|
||||
publication checks. It returns native status 0 for confirmed publication, 1
|
||||
for a known validation or publication failure, 2 for invalid input or profile,
|
||||
and 3 when the push outcome is unknown. For status 3, resolve the exact
|
||||
captured publication endpoint before retrying; the repository-specific
|
||||
[procedure](../release.md#failure-recovery) links to the upstream recovery
|
||||
instructions. Do not wait for CI or Go module availability. Licensing footer
|
||||
URLs are preparation work, not a network gate.
|
||||
|
||||
@@ -1,43 +1,32 @@
|
||||
# Testing Policy
|
||||
|
||||
**Base Policy Version:** 1.0
|
||||
**Base Policy Version:** 1.3
|
||||
|
||||
**Last Updated:** 2026-09-23
|
||||
**Last Updated**: 2026-09-27
|
||||
|
||||
**Upstream Source:** [testing.md](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/docs/policy/testing.md)
|
||||
|
||||
## Purpose
|
||||
|
||||
Our tests exist to make **incorrect changes expensive and correct changes cheap**.
|
||||
|
||||
We do not optimize for test count, line coverage, exhaustive isolation, or the fewest possible tests. We optimize for sufficient confidence in important behavior while imposing as little unnecessary friction as possible on future development.
|
||||
We optimize for sufficient confidence in important behavior while minimizing friction on future development.
|
||||
|
||||
## Every test has a cost
|
||||
|
||||
Testing is not an unqualified good. Every test imposes both an immediate cost and a continuing lifetime cost.
|
||||
|
||||
A test must be:
|
||||
|
||||
- written and reviewed;
|
||||
- understood by future maintainers and coding agents;
|
||||
- executed in maintainer-run validation;
|
||||
- diagnosed when it fails;
|
||||
- updated when legitimate behavior changes;
|
||||
- maintained as fixtures, APIs, and dependencies evolve; and
|
||||
- removed or rewritten when it becomes redundant, brittle, misleading, or obsolete.
|
||||
|
||||
Tests also create cognitive and architectural friction. They can constrain refactoring, duplicate policy, slow feedback loops, add noise to failures, and cause harmless implementation changes to require unrelated edits across the suite.
|
||||
Every test has an immediate and continuing cost: writing, review, execution,
|
||||
diagnosis, fixture maintenance, and updates when legitimate behavior changes.
|
||||
Tests can also constrain refactoring, duplicate policy, and add failure noise.
|
||||
|
||||
A test is warranted only when the confidence it provides justifies these costs.
|
||||
|
||||
Apply this cost-benefit analysis at two levels:
|
||||
Evaluate that tradeoff at two levels:
|
||||
|
||||
1. **Per test:** What realistic defect does this test detect, how consequential would that defect be, and is that protection worth the test's lifetime cost?
|
||||
2. **Across the suite:** Does this collection provide materially more confidence than a smaller, simpler suite would?
|
||||
|
||||
The preferred test suite is a **lean suite that provides sufficient confidence in the risks that matter, without redundant or low-value tests**. We seek sufficient confidence with the least unnecessary testing friction, not the fewest possible tests.
|
||||
Some friction is intentional. Tests should make dangerous changes—such as breaking compatibility, corrupting data, violating security boundaries, or reintroducing subtle bugs—require deliberate review. Keep ordinary internal changes inexpensive.
|
||||
|
||||
Some friction is intentional. Tests should make dangerous changes—such as breaking compatibility, corrupting data, violating security boundaries, or reintroducing subtle bugs—require deliberate review. They should not make ordinary internal changes needlessly expensive.
|
||||
|
||||
The cost of a test is not a reason to omit testing by default. Do not cite maintenance cost abstractly. When omitting a plausible test, be able to state why the protected failure is low-risk, already covered, obvious, reversible, or cheaper to detect elsewhere. For consequential, subtle, or difficult-to-observe behavior, the presumption should favor testing.
|
||||
Evaluate maintenance cost against the specific failure being protected. When omitting a plausible test, be able to state why the protected failure is low-risk, already covered, obvious, reversible, or cheaper to detect elsewhere. For consequential, subtle, or difficult-to-observe behavior, the presumption should favor testing.
|
||||
|
||||
## Default testing style
|
||||
|
||||
@@ -51,6 +40,12 @@ Use a **classical/Detroit-style** approach:
|
||||
|
||||
Examples of appropriate seams include clocks, randomness, subprocesses, remote APIs, object storage, email, and paid LLM calls.
|
||||
|
||||
## Promptkit Test Boundaries
|
||||
|
||||
Promptkit is a library without a CLI or hosted CI. The execution requirements,
|
||||
facade test ownership, public error guidance, and offline consumer examples
|
||||
below tailor the standard to that boundary.
|
||||
|
||||
## Test execution requirements
|
||||
|
||||
Promptkit currently uses maintainer-run validation rather than hosted CI.
|
||||
@@ -63,17 +58,24 @@ and repository hygiene.
|
||||
Introducing hosted CI later would supplement, not silently redefine, this
|
||||
documented validation model.
|
||||
|
||||
Full release-candidate validation includes race-enabled package tests, explicit
|
||||
static analysis, and execution with automated semantic checks of the three maintained
|
||||
offline consumer examples. The test run builds the packages; the examples
|
||||
also exercise ordinary compilation. Separate ordinary tests and a standalone
|
||||
build are not additional release gates. Ordinary tests remain useful during
|
||||
development. The preparation example protects assembled preparation and
|
||||
The selected ReleaseKit profile runs the race-enabled package suite and full
|
||||
vet, checks committed Go formatting, local Markdown links, and module
|
||||
distribution hygiene, and executes semantic assertions for the three
|
||||
maintained offline consumer examples. ReleaseKit owns this shared validation
|
||||
machinery and publication workflow. The test run builds the packages; the
|
||||
examples also exercise ordinary compilation. Separate ordinary tests and a
|
||||
standalone build are not additional release gates. Ordinary tests remain useful
|
||||
during development. The preparation example protects assembled preparation and
|
||||
inspection behavior. The execution example separately protects assembled
|
||||
`Run`, injected-client, validation, usage, and result behavior. The observation
|
||||
example protects assembled built-in retries, structured proposal and assessment
|
||||
runs, and application-owned interpretation of metadata.
|
||||
|
||||
The standalone `python3 scripts/test-release.py` command exercises Promptkit's
|
||||
ReleaseKit wiring against disposable local repositories. It is focused
|
||||
development validation when release tooling changes, not part of the default
|
||||
Go test suite or a recurring release check.
|
||||
|
||||
Tests in the default suite must be deterministic, offline, and independent of
|
||||
real credentials. They must not invoke paid APIs, use live network
|
||||
dependencies, or depend on mutable external services. Tests that require live
|
||||
@@ -110,7 +112,7 @@ Use each test type where it protects a distinct risk:
|
||||
|
||||
## What deserves tests
|
||||
|
||||
Prioritize tests for:
|
||||
To the extent the following features are present in the application, prioritize tests for:
|
||||
|
||||
1. Public and package-level contracts.
|
||||
2. Domain rules and important invariants.
|
||||
@@ -118,17 +120,29 @@ Prioritize tests for:
|
||||
4. Failure handling, cancellation, retries, recovery, and partial success.
|
||||
5. Serialization, schemas, compatibility, and round trips.
|
||||
6. Previously observed or plausible regressions.
|
||||
7. Representative integration and consumer workflows.
|
||||
7. Representative integration and end-to-end workflows.
|
||||
|
||||
A package-level contract is behavior relied upon by another package or major collaborator, not every observable detail of a package implementation.
|
||||
A package-level contract comprises behavior relied upon by another package or major collaborator.
|
||||
|
||||
For behavior involving **data integrity, destructive operations, compatibility, security, concurrency, idempotency, or recovery**, presume that durable tests are required unless the behavior is already credibly protected at another layer.
|
||||
Choose tests for requested behavior and established contracts. Tests must not
|
||||
introduce new restrictions, guarantees, or threat scenarios merely because they
|
||||
could be asserted. Follow the [agent policy](agents.md) when deciding whether a
|
||||
behavior belongs to the contract; current implementation behavior alone is not
|
||||
sufficient authority.
|
||||
|
||||
For established behavior involving **data integrity, destructive operations,
|
||||
compatibility, security, concurrency, idempotency, or recovery**, favor durable
|
||||
tests for realistic failures whose consequences warrant protection. Identify
|
||||
the failure and the existing contract it would violate. These categories call
|
||||
for attention, not an automatic test inventory or permission to strengthen the
|
||||
application's guarantees. Account for protection already provided at other layers.
|
||||
|
||||
Do not add tests merely because a function, branch, or line exists. Do not add a test when the same meaningful risk is already adequately protected elsewhere.
|
||||
|
||||
## Choose the right test boundary
|
||||
|
||||
Test through the narrowest stable boundary that expresses the behavior clearly.
|
||||
Choose the narrowest stable boundary that expresses the behavior clearly and
|
||||
provides durable confidence with minimal incidental coupling.
|
||||
|
||||
This is often the package API, but it may instead be:
|
||||
|
||||
@@ -136,30 +150,15 @@ This is often the package API, but it may instead be:
|
||||
- a package-level operation when several internal collaborators jointly produce the behavior; or
|
||||
- a larger integration boundary when correctness emerges from interaction with a real dependency.
|
||||
|
||||
Do not force all behavior through oversized consumer-workflow tests. Do not
|
||||
test every private helper merely because it exists. Choose the boundary that
|
||||
gives durable confidence with the least incidental coupling.
|
||||
|
||||
## Test behavior, not implementation
|
||||
|
||||
A test should protect a decision, contract, or invariant—not memorialize the current implementation.
|
||||
A test should protect a selected decision, contract, or invariant.
|
||||
|
||||
Before adding or retaining a test, ask:
|
||||
A test is suspect when it primarily detects changes to private constants or
|
||||
helpers, equivalent operation ordering, incidental formatting, equivalent
|
||||
algorithms, or internal structure that preserves behavior.
|
||||
|
||||
> What realistic defect would this test catch?
|
||||
|
||||
A test is suspect when its main purpose is to detect that someone:
|
||||
|
||||
- changed an internal constant;
|
||||
- renamed or split a private helper;
|
||||
- reordered equivalent internal operations;
|
||||
- changed incidental formatting;
|
||||
- replaced one correct algorithm with another; or
|
||||
- refactored internal object structure without changing behavior.
|
||||
|
||||
Refactoring should normally require no test edits unless the refactored structure is itself part of the contract.
|
||||
|
||||
A test can be factually correct and still have negative value. Accurately describing current behavior is not enough; the protected behavior must be important enough to justify the future friction.
|
||||
Behavioral tests should normally remain stable through internal refactoring that preserves the contract.
|
||||
|
||||
## Expected effects of different changes
|
||||
|
||||
@@ -176,13 +175,13 @@ A test failing is not the same as a test needing to be edited. Many tests may co
|
||||
|
||||
## Separate mechanism from policy
|
||||
|
||||
Configurable thresholds and defaults must not be duplicated throughout the test suite.
|
||||
Derive expectations from test-controlled configuration and relationships.
|
||||
|
||||
The following fragments are illustrative rather than standalone Go programs.
|
||||
Do not encode an internal concurrency limit indirectly:
|
||||
For this example, assume the selected contract rejects a request to start more
|
||||
processes than a configured limit. Do not encode its production default indirectly:
|
||||
|
||||
```go
|
||||
// Production policy:
|
||||
// Production default:
|
||||
const maxConcurrency = 4
|
||||
|
||||
// Brittle test:
|
||||
@@ -200,37 +199,40 @@ require.NoError(t, runner.Start(limit))
|
||||
require.ErrorIs(t, runner.Start(limit+1), ErrTooMuchConcurrency)
|
||||
```
|
||||
|
||||
The test should prove:
|
||||
For this selected contract, the test should prove:
|
||||
|
||||
- the configured limit is accepted; and
|
||||
- one beyond the configured limit is rejected.
|
||||
|
||||
Rejection is not a general requirement for concurrency control. If the selected
|
||||
contract queues excess work or limits active workers while accepting larger
|
||||
batches, test that behavior instead. Do not add rejection or a configurable
|
||||
limit solely to reproduce this example.
|
||||
|
||||
The production default should be tested exactly only when its literal value is itself a public, operational, safety, protocol, or compatibility requirement.
|
||||
|
||||
Apply the same rule to limits, timeouts, capacities, retry counts, and ranges: test relationships and behavior, not duplicated literals.
|
||||
Apply the same rule to limits, timeouts, capacities, retry counts, and ranges: test relationships and behavior.
|
||||
|
||||
For concurrency limits, test both kinds of behavior when relevant:
|
||||
For established concurrency limits, distinguish these behaviors where the contract requires them:
|
||||
|
||||
1. **Configuration enforcement:** invalid or excessive requested values are handled correctly.
|
||||
1. **Configuration handling:** values are accepted or rejected according to the selected contract.
|
||||
2. **Runtime enforcement:** observed peak concurrency never exceeds the configured limit.
|
||||
|
||||
Use a test-controlled limit and measure the behavior relative to that limit. Do not merely assert today's default value.
|
||||
Use a test-controlled limit and measure the behavior relative to that limit.
|
||||
|
||||
## Avoid semantic duplication across layers
|
||||
## Assign each behavior a test owner
|
||||
|
||||
Each behavior should have a clear test owner.
|
||||
Where these layers exist, assign ownership as follows:
|
||||
|
||||
- Parser tests own parsing cases.
|
||||
- Validator tests own validation rules.
|
||||
- Domain tests own transformations and invariants.
|
||||
- Boundary tests own external integration behavior.
|
||||
- Adapter tests own external integration behavior.
|
||||
- Orchestrator tests own coordination and failure propagation.
|
||||
- Consumer-workflow tests prove that representative assembled library use
|
||||
works.
|
||||
- External-package root tests own the public Go facade contract.
|
||||
- Consumer-workflow tests prove that representative assembled library use works.
|
||||
|
||||
Higher-level tests should not repeat every lower-level case. A single intentional policy change should not require unrelated edits across many test files.
|
||||
|
||||
Tests that are individually reasonable may still be collectively redundant. Evaluate the marginal value of each additional test in light of the protection already provided by the rest of the suite.
|
||||
Give each layer its distinct responsibility and keep expectation updates localized to the contract being changed.
|
||||
|
||||
## Use test doubles deliberately
|
||||
|
||||
@@ -251,111 +253,78 @@ Mocks are appropriate when the contract includes facts such as:
|
||||
- an expensive API is called no more than once; or
|
||||
- a security audit event is emitted.
|
||||
|
||||
Do not use mocks merely to isolate every object or reproduce the implementation's call graph.
|
||||
|
||||
## Go-specific guidance
|
||||
|
||||
Use:
|
||||
Select techniques by behavior and risk:
|
||||
|
||||
- table-driven tests for meaningful behavioral categories and boundaries;
|
||||
- `t.TempDir()` for real filesystem behavior;
|
||||
- `httptest.Server` for realistic HTTP interactions;
|
||||
- fuzz tests for parsers, normalization, path handling, and broad input spaces;
|
||||
- fuzz tests when exploring broad input spaces, such as parsers or path handling, adds useful protection beyond representative cases;
|
||||
- golden files only when the complete output is intentionally stable;
|
||||
- integration tests where correctness depends on component interaction; and
|
||||
- a small number of representative consumer-workflow tests.
|
||||
- a small number of representative end-to-end tests when assembled workflows expose meaningful risks not adequately covered at narrower boundaries.
|
||||
|
||||
Avoid exact error-string assertions unless the wording is itself contractual. Prefer `errors.Is`, `errors.As`, typed errors, or structured error fields.
|
||||
Prefer `errors.Is`, `errors.As`, typed errors, or structured error fields. Use exact error-wording assertions only when the wording is contractual.
|
||||
|
||||
At public API boundaries, prefer stable error identity, structured values, and
|
||||
the smallest semantic fragment needed to identify the failure. Do not snapshot
|
||||
complete diagnostic wording unless it is contractual.
|
||||
the smallest semantic fragment needed to identify the failure. Use complete
|
||||
diagnostic snapshots only when the wording is contractual.
|
||||
|
||||
Golden-file updates must require an explicit local flag. Ordinary validation
|
||||
runs must never update golden files automatically, and maintainers must inspect
|
||||
the semantic diff before accepting an update.
|
||||
Golden-file updates must require an explicit local flag. Ordinary validation runs must never update golden files automatically, and reviewers must inspect the semantic diff before accepting an update.
|
||||
|
||||
Keep tests readable and direct. Test helpers and fixture frameworks must earn their own maintenance cost; do not build elaborate test infrastructure for small or isolated needs.
|
||||
Keep tests readable and direct. Size test helpers and fixture frameworks to the maintenance benefit they provide.
|
||||
|
||||
## Coverage
|
||||
|
||||
Coverage is a diagnostic, not a target.
|
||||
Use coverage to locate untested critical branches and unexpectedly weak packages. Evaluate test quality through meaningful behavioral protection and choose coverage according to risk.
|
||||
|
||||
Use it to find untested critical branches and unexpectedly weak packages. Do not write low-value tests solely to increase a percentage, and do not infer test quality from coverage alone.
|
||||
|
||||
Pure domain logic will often warrant higher coverage than facade wiring or
|
||||
external adapters. Uneven coverage is acceptable when it reflects risk.
|
||||
|
||||
Increasing coverage is valuable only when the newly covered behavior protects a meaningful risk at an acceptable cost.
|
||||
Pure domain logic will often warrant higher coverage than facade wiring or external adapters. Uneven coverage is acceptable when it reflects risk.
|
||||
|
||||
## Regression tests
|
||||
|
||||
A bug fix should normally include a regression test that fails before the fix and passes afterward.
|
||||
|
||||
Retain the test when the defect could realistically recur and its consequences justify the ongoing cost. Prefer the narrowest durable test of the violated contract or invariant; do not preserve accidental implementation details from the original bug.
|
||||
Retain the test when the defect could realistically recur and its consequences justify the ongoing cost. Prefer the narrowest durable test of the violated contract or invariant.
|
||||
|
||||
Not every historical bug requires a permanent test. If the underlying design has made recurrence impossible, the test has become redundant, or a stronger invariant test now subsumes it, remove or consolidate it.
|
||||
Remove or consolidate a test when the underlying design has made recurrence impossible, its protection is redundant, or a stronger invariant test subsumes it.
|
||||
|
||||
## Deleting or rewriting tests
|
||||
|
||||
Tests are maintained code, not permanent historical artifacts.
|
||||
|
||||
Delete or rewrite a test when its maintenance cost exceeds the confidence it provides.
|
||||
Delete or rewrite tests when maintenance cost exceeds their continuing
|
||||
protective value.
|
||||
|
||||
Strong candidates include tests that:
|
||||
|
||||
- require updates after harmless internal changes;
|
||||
- directly assert private constants without protecting a real contract;
|
||||
- duplicate the same policy across several layers;
|
||||
- assert incidental private constants or snapshot large amounts of incidental output;
|
||||
- verify mock choreography rather than outcomes;
|
||||
- snapshot large amounts of incidental output;
|
||||
- test trivial private helpers already exercised through stable package behavior;
|
||||
- protect risks already covered more effectively elsewhere;
|
||||
- duplicate policy across layers, retest trivial helpers covered by package
|
||||
behavior, or protect risks covered more effectively elsewhere;
|
||||
- are flaky, misleading, obsolete, or disproportionately expensive to diagnose; or
|
||||
- no longer correspond to a plausible failure mode.
|
||||
|
||||
Several brittle tests may encode one genuine requirement. Replace them with one durable behavior-level or invariant test rather than preserving all of them.
|
||||
|
||||
Deleting a low-value test can improve the quality of the suite by reducing noise, maintenance burden, and friction around legitimate change.
|
||||
Several brittle tests may encode one genuine requirement. Consolidate them into one durable behavior-level or invariant test.
|
||||
|
||||
## Reviewing a proposed test
|
||||
|
||||
Use the following questions when the value, boundary, or durability of a proposed test is not self-evident. Significant test additions should be reviewable against them, but written answers are not required for every routine test.
|
||||
Use the following questions when a proposed test needs closer evaluation. Significant test additions should be reviewable against them; written answers are optional for routine tests.
|
||||
|
||||
1. What realistic defect would it catch?
|
||||
2. How likely is that defect?
|
||||
3. How consequential would it be?
|
||||
4. Is the behavior already protected elsewhere?
|
||||
5. At which layer should this behavior be owned?
|
||||
6. Does the test assert a durable contract or an incidental implementation detail?
|
||||
7. Could the implementation be refactored without changing the behavior and without editing this test?
|
||||
8. What should cause this test to fail?
|
||||
9. What legitimate changes should not cause this test to fail?
|
||||
10. What ongoing maintenance, execution, and diagnostic cost will the test impose?
|
||||
11. Is there a smaller or more direct test that protects the same risk?
|
||||
|
||||
Do not add the test when its expected lifetime cost exceeds its expected protective value.
|
||||
|
||||
When deciding not to test plausible behavior, record or be able to explain why the risk is low, already protected, obvious, reversible, or cheaper to detect elsewhere.
|
||||
1. Which requested behavior or established contract does it protect, and what realistic defect would violate it?
|
||||
2. How likely and consequential is that defect, and is it already protected elsewhere?
|
||||
3. What is the narrowest stable boundary that can test it clearly?
|
||||
4. Would a correct internal refactor or a change to an incidental default require test edits?
|
||||
5. Does its confidence justify its maintenance, execution, and diagnostic cost, or would a smaller test suffice?
|
||||
|
||||
## Definition of sufficient
|
||||
|
||||
A test suite is sufficient when:
|
||||
|
||||
- important contracts and invariants are protected;
|
||||
- meaningful boundaries and failure modes are exercised;
|
||||
- realistic and consequential regressions are credibly protected against silent recurrence;
|
||||
- behavior involving data integrity, destructive operations, compatibility, security, concurrency, idempotency, and recovery is credibly protected;
|
||||
- important external boundaries have realistic integration coverage;
|
||||
- representative complete workflows are tested;
|
||||
- failures provide useful signal rather than redundant noise;
|
||||
- legitimate internal changes usually do not require test edits; and
|
||||
- important requested behavior and established contracts are protected against realistic, consequential failures;
|
||||
- boundaries and component interactions have coverage where they introduce meaningful risks not protected elsewhere;
|
||||
- failures provide useful, distinct signals;
|
||||
- tests usually remain stable through legitimate internal changes; and
|
||||
- additional tests would mostly repeat existing protection or preserve inconsequential implementation details.
|
||||
|
||||
Sufficiency is a risk judgment, not a coverage percentage or test count.
|
||||
Reassess it as the library, its consumers, and the consequences of failure
|
||||
evolve.
|
||||
|
||||
The governing rule is:
|
||||
|
||||
> Test heavily where failure is consequential, subtle, or difficult to detect after the fact. Test lightly where failure is obvious, reversible, and inexpensive—and retain no test whose lifetime cost exceeds the confidence it provides.
|
||||
Reassess sufficiency as the application, its users, and the consequences of
|
||||
failure evolve.
|
||||
|
||||
162
docs/release.md
162
docs/release.md
@@ -1,94 +1,130 @@
|
||||
# Release Procedure
|
||||
|
||||
This runbook implements the [release policy](policy/release.md). A successful
|
||||
push of the selected annotated tag to origin completes the release. Never wait
|
||||
for CI to finish or succeed.
|
||||
This runbook implements the [release policy](policy/release.md). Promptkit
|
||||
publishes source-only Go module tags from `main` to `origin`. A successful push
|
||||
of the selected annotated tag, after its source commit is available on `main`,
|
||||
completes the release. Do not wait for CI or Go module availability.
|
||||
|
||||
## Prepare The Candidate
|
||||
|
||||
Required tools are Git, Go with race-detector support, POSIX shell, and Python
|
||||
3. Publication requires push access to origin. Work from the repository root.
|
||||
3. Work from the repository root with push access to `origin`.
|
||||
|
||||
Select an unused version and create `docs/releases/<version>.md` using the
|
||||
[required structure](policy/release.md#release-documentation). Describe
|
||||
compatibility impact and consumer actions and update affected canonical docs.
|
||||
For catalog updates, record selected catalog versions and any consumer action,
|
||||
linking to the catalog policy rather than repeating its contracts.
|
||||
Use the latest Go installation available locally for the release tool, the
|
||||
candidate validation, and the maintained examples. Make it the active `go`
|
||||
command and record its version:
|
||||
|
||||
Follow the [OSS policy](policy/oss.md) when preparing licensing and attribution
|
||||
changes. Each new note must end with the following navigation, replacing
|
||||
`vMAJOR.MINOR.PATCH` with the selected release tag:
|
||||
|
||||
```markdown
|
||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit/src/tag/vMAJOR.MINOR.PATCH/COPYRIGHT) | [LICENSE](https://gitea.maximumdirect.net/eric/promptkit/src/tag/vMAJOR.MINOR.PATCH/LICENSE) | [ATTRIBUTION](https://gitea.maximumdirect.net/eric/promptkit/src/tag/vMAJOR.MINOR.PATCH/docs/oss/attribution.md)
|
||||
```sh
|
||||
export GOTOOLCHAIN=local
|
||||
go version
|
||||
```
|
||||
|
||||
Check these file paths against the candidate; a future tag need not resolve
|
||||
remotely during preparation. The annotated tag derives from this same note,
|
||||
including its footer. Promptkit publishes no separate hosted release body or
|
||||
download page. If such a channel is deliberately introduced, reuse the note
|
||||
and preserve its footer. No extra network checks, artifacts, or approval gates
|
||||
are introduced, and published notes are not retrofitted.
|
||||
For a minor release, advance Promptkit's consumer minimum to the initial stable
|
||||
release of that Go series before committing the candidate:
|
||||
|
||||
Retire completed roadmaps, preserving planned or deferred work and repairing
|
||||
links. This is housekeeping, not a separate approval gate. Review changes
|
||||
through the ordinary development workflow, commit them, and push main to
|
||||
origin. No repeated release-specific diff review is required.
|
||||
```sh
|
||||
release_go_version=$(go env GOVERSION)
|
||||
release_go_series=${release_go_version#go}
|
||||
release_go_series=${release_go_series%.*}
|
||||
go mod edit "-go=${release_go_series}.0"
|
||||
```
|
||||
|
||||
For example, Go 1.26.8 normally results in `go 1.26.0`. Keep a higher patch
|
||||
minimum if a dependency or identified correctness fix requires it. For a patch
|
||||
release, leave the consumer minimum unchanged; using a newer validation
|
||||
toolchain does not itself raise that minimum. If the candidate needs a higher
|
||||
minimum, prepare a minor release instead. See the
|
||||
[consumer-minimum policy](policy/release.md#consumer-minimum-and-release-toolchain).
|
||||
|
||||
Prepare module caches before validation; this may update `go.sum` and belongs
|
||||
before the candidate commit:
|
||||
|
||||
```sh
|
||||
GOTOOLCHAIN=local go mod download
|
||||
```
|
||||
|
||||
Choose an unused stable version and create `docs/releases/<version>.md` with a
|
||||
title containing `Promptkit` and that exact version, a short summary, and any
|
||||
relevant compatibility impact or upgrade instructions. No fixed section
|
||||
headings are required. Describe a raised Go minimum and its compatibility
|
||||
impact under the [versioning policy](policy/release.md#versioning-and-compatibility).
|
||||
End each new note with the licensing footer below, using the selected version:
|
||||
|
||||
```markdown
|
||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit/src/tag/vX.Y.Z/COPYRIGHT) | [LICENSE](https://gitea.maximumdirect.net/eric/promptkit/src/tag/vX.Y.Z/LICENSE) | [ATTRIBUTION](https://gitea.maximumdirect.net/eric/promptkit/src/tag/vX.Y.Z/docs/oss/attribution.md)
|
||||
```
|
||||
|
||||
Check the footer paths against the candidate; the future tag need not resolve
|
||||
remotely during preparation. Follow the [OSS policy](policy/oss.md) for these
|
||||
links. Promptkit publishes no binaries, hosted download page, or other
|
||||
asynchronous deliverables.
|
||||
|
||||
Retire completed roadmaps while preserving deferred work and repairing incoming
|
||||
links. Review the intended changes through the ordinary development workflow,
|
||||
commit the candidate on `main`, and push `main` to `origin` before publication.
|
||||
Preparation may update module metadata; the release bootstrap and validation
|
||||
preserve the committed candidate.
|
||||
|
||||
## Validate And Publish
|
||||
|
||||
Run the following with the chosen unused version (replace the placeholder):
|
||||
For an optional local diagnostic of a committed candidate, run:
|
||||
|
||||
```sh
|
||||
./scripts/releasekit.sh check vMAJOR.MINOR.PATCH
|
||||
```
|
||||
|
||||
This is not a required step before publication. The upstream
|
||||
[candidate-check contract](https://gitea.maximumdirect.net/eric/ReleaseKit/src/tag/v0.4.1/docs/consumers/check.md)
|
||||
describes its requirements.
|
||||
|
||||
Run the single publication command with the chosen version:
|
||||
|
||||
```sh
|
||||
./scripts/release.sh vMAJOR.MINOR.PATCH
|
||||
```
|
||||
|
||||
The [publication script](../scripts/release.sh) checks a clean, synchronized
|
||||
main candidate, the version and unused tag, and the release note's title and
|
||||
required headings. It runs the [local checks](../scripts/release-check.sh)
|
||||
once, rechecks candidate identity and publication state, generates the
|
||||
annotation from the committed note plus the validated commit, and pushes only
|
||||
the selected annotated tag.
|
||||
The familiar wrapper delegates to the native ReleaseKit executable selected in
|
||||
`go.mod`. The committed [profile](../releasekit.json) selects the race-enabled
|
||||
Go suite, full vet, committed Go formatting, local Markdown links, workspace
|
||||
and filesystem replacement inspection, and semantic checks for the three
|
||||
maintained offline consumer examples. ReleaseKit also verifies the committed
|
||||
candidate and publication state, then publishes only the selected tag. Its
|
||||
checks run once for this candidate; do not repeat them as a separate sign-off.
|
||||
|
||||
There is no separate tag-message authoring, example-output inspection,
|
||||
metadata inspection, or human sign-off. Compatibility and release-note content
|
||||
remain ordinary editorial responsibilities. Development validation is useful
|
||||
but does not require a second full run before invoking the publication script.
|
||||
|
||||
The script stops successfully after the tag push. CI completion, CI success,
|
||||
remote read-back, and Go module availability are not release completion gates.
|
||||
Do not wait or poll for them. Provider probes and inference calls are never
|
||||
part of release validation.
|
||||
|
||||
## Catalog Profile Compatibility
|
||||
|
||||
The [release policy](policy/release.md#catalog-profile-compatibility) owns the
|
||||
catalog compatibility boundary. This heading preserves older incoming links.
|
||||
The native command returns status 0 for confirmed publication, 1 for a known
|
||||
validation or publication failure, 2 for invalid input or profile, and 3 when
|
||||
the push outcome remains unknown. Successful publication needs no remote
|
||||
read-back, CI wait, or installation check. Provider probes and inference calls
|
||||
are not part of validation.
|
||||
|
||||
## Failure Recovery
|
||||
|
||||
If local checks fail, fix the candidate, commit and push changes, then rerun
|
||||
the publication command. It never automatically removes a tag.
|
||||
For status 1, use the diagnostic to distinguish a validation failure from a
|
||||
known publication conflict. Correct and commit candidate defects, push `main`,
|
||||
and rerun publication with an unused version when appropriate. Status 2 means
|
||||
the invocation or profile must be corrected.
|
||||
|
||||
If a push fails or its result is ambiguous, set `RELEASE_VERSION` to the
|
||||
attempted version and inspect origin before retrying:
|
||||
For status 3, do not retry or remove the local tag until the remote outcome is
|
||||
resolved. ReleaseKit reports the attempted version, tag object, and recovery
|
||||
state, including the captured publication endpoint when available. Use that
|
||||
endpoint—not merely the current `origin` URL—to query the exact tag; follow the
|
||||
upstream
|
||||
[failure-recovery procedure](https://gitea.maximumdirect.net/eric/ReleaseKit/src/tag/v0.4.1/docs/release.md#resolve-failures):
|
||||
|
||||
```sh
|
||||
git ls-remote --tags origin "refs/tags/$RELEASE_VERSION" "refs/tags/$RELEASE_VERSION^{}"
|
||||
git rev-parse "refs/tags/$RELEASE_VERSION" "refs/tags/$RELEASE_VERSION^{}"
|
||||
releasekit_endpoint='COPY_CAPTURED_ENDPOINT'
|
||||
releasekit_version='vX.Y.Z'
|
||||
test "$(git ls-remote --get-url -- "$releasekit_endpoint")" = "$releasekit_endpoint" &&
|
||||
git ls-remote --refs -- "$releasekit_endpoint" "refs/tags/$releasekit_version"
|
||||
```
|
||||
|
||||
Matching remote tag and peeled commit values confirm publication, even if the
|
||||
push command reported an error. If the remote query fails, publication status
|
||||
is still unknown. Do not delete or recreate the local tag until absence on
|
||||
origin is confirmed. Once confirmed unpublished, an unsuccessful attempt's
|
||||
local tag can be removed and the publication command rerun. Never overwrite
|
||||
a published tag.
|
||||
|
||||
CI or module-resolution problems may be investigated separately; temporary
|
||||
service, network, or authentication failures do not require another release.
|
||||
Only an actual defect requiring changes to published contents warrants a new
|
||||
appropriate version. See the [recovery policy](policy/release.md#immutability-and-failure-recovery).
|
||||
Compare the returned object ID with the reported tag object. An exact match
|
||||
confirms publication. A missing ref confirms absence only when the endpoint
|
||||
identity check and query both succeed. If either check fails, the outcome
|
||||
remains unknown; correct the transport configuration and resolve it before
|
||||
retrying. Only after authoritative absence is confirmed may an unpublished
|
||||
local tag be removed before correcting the candidate or retrying. Never move,
|
||||
overwrite, or delete a published tag.
|
||||
|
||||
## Policy Changes
|
||||
|
||||
|
||||
38
docs/releases/v0.11.2.md
Normal file
38
docs/releases/v0.11.2.md
Normal file
@@ -0,0 +1,38 @@
|
||||
# Promptkit v0.11.2
|
||||
|
||||
This patch aligns repository policies with the current standard and clarifies
|
||||
licensing and attribution for the copied policy documents.
|
||||
|
||||
## Summary
|
||||
|
||||
The update adds policy provenance, selective contributor reading guidance, and
|
||||
consistent upstream policy language while preserving Promptkit-specific rules.
|
||||
|
||||
## Compatibility
|
||||
|
||||
No public API, runtime behavior, dependency, or catalog changes are introduced.
|
||||
Promptkit remains LGPL-3.0-or-later. The [project grant](../../COPYRIGHT)
|
||||
identifies the separate CC-BY-SA-4.0 scope for copied or adapted policy material;
|
||||
the [attribution notices](../oss/attribution.md) identify the affected documents.
|
||||
The existing release checks and successful-tag-push completion point are unchanged.
|
||||
|
||||
## Upgrade
|
||||
|
||||
```sh
|
||||
go get gitea.maximumdirect.net/eric/promptkit@v0.11.2
|
||||
```
|
||||
|
||||
No code or configuration migration is required from v0.11.1.
|
||||
|
||||
## Changes
|
||||
|
||||
- Align adopted policies with Standard Repository Policies while retaining
|
||||
library-specific requirements and deliberate local decisions.
|
||||
- Add per-document upstream source links and the
|
||||
[policy-set provenance document](../policy/meta.md).
|
||||
- Add selective policy reading to the [contributor guide](../development.md)
|
||||
and update agent orientation.
|
||||
- Standardize factual attribution entries and identify copied-policy licensing.
|
||||
- Retire the completed policy-alignment roadmap.
|
||||
|
||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.11.2/COPYRIGHT) | [LICENSE](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.11.2/LICENSE) | [ATTRIBUTION](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.11.2/docs/oss/attribution.md)
|
||||
74
docs/releases/v0.12.0.md
Normal file
74
docs/releases/v0.12.0.md
Normal file
@@ -0,0 +1,74 @@
|
||||
# Promptkit v0.12.0
|
||||
|
||||
This release replaces Promptkit's configuration loader with ConfigKit and
|
||||
migrates maintained catalogs to keyed `/v3` assets. It intentionally changes
|
||||
the public construction API and configuration format.
|
||||
|
||||
## Summary
|
||||
|
||||
Backends, profiles, and versioned prompts now share one layered configuration
|
||||
snapshot built when `NewEngine` is called. ConfigKit supplies composition,
|
||||
inheritance, presence, and source history; Promptkit validates the final
|
||||
effective definitions and captures prompt content and templates. Consumers can
|
||||
inspect the captured configuration explicitly without triggering a reload.
|
||||
|
||||
## Compatibility
|
||||
|
||||
This is an incompatible minor release under Promptkit's pre-1.0
|
||||
[versioning policy](../policy/release.md#versioning-and-compatibility). The
|
||||
former source options, `Config.PromptDir`, `Config.ProfileDir`, and the old
|
||||
profile/backend convenience values have been removed. Configuration documents
|
||||
now use literal keys beneath `backends`, `profiles`, and `prompts`; embedded
|
||||
`id`/`version` fields and `base_profile` are no longer accepted. Source groups
|
||||
compose strictly within a group and overlay in the supplied order. Explicit
|
||||
zero, false, empty, and null values can now differ from omission; maps overlay
|
||||
recursively, and an empty map does not clear an inherited map.
|
||||
|
||||
All final definitions, including unused ones, are validated during engine
|
||||
construction. Prompt text and static templates are captured then; an existing
|
||||
engine does not automatically reload changed configuration or message files.
|
||||
Request artifacts, schemas, credentials, and provider operations retain their
|
||||
runtime boundaries. See the [format reference](../formats.md) for the current
|
||||
configuration contract and the [consumer migration guide](../consumers/pkg-promptkit.md#migrate-from-the-previous-configuration-api)
|
||||
for API and source-selection changes.
|
||||
|
||||
Maintained OpenRouter and Rakestrawhome catalogs are selected as `/v3` modules,
|
||||
both at `v3.20260930.0`. Their asset format changed; consumers that read catalog
|
||||
assets directly must adopt the keyed format. The maintained Promptkit profile
|
||||
IDs remain available under the [catalog compatibility policy](../policy/release.md#catalog-profile-compatibility),
|
||||
which does not promise fixed profile data.
|
||||
|
||||
## Upgrade
|
||||
|
||||
```sh
|
||||
go get gitea.maximumdirect.net/eric/promptkit@v0.12.0
|
||||
```
|
||||
|
||||
Replace removed source and backend registration options with
|
||||
`Config.Sources`, explicitly ordered ConfigKit underlays, and optionally a
|
||||
highest-priority main input. Use `DirectoryInputs` or `FilesystemInputs` for
|
||||
selected directories, `ProfileSource` and `BackendSource` for typed
|
||||
programmatic values, or `configkit.Mapping` for explicit null and combined
|
||||
documents. Move profile `base_profile` to `extends` and embedded identities to
|
||||
mapping keys. Review zero/false/null values and any assumptions that replacing
|
||||
a model or backend discards unrelated explicit settings. Rebuild the engine
|
||||
when definitions or referenced message files change. The
|
||||
[consumer guide](../consumers/pkg-promptkit.md#migrate-from-the-previous-configuration-api)
|
||||
provides the migration details and maintained examples.
|
||||
|
||||
## Changes
|
||||
|
||||
- Add ConfigKit v0.5.0 as the configuration boundary and consume released
|
||||
OpenRouter and Rakestrawhome `/v3` catalog modules.
|
||||
- Add keyed backend, profile, and prompt definitions with final-value domain
|
||||
validation, explicit presence/null behavior, and source-relative content
|
||||
capture.
|
||||
- Add `ConfigurationSources`, directory discovery helpers, programmatic
|
||||
profile/backend source builders, and read-only configuration inspection.
|
||||
- Preserve execution workflows while rendering captured templates, applying
|
||||
backend defaults after inheritance, and honoring explicit suppression of
|
||||
provider settings.
|
||||
- Remove superseded loader, catalog preflight, and inheritance machinery;
|
||||
update examples, format guidance, and internal documentation.
|
||||
|
||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.12.0/COPYRIGHT) | [LICENSE](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.12.0/LICENSE) | [ATTRIBUTION](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.12.0/docs/oss/attribution.md)
|
||||
69
docs/releases/v0.13.0.md
Normal file
69
docs/releases/v0.13.0.md
Normal file
@@ -0,0 +1,69 @@
|
||||
# Promptkit v0.13.0
|
||||
|
||||
This release adds prompt requirement discovery and profile/prompt listing,
|
||||
reduces avoidable configuration and execution allocations, and standardizes
|
||||
maintainer publication through ReleaseKit. The minimum supported Go version
|
||||
increases to 1.26.0.
|
||||
|
||||
## Discover Requirements And Available Definitions
|
||||
|
||||
- `Engine.InspectRequirements` reports recognizable input and caller-variable
|
||||
dependencies for a selected prompt and supplied caller context before input
|
||||
artifacts exist. It classifies dependencies as required, conditional, or
|
||||
optional and reports structured limitations when discovery is incomplete.
|
||||
Discovery coverage is not proof that requirements are satisfied or execution
|
||||
will succeed. See the [requirement inspection guide](../consumers/pkg-promptkit.md#inspect-requirements-before-preparation)
|
||||
and [GoDoc](../../requirements.go).
|
||||
- `Engine.ListProfiles` returns effective profile IDs, and `Engine.ListPrompts`
|
||||
returns exact ID/version pairs as `PromptIdentity` values. Lists come from the
|
||||
engine's captured configuration, are lexically ordered and caller-owned, and
|
||||
include consumer definitions and maintained profiles that remain effective.
|
||||
Listing performs no provider or execution work. See the
|
||||
[discovery guide](../consumers/pkg-promptkit.md#discover-available-profiles-and-prompts)
|
||||
and [GoDoc](../../listing.go).
|
||||
|
||||
## Efficiency And Tooling
|
||||
|
||||
- Reduce response-sized copies in basic validation and rendered-message output,
|
||||
and encode valid provider extra parameters only as part of the complete
|
||||
request rather than serializing them twice.
|
||||
- Use bounded initial file-size reservations without limiting accepted content,
|
||||
releasing excess backing storage for small or empty completed artifacts.
|
||||
- Reuse owned preparation values when creating prepared-execution snapshots,
|
||||
while preserving independent inspection details and caller-owned data.
|
||||
- Use lightweight captured profile identities and prompt metadata for inspection
|
||||
instead of copying execution targets and definitions unnecessarily.
|
||||
- Upgrade ConfigKit to v0.6.0 for more efficient loading, resolution, metadata,
|
||||
and inspection processing. Configuration semantics remain unchanged; these
|
||||
improvements primarily benefit construction and explicit operator inspection,
|
||||
not upstream inference latency.
|
||||
- Adopt module-selected ReleaseKit v0.3.0 for maintainer validation and tag
|
||||
publication. The [release procedure](../release.md) owns the current commands
|
||||
and recovery workflow; successful tag publication remains the completion
|
||||
point, without waiting for CI.
|
||||
|
||||
These changes preserve existing execution and ownership contracts. Benefits
|
||||
depend on workload; this release introduces no fixed performance guarantee.
|
||||
|
||||
## Compatibility And Upgrade
|
||||
|
||||
Consumers upgrading from v0.12.0 need **Go 1.26.0 or newer**, rather than the
|
||||
previous 1.25.5 minimum. Update application build environments and any CI images
|
||||
still using Go 1.25 before upgrading:
|
||||
|
||||
```sh
|
||||
go get gitea.maximumdirect.net/eric/promptkit@v0.13.0
|
||||
```
|
||||
|
||||
No application-code or configuration migration is required from v0.12.0 beyond
|
||||
the Go toolchain update. The new discovery methods are optional additions;
|
||||
existing construction, inspection, preparation, and execution workflows remain
|
||||
available. See [Go compatibility](../consumers/pkg-promptkit.md#go-compatibility)
|
||||
for the distinction between the consumer minimum and maintainer toolchain.
|
||||
|
||||
Maintainers should follow the current release runbook rather than the retired
|
||||
`scripts/release-check.sh`. The publication entry point remains
|
||||
`scripts/release.sh`; ReleaseKit now owns its shared validation and publication
|
||||
mechanics.
|
||||
|
||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.13.0/COPYRIGHT) | [LICENSE](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.13.0/LICENSE) | [ATTRIBUTION](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.13.0/docs/oss/attribution.md)
|
||||
48
docs/releases/v0.14.0.md
Normal file
48
docs/releases/v0.14.0.md
Normal file
@@ -0,0 +1,48 @@
|
||||
# Promptkit v0.14.0
|
||||
|
||||
This release adds comparison of captured configuration inspections and updates
|
||||
ConfigKit and the maintainer release tool. The minimum supported Go version
|
||||
remains 1.26.0.
|
||||
|
||||
## Compare Captured Configurations
|
||||
|
||||
`ConfigurationInspection.Diff` compares the effective configuration captured by
|
||||
two engines. Consumers can use it to review a candidate configuration, inspect
|
||||
an overlay's effect, or present configuration changes to operators.
|
||||
|
||||
Comparison delegates to ConfigKit and returns its difference records, including
|
||||
presence information, paired redaction, and protected-change indicators. It
|
||||
does not reload sources or contact providers. Source-history differences alone
|
||||
do not produce value differences.
|
||||
|
||||
An empty diff is not proof of equivalent execution: referenced file contents,
|
||||
resolved credentials, later execution preparation, and request-specific
|
||||
overrides are outside the comparison. Records can contain ordinary application
|
||||
data; callers retain responsibility for their disclosure.
|
||||
|
||||
See the [configuration comparison guide](../consumers/pkg-promptkit.md#inspect-the-captured-configuration)
|
||||
and [GoDoc](../../configuration_inspection.go) for usage and the exact contract.
|
||||
|
||||
## Dependency And Tooling Updates
|
||||
|
||||
- Upgrade ConfigKit from v0.6.0 to v0.7.1, providing the upstream comparison API.
|
||||
v0.7.1 also updates ConfigKit's maintainer tooling without changing its runtime
|
||||
behavior from v0.7.0.
|
||||
- Upgrade the module-selected ReleaseKit tool from v0.3.0 to v0.4.1. Promptkit
|
||||
retains its existing root-module profile, publication entry point, and release
|
||||
gates. See the [release procedure](../release.md) for maintainer commands.
|
||||
|
||||
## Compatibility And Upgrade
|
||||
|
||||
No application-code, configuration, or Go-toolchain migration is required from
|
||||
v0.13.0. Configuration comparison is an optional API addition; existing
|
||||
construction, inspection, preparation, and execution workflows remain available.
|
||||
|
||||
```sh
|
||||
go get gitea.maximumdirect.net/eric/promptkit@v0.14.0
|
||||
```
|
||||
|
||||
See [Go compatibility](../consumers/pkg-promptkit.md#go-compatibility) for the
|
||||
distinction between the consumer minimum and maintainer toolchain.
|
||||
|
||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.14.0/COPYRIGHT) | [LICENSE](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.14.0/LICENSE) | [ATTRIBUTION](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.14.0/docs/oss/attribution.md)
|
||||
34
docs/releases/v0.15.0.md
Normal file
34
docs/releases/v0.15.0.md
Normal file
@@ -0,0 +1,34 @@
|
||||
# Promptkit v0.15.0
|
||||
|
||||
This release adds error-only validation of a concrete prompt request without
|
||||
calling a model. The minimum supported Go version remains 1.26.0.
|
||||
|
||||
## Validate A Concrete Request
|
||||
|
||||
`Engine.ValidateRequest` accepts an ordinary `RunRequest` and returns nil when
|
||||
it passes preparation, or the existing preparation error. Consumers can use it
|
||||
for fail-fast checks, including downstream CLI validation commands, without
|
||||
receiving a prepared snapshot.
|
||||
|
||||
The method shares `Prepare`'s request conversion and preparation path. Unlike
|
||||
requirements inspection, it checks the actual supplied values and bindings.
|
||||
Artifact and schema resource I/O still occurs; success does not certify provider
|
||||
acceptance or later execution success. A subsequent run prepares independently.
|
||||
Validation is optional, not a required step before preparation or execution.
|
||||
|
||||
See the [request validation guide](../consumers/pkg-promptkit.md#validate-a-concrete-request)
|
||||
and [GoDoc](../../engine.go) for usage and the exact contract. Consumers that
|
||||
need execution tied to a checked snapshot can use the existing
|
||||
[prepared execution workflow](../consumers/pkg-promptkit.md#prepare-now-and-execute-the-same-snapshot-later).
|
||||
|
||||
## Compatibility And Upgrade
|
||||
|
||||
No application-code, configuration, or Go-toolchain migration is required from
|
||||
v0.14.0. This is an optional API addition; existing inspection, preparation, and
|
||||
execution workflows remain available.
|
||||
|
||||
```sh
|
||||
go get gitea.maximumdirect.net/eric/promptkit@v0.15.0
|
||||
```
|
||||
|
||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.15.0/COPYRIGHT) | [LICENSE](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.15.0/LICENSE) | [ATTRIBUTION](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.15.0/docs/oss/attribution.md)
|
||||
86
docs/releases/v0.16.0.md
Normal file
86
docs/releases/v0.16.0.md
Normal file
@@ -0,0 +1,86 @@
|
||||
# Promptkit v0.16.0
|
||||
|
||||
This release adds explicit smoke diagnostics, configuration collections,
|
||||
namespace-scoped engine views, and expanded resource discovery. It also makes
|
||||
intentional breaking changes to configuration ownership and prompt selection.
|
||||
The minimum supported Go version remains 1.26.0.
|
||||
|
||||
## Configuration Collections And Discovery
|
||||
|
||||
Applications can compose caller-selected collections of backends, profiles,
|
||||
and prompts. Collection resources have qualified identities such as
|
||||
`support/answer`; root resources retain unqualified identities. `CollectionInput`
|
||||
provides a convenience for mounting a selected ConfigKit input into a collection.
|
||||
Promptkit now uses ConfigKit v0.8.0, including its optional source specification
|
||||
header support.
|
||||
|
||||
`Engine.WithNamespaces` creates an immutable view with separate prompt, profile,
|
||||
and backend scopes for short caller-supplied IDs. Views share the underlying
|
||||
configuration, clients, capacity pools, and prepared-execution owner. Explicitly
|
||||
qualified IDs and configured references keep their exact identities.
|
||||
|
||||
`ListProfiles` and `ListPrompts` support namespace filters. New `ListBackends`,
|
||||
`ListNamespaces`, and `ListCollections` methods expose captured resource and
|
||||
collection inventories. See the [collection format](../formats.md#source-headers-collections-and-resource-ids),
|
||||
[scoped-view guide](../consumers/pkg-promptkit.md#use-scoped-engine-views), and
|
||||
[discovery guide](../consumers/pkg-promptkit.md#discover-available-resources-and-collections).
|
||||
|
||||
## Explicit Smoke Diagnostics
|
||||
|
||||
`Engine.SmokeTest` provides backend reachability, authentication, text inference,
|
||||
and structured-output inference diagnostics. Reports provide factual
|
||||
observations, timing, and available usage rather than an overall readiness
|
||||
verdict. Optional Boolean facts distinguish negative observations from unknown
|
||||
or inapplicable checks; matching a JSON schema does not prove that a provider
|
||||
enforced it.
|
||||
|
||||
A concluded diagnostic can report failure with a nil Go error. Invalid requests
|
||||
return ordinary errors, and caller cancellation can return partial observations
|
||||
with a recognizable context error. Diagnostics can contact providers and incur
|
||||
inference costs only when explicitly invoked; they are not automatic execution
|
||||
or release checks. See the [smoke diagnostic guide](../consumers/pkg-promptkit.md#run-explicit-smoke-diagnostics)
|
||||
and [API GoDoc](../../smoke_diagnostics.go).
|
||||
|
||||
## Breaking Changes And Upgrade
|
||||
|
||||
Consumers upgrading from v0.15.0 should follow the
|
||||
[configuration and API migration guide](../consumers/configuration-migration.md):
|
||||
|
||||
- Supply backend and profile definitions explicitly. Promptkit no longer bundles
|
||||
provider catalogs or the `promptkit-openrouter-standard` and
|
||||
`promptkit-openrouter-auto` profiles. Conventional backend names do not install
|
||||
definitions automatically. Applications choose and maintain their sources.
|
||||
- Convert version-keyed prompt mappings into one definition per prompt ID.
|
||||
Prompt `version` is now an optional descriptive label, not a selector.
|
||||
Choose deliberately which former definition to install.
|
||||
- Remove `PromptVersion` selectors from `RunRequest`,
|
||||
`RequirementInspectionRequest`, and `SmokeTestRequest`, where applicable.
|
||||
Change prompt inspection calls to `InspectPrompt(ctx, promptID)`. Update code
|
||||
consuming version-bearing inspection, identity, result, or observation fields
|
||||
to handle optional labels, including nil and explicitly empty labels.
|
||||
- When moving resources into collections, update stored backend, profile, and
|
||||
diagnostic-default references to their intended qualified IDs. Mounting a
|
||||
source does not rewrite references; profile inheritance remains within its
|
||||
sibling mapping. Root configuration `version`, when supplied, is now a source
|
||||
specification header, distinct from descriptive resource labels.
|
||||
|
||||
The migration guide includes identity mappings and before/after prompt examples.
|
||||
The [maintained offline example](../../examples/go-library/prepare/main.go)
|
||||
demonstrates explicit composition, source-relative content, scoped preparation,
|
||||
and inventory without contacting a provider.
|
||||
|
||||
After adapting configuration and API calls:
|
||||
|
||||
```sh
|
||||
go get gitea.maximumdirect.net/eric/promptkit@v0.16.0
|
||||
```
|
||||
|
||||
## Correctness And Efficiency
|
||||
|
||||
Collection prompt materialization shares one lazy origin-metadata snapshot,
|
||||
avoiding repeated full metadata copies across collections. Internal prepared
|
||||
snapshots detach optional prompt labels. Smoke diagnostics preserve caller
|
||||
cancellation and report actual inference destinations and known-zero usage when
|
||||
no generation request was dispatched.
|
||||
|
||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.16.0/COPYRIGHT) | [LICENSE](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.16.0/LICENSE) | [ATTRIBUTION](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.16.0/docs/oss/attribution.md)
|
||||
@@ -42,20 +42,21 @@ checkpoint equality or when a broader semantic identity design is selected.
|
||||
### Eager Source Validation
|
||||
|
||||
**Reason for deferral:** Exact prompt and profile inspection may already
|
||||
provide a sufficiently small validation surface. Experience from downstream
|
||||
adoption should establish whether an engine-wide operation would add enough
|
||||
value to justify its broader contract.
|
||||
provide a sufficiently small validation surface alongside construction-time
|
||||
definition validation. Experience from downstream adoption should establish
|
||||
whether a separate operation would add enough value to justify its broader
|
||||
contract.
|
||||
|
||||
Promptkit could provide an explicit offline operation that discovers and
|
||||
structurally validates configured prompt, profile, and schema sources without
|
||||
model generation. The normal `NewEngine` path would remain lazy.
|
||||
Promptkit could provide an explicit offline operation that also checks schema
|
||||
resources without model generation. `NewEngine` already validates effective
|
||||
definitions and captures prompt content; schema resources remain
|
||||
request-dependent.
|
||||
|
||||
An eager operation would need coherent handling for duplicate prompt IDs and
|
||||
versions, strict YAML decoding, referenced content files, profile/backend
|
||||
membership, schema syntax and transitive references, context cancellation,
|
||||
and source-specific public errors. Credential declarations must remain
|
||||
separate from credential values; checking current environment availability,
|
||||
if supported at all, should be an explicit option and must not expose secrets.
|
||||
Such an operation would need coherent handling for schema syntax and transitive
|
||||
references, context cancellation, and source-specific public errors. Credential
|
||||
declarations must remain separate from credential values; checking current
|
||||
environment availability, if supported at all, should be an explicit option
|
||||
and must not expose secrets.
|
||||
|
||||
Reconsider this idea after downstream use of `InspectPrompt`,
|
||||
`InspectProfile`, and fixture-based preparation demonstrates a concrete gap.
|
||||
|
||||
@@ -38,13 +38,8 @@ consumers.
|
||||
|
||||
## Ideas
|
||||
|
||||
### Remaining Standard Policy Alignment
|
||||
|
||||
Review the internal standard's architecture and testing clarifications about
|
||||
adopted constraints, threat scope, and tests that must not invent contracts.
|
||||
Adapt relevant guidance to Promptkit's library boundaries without introducing
|
||||
new runtime restrictions or release gates. This is separate from the completed
|
||||
licensing-documentation alignment.
|
||||
No ideas are currently awaiting selection. Active feature work belongs in its
|
||||
focused roadmap rather than this catalog.
|
||||
|
||||
## Entry Format
|
||||
|
||||
|
||||
281
engine.go
281
engine.go
@@ -8,6 +8,7 @@ import (
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
internalengine "gitea.maximumdirect.net/eric/promptkit/internal/engine"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/observation"
|
||||
@@ -15,7 +16,7 @@ import (
|
||||
)
|
||||
|
||||
// ErrInvalidConfig identifies invalid engine construction, including missing
|
||||
// required configuration, invalid options or backend registrations, and a nil
|
||||
// effective configuration, invalid options, and a nil
|
||||
// Engine receiver.
|
||||
var ErrInvalidConfig = errors.New("invalid engine configuration")
|
||||
|
||||
@@ -23,9 +24,8 @@ var (
|
||||
// ErrInvalidRequest identifies a request whose required values, overrides,
|
||||
// credentials, or effective settings are invalid.
|
||||
ErrInvalidRequest = errors.New("invalid run request")
|
||||
// ErrPromptNotFound identifies a requested prompt ID or version that is not
|
||||
// present in the selected prompt source. It does not also match
|
||||
// ErrPromptLoad.
|
||||
// ErrPromptNotFound identifies a requested prompt ID that is not present in
|
||||
// the captured configuration. It does not also match ErrPromptLoad.
|
||||
ErrPromptNotFound = errors.New("prompt not found")
|
||||
// ErrProfileNotFound identifies a selected profile ID that is absent from
|
||||
// every configured profile source. It does not also match ErrProfileLoad.
|
||||
@@ -34,13 +34,13 @@ var (
|
||||
// nor the selected prompt's default profile is present. Such an error also
|
||||
// matches ErrInvalidRequest.
|
||||
ErrProfileRequired = errors.New("profile selection is required")
|
||||
// ErrPromptLoad identifies a failure to read, decode, validate, select, or
|
||||
// hash a prompt definition, except for the not-found case represented by
|
||||
// ErrPromptNotFound.
|
||||
// ErrPromptLoad identifies a prompt selection failure other than the
|
||||
// not-found case represented by ErrPromptNotFound. Invalid definitions
|
||||
// instead fail construction with ErrInvalidConfig.
|
||||
ErrPromptLoad = errors.New("failed to load prompt definition")
|
||||
// ErrProfileLoad identifies a failure to read, decode, validate, or select
|
||||
// an execution profile or resolve its backend, except for the profile
|
||||
// not-found case represented by ErrProfileNotFound.
|
||||
// ErrProfileLoad identifies a profile selection or resolution failure other
|
||||
// than the not-found case represented by ErrProfileNotFound. Invalid
|
||||
// definitions instead fail construction with ErrInvalidConfig.
|
||||
ErrProfileLoad = errors.New("failed to load execution profile")
|
||||
// ErrAPIKeyEnvMissing identifies an explicitly required APIKeyEnv whose
|
||||
// environment variable is unset or empty after direct RunRequest.APIKey
|
||||
@@ -81,31 +81,40 @@ var (
|
||||
ErrValidation = errors.New("failed to validate output")
|
||||
)
|
||||
|
||||
// Engine inspects prompts and profiles and prepares and runs Promptkit prompt
|
||||
// requests.
|
||||
// Engine inspects prompts and profiles, enumerates prompt requirements, and
|
||||
// prepares and runs Promptkit prompt requests.
|
||||
//
|
||||
// An Engine is safe for concurrent calls to [Engine.InspectPrompt],
|
||||
// [Engine.InspectProfile], [Engine.Prepare], [Engine.PrepareExecution],
|
||||
// [Engine.Run], and [Engine.RunPrepared]. Each Engine owns independent
|
||||
// backend-capacity pools that coordinate Run and RunPrepared admission and
|
||||
// model generation. Injected collaborators may still be invoked concurrently
|
||||
// across different backend pools or for unlimited backends.
|
||||
// An Engine is safe for concurrent calls to [Engine.WithNamespaces],
|
||||
// [Engine.InspectConfiguration],
|
||||
// [Engine.InspectPrompt],
|
||||
// [Engine.InspectRequirements], [Engine.InspectProfile], [Engine.Prepare], [Engine.PrepareExecution],
|
||||
// [Engine.Run], [Engine.RunPrepared], [Engine.SmokeTest], [Engine.ListProfiles],
|
||||
// [Engine.ListPrompts], [Engine.ListBackends], [Engine.ListNamespaces], and
|
||||
// [Engine.ListCollections]. Engines constructed separately by [NewEngine] own
|
||||
// independent backend-capacity pools that coordinate Run, RunPrepared, and
|
||||
// inference SmokeTest admission and model generation. Namespace views returned
|
||||
// by [Engine.WithNamespaces] share their underlying engine's pools, clients,
|
||||
// and prepared-execution owner. Injected collaborators may still be invoked
|
||||
// concurrently across different backend pools or for unlimited backends.
|
||||
type Engine struct {
|
||||
internal *internalengine.Engine
|
||||
}
|
||||
|
||||
// Config selects the directory-backed sources and built-in model-client
|
||||
// transport used by [NewEngine]. Config has no stable JSON representation.
|
||||
// ConfigurationSources selects ordered ConfigKit underlays and an optional
|
||||
// highest-priority main input. Each underlay's inputs strictly compose within
|
||||
// their group; later underlays overlay earlier ones. A nil Main uses
|
||||
// ConfigKit's empty source. A non-nil Main is passed through unchanged and
|
||||
// may fail construction if invalid.
|
||||
type ConfigurationSources struct {
|
||||
Underlays []configkit.Underlay
|
||||
Main *configkit.Input
|
||||
}
|
||||
|
||||
// Config selects configuration sources and built-in model-client transport
|
||||
// used by [NewEngine]. Config has no stable JSON representation.
|
||||
type Config struct {
|
||||
// PromptDir is the directory searched recursively for prompt definitions.
|
||||
// It is required unless a WithPromptFS or WithPromptFile option supplies the
|
||||
// prompt source.
|
||||
PromptDir string
|
||||
// ProfileDir is an optional ordinary configured source whose profiles take
|
||||
// precedence over application fallback and maintained catalog profiles. An
|
||||
// empty value selects the lower-precedence sources unless a profile-source
|
||||
// option supplies the ordinary source.
|
||||
ProfileDir string
|
||||
// Sources combines maintained definitions with caller-selected inputs.
|
||||
Sources ConfigurationSources
|
||||
// SchemaDir is the root for JSON Schema files. An empty value uses the
|
||||
// current directory. WithSchemaFS or WithSchemaFile replaces this source.
|
||||
SchemaDir string
|
||||
@@ -128,12 +137,9 @@ type Config struct {
|
||||
// Option customizes engine construction.
|
||||
//
|
||||
// NewEngine applies options in argument order and ignores nil options. Within
|
||||
// each prompt-source, ordinary-profile-source, fallback-profile-source,
|
||||
// in-memory-profile, schema-source, model-client, and artifact-reader
|
||||
// category, the last non-nil valid option replaces earlier options in that
|
||||
// category. WithBackend is the additive exception: unique registrations
|
||||
// accumulate, and a repeated backend ID is an error rather than a replacement.
|
||||
// An invalid option fails construction even if a later option would replace it.
|
||||
// each schema-source, model-client, and artifact-reader category, the last
|
||||
// non-nil valid option replaces earlier options. An invalid option fails
|
||||
// construction even if a later option would replace it.
|
||||
type Option interface {
|
||||
apply(*engineOptions) error
|
||||
}
|
||||
@@ -181,102 +187,6 @@ func WithArtifactReader(reader ArtifactReader) Option {
|
||||
})
|
||||
}
|
||||
|
||||
// WithPromptFS loads prompt definitions from fsys under root.
|
||||
//
|
||||
// The source uses the same strict prompt YAML rules as configured prompt
|
||||
// directories, and prompt content_file paths resolve within this source.
|
||||
// fsys must be non-nil and root must be non-empty; otherwise NewEngine fails
|
||||
// with ErrInvalidConfig. This option replaces Config.PromptDir and earlier
|
||||
// prompt-source options.
|
||||
func WithPromptFS(fsys fs.FS, root string) Option {
|
||||
return optionFunc(func(options *engineOptions) error {
|
||||
return options.internal.SetPromptFS(fsys, root)
|
||||
})
|
||||
}
|
||||
|
||||
// WithPromptFile loads prompt definitions from the single prompt file at path.
|
||||
//
|
||||
// Relative prompt content_file paths resolve from the file's directory. path
|
||||
// must name an existing non-directory file when NewEngine applies the option.
|
||||
// This option replaces Config.PromptDir and earlier prompt-source options.
|
||||
func WithPromptFile(path string) Option {
|
||||
return optionFunc(func(options *engineOptions) error {
|
||||
return options.internal.SetPromptFile(path)
|
||||
})
|
||||
}
|
||||
|
||||
// WithProfileFS loads execution profiles from fsys under root.
|
||||
//
|
||||
// Profiles from this ordinary configured source take precedence over
|
||||
// application fallback and built-in profiles. Profile YAML must use api_key_env
|
||||
// for environment-based credentials; raw API keys are rejected. fsys must be
|
||||
// non-nil and root must be non-empty; otherwise NewEngine fails with
|
||||
// ErrInvalidConfig. This option replaces Config.ProfileDir and earlier file or
|
||||
// FS profile-source options, but remains below WithProfiles in precedence.
|
||||
func WithProfileFS(fsys fs.FS, root string) Option {
|
||||
return optionFunc(func(options *engineOptions) error {
|
||||
return options.internal.SetProfileFS(fsys, root)
|
||||
})
|
||||
}
|
||||
|
||||
// WithProfileFile loads execution profiles from the single profile file at path.
|
||||
//
|
||||
// The profile takes precedence over application fallback and built-in profiles.
|
||||
// Profile YAML must use api_key_env for environment-based credentials; raw API
|
||||
// keys are rejected. path must name an existing non-directory file when
|
||||
// NewEngine applies the option. This option replaces Config.ProfileDir and
|
||||
// earlier file or FS profile-source options, but remains below WithProfiles in
|
||||
// precedence.
|
||||
func WithProfileFile(path string) Option {
|
||||
return optionFunc(func(options *engineOptions) error {
|
||||
return options.internal.SetProfileFile(path)
|
||||
})
|
||||
}
|
||||
|
||||
// WithFallbackProfileFS supplies application-owned fallback profile
|
||||
// definitions from fsys under root.
|
||||
//
|
||||
// Profile lookup checks, in order, profiles supplied by WithProfiles; the
|
||||
// ordinary configured source selected by WithProfileFile, WithProfileFS, or
|
||||
// Config.ProfileDir; this fallback source; and Promptkit's maintained catalog
|
||||
// profiles. Each source supplies a complete profile definition; profile fields
|
||||
// are not merged between sources. Only an absent profile ID proceeds to the
|
||||
// next source. A matching read, parse, duplicate, validation, or credential
|
||||
// format failure stops resolution.
|
||||
//
|
||||
// Files use the ordinary strict profile YAML and api_key_env credential rules.
|
||||
// Loading and validation are lazy: NewEngine validates this option's arguments
|
||||
// but does not read profile files. fsys must be non-nil and root must be
|
||||
// nonblank; otherwise NewEngine returns an error matching ErrInvalidConfig.
|
||||
// Repeating this option replaces the earlier valid fallback source.
|
||||
//
|
||||
// This option controls profile-definition lookup, not provider or generation
|
||||
// failover.
|
||||
func WithFallbackProfileFS(fsys fs.FS, root string) Option {
|
||||
return optionFunc(func(options *engineOptions) error {
|
||||
return options.internal.SetFallbackProfileFS(fsys, root)
|
||||
})
|
||||
}
|
||||
|
||||
// WithProfiles configures in-memory profiles that take precedence over
|
||||
// ordinary configured, application fallback, and built-in profiles.
|
||||
//
|
||||
// NewEngine locally validates and copies every profile. IDs must be unique
|
||||
// within one call. An invalid local definition, duplicate ID, or unsupported
|
||||
// ExtraParams value makes construction fail with ErrInvalidConfig. A derived
|
||||
// profile's base reference and resolved target completeness are checked when it
|
||||
// is selected or inspected. Repeating WithProfiles replaces the complete
|
||||
// earlier in-memory set rather than merging it.
|
||||
func WithProfiles(profiles ...Profile) Option {
|
||||
return optionFunc(func(options *engineOptions) error {
|
||||
definitions := make([]domain.ExecutionProfile, len(profiles))
|
||||
for index, publicProfile := range profiles {
|
||||
definitions[index] = toDomainProfile(publicProfile)
|
||||
}
|
||||
return options.internal.SetMemoryProfiles(definitions)
|
||||
})
|
||||
}
|
||||
|
||||
// WithSchemaFS loads JSON Schema documents from fsys under root.
|
||||
//
|
||||
// Prompt schema_path values resolve within this source when schema validation
|
||||
@@ -302,14 +212,15 @@ func WithSchemaFile(path string) Option {
|
||||
|
||||
// NewEngine constructs an Engine from configuration and options.
|
||||
//
|
||||
// Options are applied in order according to [Option]. PromptDir is required
|
||||
// unless a prompt-source option is present. Construction validates option
|
||||
// arguments, in-memory profiles, and backend registrations but defers reading
|
||||
// and validating prompt, file-backed profile, and schema contents until Prepare
|
||||
// or Run needs them.
|
||||
// ConfigKit composes caller-selected underlays in their given order, followed
|
||||
// by the optional main input. Construction validates the final effective
|
||||
// backend, profile, and prompt definitions and captures referenced prompt
|
||||
// content. A missing prompt collection is allowed. Schema documents and
|
||||
// request artifacts remain runtime resources.
|
||||
//
|
||||
// NewEngine returns an error matching ErrInvalidConfig for invalid
|
||||
// configuration, options, or backend-capacity policies. Each constructed
|
||||
// configuration or options, preserving a ConfigKit LoadError when loading
|
||||
// fails. Each constructed
|
||||
// Engine has independent backend-capacity pools. Construction does not perform
|
||||
// model requests or require credentials. A negative Config.TransientRetries
|
||||
// is invalid.
|
||||
@@ -320,28 +231,27 @@ func NewEngine(cfg Config, opts ...Option) (*Engine, error) {
|
||||
continue
|
||||
}
|
||||
if err := opt.apply(&options); err != nil {
|
||||
return nil, fmt.Errorf("%w: %v", ErrInvalidConfig, err)
|
||||
return nil, fmt.Errorf("%w: %w", ErrInvalidConfig, err)
|
||||
}
|
||||
}
|
||||
|
||||
assembled, err := internalengine.New(
|
||||
internalengine.Config{PromptDir: cfg.PromptDir, ProfileDir: cfg.ProfileDir, SchemaDir: cfg.SchemaDir, Timeout: cfg.Timeout, HTTPClient: cfg.HTTPClient, TransientRetries: copyIntPtr(cfg.TransientRetries)},
|
||||
internalengine.Config{Sources: internalengine.ConfigurationSources{Underlays: cfg.Sources.Underlays, Main: cfg.Sources.Main}, SchemaDir: cfg.SchemaDir, Timeout: cfg.Timeout, HTTPClient: cfg.HTTPClient, TransientRetries: copyIntPtr(cfg.TransientRetries)},
|
||||
options.internal,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: %v", ErrInvalidConfig, err)
|
||||
return nil, fmt.Errorf("%w: %w", ErrInvalidConfig, err)
|
||||
}
|
||||
return &Engine{internal: assembled}, nil
|
||||
}
|
||||
|
||||
// InspectPrompt resolves one explicit prompt definition without selecting a
|
||||
// profile or starting execution work.
|
||||
// InspectPrompt resolves one prompt ID without selecting a profile or starting
|
||||
// execution work.
|
||||
//
|
||||
// InspectPrompt requires a nonblank promptID. It passes nonblank promptID and
|
||||
// promptVersion values unchanged to the engine's ordinary, case-sensitive
|
||||
// prompt selection. An empty version succeeds only when that source has one
|
||||
// selected ID; a nonempty version selects one exact ID/version pair. The
|
||||
// configured prompt source is used without merging, fallback, or enumeration.
|
||||
// InspectPrompt requires a nonblank promptID and performs an exact,
|
||||
// case-sensitive lookup in the captured configuration snapshot. A
|
||||
// namespace-scoped view qualifies slashless caller IDs before lookup; already
|
||||
// qualified IDs pass through. Prompt version labels do not select definitions.
|
||||
//
|
||||
// A successful result proves that the selected definition and any referenced
|
||||
// message content files were structurally loaded. Inputs are returned in
|
||||
@@ -349,32 +259,30 @@ func NewEngine(cfg Config, opts ...Option) (*Engine, error) {
|
||||
// resolved. OutputContract is the normalized declared contract, with a JSON
|
||||
// Schema path when declared but without loading or compiling that schema.
|
||||
// PromptHash is the same opaque equality value as PreparedRun.PromptHash for
|
||||
// the selected definition and observed source state; its spelling, length,
|
||||
// the selected captured definition; its spelling, length,
|
||||
// encoding, algorithm, and security properties are not contracts.
|
||||
//
|
||||
// This method does not return prompt bodies, templates, source paths, schemas,
|
||||
// rendered messages, or execution settings. It does not resolve a profile or
|
||||
// credential, read artifacts or schemas, render, validate, admit capacity,
|
||||
// contact a provider, or generate model output. The returned PromptInspection
|
||||
// and its input slice are caller-owned. Filesystem-backed inspection is a
|
||||
// point-in-time lookup and does not freeze a definition for later execution.
|
||||
// and its input slice are caller-owned. Inspection does not reload configuration
|
||||
// or content files; construct another Engine to observe source changes.
|
||||
//
|
||||
// A nil Engine returns an error matching ErrInvalidConfig. A blank prompt ID
|
||||
// matches ErrInvalidRequest. An absent exact ID or version matches
|
||||
// ErrPromptNotFound and not ErrPromptLoad. Malformed, unreadable, duplicate,
|
||||
// ambiguous, referenced-content, or hashing failures match ErrPromptLoad.
|
||||
// Cancellation during lookup matches ErrPromptLoad while preserving the
|
||||
// context error. InspectPrompt returns no partial result on error.
|
||||
// matches ErrInvalidRequest. An absent exact ID matches ErrPromptNotFound and
|
||||
// not ErrPromptLoad. Invalid definitions and resource failures reject
|
||||
// construction instead. Cancellation during lookup preserves the context
|
||||
// error. InspectPrompt returns no partial result on error.
|
||||
func (e *Engine) InspectPrompt(
|
||||
ctx context.Context,
|
||||
promptID string,
|
||||
promptVersion string,
|
||||
) (*PromptInspection, error) {
|
||||
if e == nil || e.internal == nil {
|
||||
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
|
||||
}
|
||||
|
||||
inspection, err := e.internal.InspectPrompt(ctx, promptID, promptVersion)
|
||||
inspection, err := e.internal.InspectPrompt(ctx, promptID)
|
||||
if err != nil {
|
||||
return nil, mapPublicError(err)
|
||||
}
|
||||
@@ -384,10 +292,9 @@ func (e *Engine) InspectPrompt(
|
||||
// InspectProfile resolves one explicit profile without selecting a prompt or
|
||||
// starting execution work.
|
||||
//
|
||||
// InspectProfile trims surrounding whitespace from profileID and looks up the
|
||||
// resulting nonblank ID exactly and case-sensitively through the engine's
|
||||
// in-memory, ordinary configured-source, application fallback, and built-in
|
||||
// profile precedence. It applies the framework timeout baseline, selected
|
||||
// InspectProfile looks up a nonblank profileID exactly and case-sensitively
|
||||
// in the captured effective configuration. It applies the framework timeout
|
||||
// baseline, selected
|
||||
// backend, and then selected profile to EffectiveModelParams without a request
|
||||
// override. BackendID is empty for an endpoint-only profile.
|
||||
//
|
||||
@@ -398,16 +305,15 @@ func (e *Engine) InspectPrompt(
|
||||
// absent or blank named environment variable is not an error.
|
||||
//
|
||||
// The returned ProfileInspection and all nested mutable values are
|
||||
// caller-owned. Filesystem-backed inspection is a point-in-time lookup and
|
||||
// does not freeze the profile for a later execution. This method does not load
|
||||
// caller-owned. Inspection does not reload configuration; construct another
|
||||
// Engine to observe source changes. This method does not load
|
||||
// a prompt, render, read artifacts or schemas, admit backend capacity, contact
|
||||
// a provider, or generate model output.
|
||||
//
|
||||
// A nil Engine returns an error matching ErrInvalidConfig. A blank profile ID
|
||||
// matches ErrInvalidRequest. An absent exact ID matches ErrProfileNotFound and
|
||||
// not ErrProfileLoad. Malformed or unreadable profile data, an unknown backend,
|
||||
// or an invalid resolved target matches ErrProfileLoad. Cancellation during
|
||||
// profile loading matches ErrProfileLoad while preserving the context error.
|
||||
// not ErrProfileLoad. Invalid definitions or unknown backends reject
|
||||
// construction instead. Cancellation preserves the context error.
|
||||
// InspectProfile returns no partial result on error.
|
||||
func (e *Engine) InspectProfile(ctx context.Context, profileID string) (*ProfileInspection, error) {
|
||||
if e == nil || e.internal == nil {
|
||||
@@ -441,6 +347,32 @@ func (e *Engine) InspectProfile(ctx context.Context, profileID string) (*Profile
|
||||
// relationship to ctx.Err is promised. Prepare returns no partial result on
|
||||
// error. It validates RunRequest.TransientRetries but makes no model attempts.
|
||||
func (e *Engine) Prepare(ctx context.Context, req RunRequest) (*PreparedRun, error) {
|
||||
prepared, err := e.prepareRequest(ctx, req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return fromDomainPreparedRun(prepared), nil
|
||||
}
|
||||
|
||||
// ValidateRequest reports whether req passes the same preparation path used by
|
||||
// [Engine.Prepare] at the time of this call. A nil error means this concrete
|
||||
// request passed preparation. It checks only this invocation and returns errors
|
||||
// as encountered rather than aggregating them. All supplied inputs follow the
|
||||
// same loading behavior as [Engine.Prepare], including inputs unused by the
|
||||
// selected template branch. The method does not call an LLM or retain a
|
||||
// prepared snapshot or executable handle, but preparation may still perform
|
||||
// resource I/O and incur its associated time and memory costs.
|
||||
//
|
||||
// Its errors and context handling follow [Engine.Prepare], including an error
|
||||
// matching ErrInvalidConfig when the Engine is nil. Success does not guarantee
|
||||
// provider acceptance or later execution success. Use [Engine.PrepareExecution]
|
||||
// when execution must use the snapshot produced by preparation.
|
||||
func (e *Engine) ValidateRequest(ctx context.Context, req RunRequest) error {
|
||||
_, err := e.prepareRequest(ctx, req)
|
||||
return err
|
||||
}
|
||||
|
||||
func (e *Engine) prepareRequest(ctx context.Context, req RunRequest) (*domain.PreparedRun, error) {
|
||||
ctx = observation.WithoutExecutionState(ctx)
|
||||
if e == nil || e.internal == nil {
|
||||
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
|
||||
@@ -455,15 +387,16 @@ func (e *Engine) Prepare(ctx context.Context, req RunRequest) (*PreparedRun, err
|
||||
if err != nil {
|
||||
return nil, mapPublicError(err)
|
||||
}
|
||||
return fromDomainPreparedRun(prepared), nil
|
||||
return prepared, nil
|
||||
}
|
||||
|
||||
// PrepareExecution completely prepares a prompt request without calling the
|
||||
// configured LLMClient or reserving backend admission capacity. Validated
|
||||
// RunRequest.AppendedMessages are included in the frozen effective messages.
|
||||
//
|
||||
// The returned opaque handle is bound to this Engine and permits one
|
||||
// [Engine.RunPrepared] invocation. Preparation freezes the selected sources,
|
||||
// The returned opaque handle is bound to this Engine's underlying execution
|
||||
// owner, shared with its namespace views, and permits one [Engine.RunPrepared]
|
||||
// invocation in total. Preparation freezes the selected sources,
|
||||
// rendered messages, effective settings, inputs, provider structured-output
|
||||
// metadata, and validation resources needed by that invocation. The handle
|
||||
// retains a direct RunRequest.APIKey only in private execution state;
|
||||
@@ -495,8 +428,9 @@ func (e *Engine) PrepareExecution(ctx context.Context, req RunRequest) (*Prepare
|
||||
}
|
||||
|
||||
// Run prepares a request, invokes the configured LLMClient, and validates the
|
||||
// generated output. Each call resolves current sources and composes a fresh,
|
||||
// stateless effective prompt with any validated RunRequest.AppendedMessages.
|
||||
// generated output. Each call uses the Engine's captured configuration and
|
||||
// composes a fresh effective prompt with any validated
|
||||
// RunRequest.AppendedMessages.
|
||||
//
|
||||
// A content-validation failure is a successful run whose
|
||||
// RunResult.Validation has Status ValidationFailed. When its output contract
|
||||
@@ -574,6 +508,11 @@ func (e *Engine) run(ctx context.Context, req RunRequest) (*RunResult, error) {
|
||||
// RunPrepared atomically claims and executes a handle created by
|
||||
// [Engine.PrepareExecution].
|
||||
//
|
||||
// The originating Engine and namespace views sharing its underlying owner may
|
||||
// claim the handle. Execution uses its captured selections without applying the
|
||||
// invoking view's namespaces. An Engine constructed separately by [NewEngine]
|
||||
// is a foreign owner.
|
||||
//
|
||||
// A valid owning-Engine invocation consumes the handle's one attempt before
|
||||
// credential revalidation, backend admission, generation, or validation.
|
||||
// Cancellation, capacity rejection, generation failure, operational
|
||||
|
||||
3705
engine_test.go
3705
engine_test.go
File diff suppressed because it is too large
Load Diff
10
errors.go
10
errors.go
@@ -72,10 +72,6 @@ func publicErrorFor(err error) error {
|
||||
return ErrPromptLoad
|
||||
case errors.Is(err, usecase.ErrProfileLoad):
|
||||
return ErrProfileLoad
|
||||
case errors.Is(err, promptdef.ErrInvalidYAML), errors.Is(err, promptdef.ErrInvalidPromptDefinition):
|
||||
return ErrPromptLoad
|
||||
case isProfileLoadCause(err):
|
||||
return ErrProfileLoad
|
||||
case errors.Is(err, usecase.ErrAPIKeyEnvMissing):
|
||||
return errors.Join(ErrInvalidRequest, ErrAPIKeyEnvMissing)
|
||||
case errors.Is(err, capacity.ErrCapacityExceeded):
|
||||
@@ -94,9 +90,3 @@ func publicErrorFor(err error) error {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func isProfileLoadCause(err error) bool {
|
||||
return errors.Is(err, profile.ErrInvalidYAML) ||
|
||||
errors.Is(err, profile.ErrInvalidProfile) ||
|
||||
errors.Is(err, profile.ErrRawAPIKeyNotAllowed)
|
||||
}
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
@@ -78,14 +79,24 @@ type summary struct {
|
||||
|
||||
func run() (summary, error) {
|
||||
transport := &scriptedTransport{}
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{HTTPClient: &http.Client{Transport: transport}},
|
||||
promptkit.WithPromptFS(assets, "prompts"),
|
||||
promptkit.WithSchemaFS(assets, "schemas"),
|
||||
promptkit.WithBackend(promptkit.Backend{ID: "offline-backend", Endpoint: "https://example.invalid/v1", ConcurrencyLimit: 1}),
|
||||
promptkit.WithProfiles(
|
||||
promptkit.Profile{ID: "proposal-profile", BackendID: "offline-backend", Model: "proposal-model"},
|
||||
promptkit.Profile{ID: "assessment-profile", BackendID: "offline-backend", Model: "assessment-model"},
|
||||
))
|
||||
inputs, err := promptkit.FilesystemInputs("example prompts", assets, "prompts")
|
||||
if err != nil {
|
||||
return summary{}, err
|
||||
}
|
||||
backendID, proposalModel, assessmentModel := "offline-backend", "proposal-model", "assessment-model"
|
||||
inputs = append(inputs,
|
||||
configkit.Input{Source: promptkit.BackendSource("example backend", map[string]promptkit.BackendConfig{
|
||||
backendID: promptkit.LocalBackend("https://example.invalid/v1", 1),
|
||||
})},
|
||||
configkit.Input{Source: promptkit.ProfileSource("example profiles", map[string]promptkit.ProfileConfig{
|
||||
"proposal-profile": {Backend: &backendID, Model: &proposalModel},
|
||||
"assessment-profile": {Backend: &backendID, Model: &assessmentModel},
|
||||
})},
|
||||
)
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{
|
||||
Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: inputs}}},
|
||||
HTTPClient: &http.Client{Transport: transport},
|
||||
}, promptkit.WithSchemaFS(assets, "schemas"))
|
||||
if err != nil {
|
||||
return summary{}, err
|
||||
}
|
||||
|
||||
@@ -1,17 +1,18 @@
|
||||
id: example.assessment
|
||||
version: "1.0.0"
|
||||
default_profile: assessment-profile
|
||||
description: Assess a structured proposal independently of schema validity.
|
||||
inputs:
|
||||
- name: proposal
|
||||
required: true
|
||||
content_type: application/json
|
||||
messages:
|
||||
- role: system
|
||||
content: Return a concise assessment as JSON.
|
||||
- role: user
|
||||
content: '{{input "proposal"}}'
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: assessment.schema.json
|
||||
prompts:
|
||||
example.assessment:
|
||||
version: "1.0.0"
|
||||
default_profile: assessment-profile
|
||||
description: Assess a structured proposal independently of schema validity.
|
||||
inputs:
|
||||
- name: proposal
|
||||
required: true
|
||||
content_type: application/json
|
||||
messages:
|
||||
- role: system
|
||||
content: Return a concise assessment as JSON.
|
||||
- role: user
|
||||
content: '{{input "proposal"}}'
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: assessment.schema.json
|
||||
|
||||
@@ -1,17 +1,18 @@
|
||||
id: example.proposal
|
||||
version: "1.0.0"
|
||||
default_profile: proposal-profile
|
||||
description: Propose a bounded improvement from a synthetic idea.
|
||||
inputs:
|
||||
- name: idea
|
||||
required: true
|
||||
content_type: text/plain
|
||||
messages:
|
||||
- role: system
|
||||
content: Return a concise proposal as JSON.
|
||||
- role: user
|
||||
content: '{{input "idea"}}'
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: proposal.schema.json
|
||||
prompts:
|
||||
example.proposal:
|
||||
version: "1.0.0"
|
||||
default_profile: proposal-profile
|
||||
description: Propose a bounded improvement from a synthetic idea.
|
||||
inputs:
|
||||
- name: idea
|
||||
required: true
|
||||
content_type: text/plain
|
||||
messages:
|
||||
- role: system
|
||||
content: Return a concise proposal as JSON.
|
||||
- role: user
|
||||
content: '{{input "idea"}}'
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: proposal.schema.json
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
backends:
|
||||
offline:
|
||||
endpoint: https://review.example.invalid/v1
|
||||
profiles:
|
||||
audit:
|
||||
backend: review/offline
|
||||
model: review-model
|
||||
prompts:
|
||||
check:
|
||||
default_profile: review/audit
|
||||
description: Inspect a review item without contacting a model provider.
|
||||
messages:
|
||||
- role: user
|
||||
content: Review the supplied synthetic example.
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
@@ -0,0 +1,5 @@
|
||||
version: "1"
|
||||
configkit:
|
||||
imports:
|
||||
- path: fragments/resources.yaml
|
||||
release_version: "1.4.0"
|
||||
@@ -0,0 +1,28 @@
|
||||
backends:
|
||||
offline:
|
||||
endpoint: https://example.invalid/v1
|
||||
profiles:
|
||||
base:
|
||||
backend: support/offline
|
||||
model: offline-model
|
||||
timeout_seconds: 90
|
||||
fast:
|
||||
extends: base
|
||||
timeout_seconds: 60
|
||||
prompts:
|
||||
prepare:
|
||||
version: "1.0.0"
|
||||
default_profile: support/fast
|
||||
description: Prepare a prompt without contacting a model provider.
|
||||
inputs:
|
||||
- name: note
|
||||
required: true
|
||||
content_type: text/plain
|
||||
messages:
|
||||
- role: system
|
||||
content: Summarize the note in one sentence.
|
||||
- role: user
|
||||
content_file: ../messages/note.md
|
||||
output:
|
||||
format: text
|
||||
validation_mode: basic
|
||||
@@ -0,0 +1 @@
|
||||
{{input "note"}}
|
||||
@@ -0,0 +1,4 @@
|
||||
# A later input intentionally replaces only the inherited profile timeout.
|
||||
profiles:
|
||||
fast:
|
||||
timeout_seconds: 30
|
||||
@@ -6,40 +6,143 @@ import (
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
type summary struct {
|
||||
PromptID string `json:"prompt_id"`
|
||||
PromptVersion string `json:"prompt_version"`
|
||||
SelectedProfile string `json:"selected_profile"`
|
||||
Model string `json:"model"`
|
||||
MessageCount int `json:"message_count"`
|
||||
PromptID string `json:"prompt_id"`
|
||||
PromptVersion *string `json:"prompt_version,omitempty"`
|
||||
SelectedProfile string `json:"selected_profile"`
|
||||
Model string `json:"model"`
|
||||
MessageCount int `json:"message_count"`
|
||||
ProfileTimeout int `json:"profile_timeout_seconds"`
|
||||
Namespaces []string `json:"namespaces"`
|
||||
Collections []collectionSummary `json:"collections"`
|
||||
ScopedPrompts []string `json:"scoped_prompts"`
|
||||
ReviewPrompts []string `json:"review_prompts"`
|
||||
}
|
||||
|
||||
type collectionSummary struct {
|
||||
Namespace string `json:"namespace"`
|
||||
ReleaseVersion *string `json:"release_version"`
|
||||
}
|
||||
|
||||
func main() {
|
||||
engine, err := promptkit.NewEngine(
|
||||
promptkit.Config{},
|
||||
promptkit.WithPromptFile("examples/go-library/prepare/prompt.yaml"),
|
||||
promptkit.WithProfiles(promptkit.Profile{
|
||||
ID: "offline-example",
|
||||
Endpoint: "https://example.invalid/v1",
|
||||
Model: "offline-model",
|
||||
}),
|
||||
support, err := promptkit.CollectionInput(
|
||||
"support",
|
||||
configkit.File("examples/go-library/prepare/collections/support/collection.yaml"),
|
||||
)
|
||||
if err != nil {
|
||||
exit(err)
|
||||
}
|
||||
|
||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||
PromptID: "example.prepare",
|
||||
Inputs: map[string]promptkit.ArtifactRef{
|
||||
"note": promptkit.Inline("Ada finished the migration review."),
|
||||
review, err := promptkit.CollectionInput(
|
||||
"review",
|
||||
configkit.File("examples/go-library/prepare/collections/review/collection.yaml"),
|
||||
)
|
||||
if err != nil {
|
||||
exit(err)
|
||||
}
|
||||
override, err := promptkit.CollectionInput(
|
||||
"support",
|
||||
configkit.File("examples/go-library/prepare/collections/support/operator-overrides.yaml"),
|
||||
)
|
||||
if err != nil {
|
||||
exit(err)
|
||||
}
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||
Underlays: []configkit.Underlay{
|
||||
{Inputs: []configkit.Input{support, review}},
|
||||
{Inputs: []configkit.Input{override}},
|
||||
},
|
||||
}})
|
||||
if err != nil {
|
||||
exit(err)
|
||||
}
|
||||
ctx := context.Background()
|
||||
scoped, err := engine.WithNamespaces(promptkit.Namespaces{
|
||||
Prompt: "support", Profile: "support", Backend: "support",
|
||||
})
|
||||
if err != nil {
|
||||
exit(err)
|
||||
}
|
||||
view, err := engine.InspectConfiguration()
|
||||
if err != nil {
|
||||
exit(err)
|
||||
}
|
||||
if _, found := view.Explain(configkit.Keys(
|
||||
"collections", "support", "profiles", "fast", "timeout_seconds",
|
||||
)); !found {
|
||||
exit(fmt.Errorf("collection profile override was not found"))
|
||||
}
|
||||
requirements, err := scoped.InspectRequirements(ctx, promptkit.RequirementInspectionRequest{
|
||||
PromptID: "prepare",
|
||||
})
|
||||
if err != nil {
|
||||
exit(err)
|
||||
}
|
||||
if !requirements.Complete || len(requirements.Inputs) != 1 ||
|
||||
requirements.Inputs[0].Name != "note" ||
|
||||
requirements.Inputs[0].Kind != promptkit.RequirementRequired {
|
||||
exit(fmt.Errorf("required note input was not discovered before preparation"))
|
||||
}
|
||||
|
||||
request := promptkit.RunRequest{
|
||||
PromptID: "prepare",
|
||||
ProfileID: "fast",
|
||||
Inputs: map[string]promptkit.ArtifactRef{
|
||||
"note": promptkit.Inline("Ada finished the migration review."),
|
||||
},
|
||||
}
|
||||
if err := scoped.ValidateRequest(ctx, request); err != nil {
|
||||
exit(err)
|
||||
}
|
||||
prepared, err := scoped.Prepare(ctx, request)
|
||||
if err != nil {
|
||||
exit(err)
|
||||
}
|
||||
profile, err := scoped.InspectProfile(ctx, "fast")
|
||||
if err != nil {
|
||||
exit(err)
|
||||
}
|
||||
if profile.EffectiveModelParams.TimeoutSeconds != 30 {
|
||||
exit(fmt.Errorf(
|
||||
"later profile override was not applied: timeout is %d seconds",
|
||||
profile.EffectiveModelParams.TimeoutSeconds,
|
||||
))
|
||||
}
|
||||
namespaces, err := scoped.ListNamespaces(ctx)
|
||||
if err != nil {
|
||||
exit(err)
|
||||
}
|
||||
collections, err := scoped.ListCollections(ctx)
|
||||
if err != nil {
|
||||
exit(err)
|
||||
}
|
||||
scopedPrompts, err := scoped.ListPrompts(ctx)
|
||||
if err != nil {
|
||||
exit(err)
|
||||
}
|
||||
reviewPrompts, err := scoped.ListPrompts(ctx, promptkit.NamespaceFilter{
|
||||
Mode: promptkit.NamespaceNamed, Name: "review",
|
||||
})
|
||||
if err != nil {
|
||||
exit(err)
|
||||
}
|
||||
scopedPromptIDs := make([]string, len(scopedPrompts))
|
||||
for index := range scopedPrompts {
|
||||
scopedPromptIDs[index] = scopedPrompts[index].ID
|
||||
}
|
||||
reviewPromptIDs := make([]string, len(reviewPrompts))
|
||||
for index := range reviewPrompts {
|
||||
reviewPromptIDs[index] = reviewPrompts[index].ID
|
||||
}
|
||||
collectionSummaries := make([]collectionSummary, len(collections))
|
||||
for index := range collections {
|
||||
collectionSummaries[index] = collectionSummary{
|
||||
Namespace: collections[index].Namespace, ReleaseVersion: collections[index].ReleaseVersion,
|
||||
}
|
||||
}
|
||||
|
||||
encoder := json.NewEncoder(os.Stdout)
|
||||
encoder.SetIndent("", " ")
|
||||
@@ -49,6 +152,11 @@ func main() {
|
||||
SelectedProfile: prepared.SelectedProfileID,
|
||||
Model: prepared.EffectiveModelParams.Model,
|
||||
MessageCount: len(prepared.Messages),
|
||||
ProfileTimeout: profile.EffectiveModelParams.TimeoutSeconds,
|
||||
Namespaces: namespaces,
|
||||
Collections: collectionSummaries,
|
||||
ScopedPrompts: scopedPromptIDs,
|
||||
ReviewPrompts: reviewPromptIDs,
|
||||
}); err != nil {
|
||||
exit(err)
|
||||
}
|
||||
|
||||
@@ -1,16 +0,0 @@
|
||||
id: example.prepare
|
||||
version: "1.0.0"
|
||||
default_profile: offline-example
|
||||
description: Prepare a prompt without contacting a model provider.
|
||||
inputs:
|
||||
- name: note
|
||||
required: true
|
||||
content_type: text/plain
|
||||
messages:
|
||||
- role: system
|
||||
content: Summarize the note in one sentence.
|
||||
- role: user
|
||||
content: '{{input "note"}}'
|
||||
output:
|
||||
format: text
|
||||
validation_mode: basic
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
@@ -38,16 +39,13 @@ type summary struct {
|
||||
}
|
||||
|
||||
func main() {
|
||||
engine, err := promptkit.NewEngine(
|
||||
promptkit.Config{},
|
||||
promptkit.WithPromptFile("examples/go-library/run/prompt.yaml"),
|
||||
promptkit.WithProfiles(promptkit.Profile{
|
||||
ID: "offline-example",
|
||||
Endpoint: "https://example.invalid/v1",
|
||||
Model: "offline-model",
|
||||
}),
|
||||
promptkit.WithLLMClient(deterministicClient{}),
|
||||
)
|
||||
endpoint, model := "https://example.invalid/v1", "offline-model"
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: promptkit.ProfileSource("example profile", map[string]promptkit.ProfileConfig{
|
||||
"offline-example": {Endpoint: &endpoint, Model: &model},
|
||||
})}}}},
|
||||
Main: &configkit.Input{Source: configkit.File("examples/go-library/run/prompt.yaml")},
|
||||
}}, promptkit.WithLLMClient(deterministicClient{}))
|
||||
if err != nil {
|
||||
exit(err)
|
||||
}
|
||||
|
||||
@@ -1,16 +1,17 @@
|
||||
id: example.run
|
||||
version: "1.0.0"
|
||||
default_profile: offline-example
|
||||
description: Run a prompt with a deterministic injected model client.
|
||||
inputs:
|
||||
- name: note
|
||||
required: true
|
||||
content_type: text/plain
|
||||
messages:
|
||||
- role: system
|
||||
content: Summarize the note in one sentence.
|
||||
- role: user
|
||||
content: '{{input "note"}}'
|
||||
output:
|
||||
format: text
|
||||
validation_mode: basic
|
||||
prompts:
|
||||
example.run:
|
||||
version: "1.0.0"
|
||||
default_profile: offline-example
|
||||
description: Run a prompt with a deterministic injected model client.
|
||||
inputs:
|
||||
- name: note
|
||||
required: true
|
||||
content_type: text/plain
|
||||
messages:
|
||||
- role: system
|
||||
content: Summarize the note in one sentence.
|
||||
- role: user
|
||||
content: '{{input "note"}}'
|
||||
output:
|
||||
format: text
|
||||
validation_mode: basic
|
||||
|
||||
@@ -18,9 +18,8 @@ func (r RunRequest) GoString() string {
|
||||
|
||||
func (r RunRequest) redactedString() string {
|
||||
return fmt.Sprintf(
|
||||
"promptkit.RunRequest{PromptID:%q PromptVersion:%q ProfileID:%q APIKeySet:%t Inputs:%d Vars:%d ExecutionSet:%t ValidationSet:%t AppendedMessages:%d}",
|
||||
"promptkit.RunRequest{PromptID:%q ProfileID:%q APIKeySet:%t Inputs:%d Vars:%d ExecutionSet:%t ValidationSet:%t AppendedMessages:%d}",
|
||||
r.PromptID,
|
||||
r.PromptVersion,
|
||||
r.ProfileID,
|
||||
r.APIKey != "",
|
||||
len(r.Inputs),
|
||||
|
||||
178
frozen_rendering_contract_test.go
Normal file
178
frozen_rendering_contract_test.go
Normal file
@@ -0,0 +1,178 @@
|
||||
package promptkit_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"testing/fstest"
|
||||
|
||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
const capturedPromptDocument = `
|
||||
prompts:
|
||||
captured:
|
||||
version: "1"
|
||||
default_profile: offline
|
||||
session_id: '{{.session}}'
|
||||
messages:
|
||||
- role: user
|
||||
content_file: body.tmpl
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
`
|
||||
|
||||
func capturedPromptEngine(t *testing.T, source fs.FS) *promptkit.Engine {
|
||||
t.Helper()
|
||||
inputs, err := promptkit.FilesystemInputs("application prompts", source, "config")
|
||||
if err != nil {
|
||||
t.Fatalf("select prompt inputs: %v", err)
|
||||
}
|
||||
endpoint, model := "https://example.invalid/v1", "offline-model"
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||
Underlays: []configkit.Underlay{
|
||||
{Inputs: inputs},
|
||||
{Inputs: []configkit.Input{{Source: promptkit.ProfileSource("offline profile", map[string]promptkit.ProfileConfig{
|
||||
"offline": {Endpoint: &endpoint, Model: &model},
|
||||
})}}},
|
||||
},
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatalf("construct engine: %v", err)
|
||||
}
|
||||
return engine
|
||||
}
|
||||
|
||||
func TestEngineCapturesPromptContentUntilReconstruction(t *testing.T) {
|
||||
files := fstest.MapFS{
|
||||
"config/prompt.yaml": &fstest.MapFile{Data: []byte(capturedPromptDocument)},
|
||||
"config/body.tmpl": &fstest.MapFile{Data: []byte("before {{.name}}")},
|
||||
}
|
||||
first := capturedPromptEngine(t, files)
|
||||
files["config/body.tmpl"] = &fstest.MapFile{Data: []byte("after {{.name}}")}
|
||||
request := promptkit.RunRequest{PromptID: "captured", Vars: map[string]string{"name": "Ada", "session": "session-1"}}
|
||||
oldPrepared, err := first.Prepare(context.Background(), request)
|
||||
if err != nil {
|
||||
t.Fatalf("prepare with existing engine: %v", err)
|
||||
}
|
||||
second := capturedPromptEngine(t, files)
|
||||
newPrepared, err := second.Prepare(context.Background(), request)
|
||||
if err != nil {
|
||||
t.Fatalf("prepare with rebuilt engine: %v", err)
|
||||
}
|
||||
if oldPrepared.Messages[0].Content != "before Ada" || newPrepared.Messages[0].Content != "after Ada" || oldPrepared.PromptHash == newPrepared.PromptHash {
|
||||
t.Fatalf("captured content or hash did not follow engine lifetime: old=%+v new=%+v", oldPrepared, newPrepared)
|
||||
}
|
||||
}
|
||||
|
||||
type unreadableAfterConstructionFS struct {
|
||||
fs.FS
|
||||
closed atomic.Bool
|
||||
reads atomic.Int64
|
||||
}
|
||||
|
||||
func (source *unreadableAfterConstructionFS) Open(name string) (fs.File, error) {
|
||||
if source.closed.Load() {
|
||||
source.reads.Add(1)
|
||||
return nil, fs.ErrPermission
|
||||
}
|
||||
return source.FS.Open(name)
|
||||
}
|
||||
|
||||
func TestCapturedTemplatesBindInputsIndependentlyAcrossConcurrentPreparations(t *testing.T) {
|
||||
source := &unreadableAfterConstructionFS{FS: fstest.MapFS{
|
||||
"config/prompt.yaml": &fstest.MapFile{Data: []byte(capturedPromptDocument)},
|
||||
"config/body.tmpl": &fstest.MapFile{Data: []byte(`{{.name}}: {{input "note"}}`)},
|
||||
}}
|
||||
engine := capturedPromptEngine(t, source)
|
||||
source.closed.Store(true)
|
||||
|
||||
const calls = 24
|
||||
var wait sync.WaitGroup
|
||||
errs := make(chan error, calls)
|
||||
for index := 0; index < calls; index++ {
|
||||
wait.Add(1)
|
||||
go func(index int) {
|
||||
defer wait.Done()
|
||||
name := fmt.Sprintf("caller-%d", index)
|
||||
body := fmt.Sprintf("body-%d", index)
|
||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||
PromptID: "captured", SessionID: "direct-session",
|
||||
Vars: map[string]string{"name": name},
|
||||
Inputs: map[string]promptkit.ArtifactRef{"note": promptkit.Inline(body)},
|
||||
})
|
||||
if err == nil && (prepared.Messages[0].Content != name+": "+body || prepared.SessionID != "direct-session") {
|
||||
err = fmt.Errorf("unexpected prepared content: %+v", prepared)
|
||||
}
|
||||
errs <- err
|
||||
}(index)
|
||||
}
|
||||
wait.Wait()
|
||||
close(errs)
|
||||
for err := range errs {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if source.reads.Load() != 0 {
|
||||
t.Fatalf("preparation reopened captured source %d times", source.reads.Load())
|
||||
}
|
||||
|
||||
_, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||
PromptID: "captured", Vars: map[string]string{"name": "Ada"},
|
||||
Inputs: map[string]promptkit.ArtifactRef{"note": promptkit.Inline("body")},
|
||||
})
|
||||
if !errors.Is(err, promptkit.ErrPromptRender) {
|
||||
t.Fatalf("missing session variable = %v, want runtime render error", err)
|
||||
}
|
||||
_, err = engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||
PromptID: "captured", SessionID: "direct-session", Vars: map[string]string{"name": "Ada"},
|
||||
})
|
||||
if !errors.Is(err, promptkit.ErrPromptRender) {
|
||||
t.Fatalf("missing input helper artifact = %v, want runtime render error", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreparationStillReadsRequestArtifactsAndSchemasAfterConstruction(t *testing.T) {
|
||||
endpoint, model := "https://example.invalid/v1", "offline-model"
|
||||
schemas := fstest.MapFS{}
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: promptkit.ProfileSource("profile", map[string]promptkit.ProfileConfig{
|
||||
"offline": {Endpoint: &endpoint, Model: &model},
|
||||
})}}}},
|
||||
Main: &configkit.Input{Source: configkit.Mapping("structured prompt", map[string]any{
|
||||
"prompts": map[string]any{"structured": map[string]any{
|
||||
"version": "1",
|
||||
"default_profile": "offline",
|
||||
"inputs": []any{map[string]any{"name": "note", "required": true}},
|
||||
"messages": []any{map[string]any{"role": "user", "content": `{{input "note"}}`}},
|
||||
"output": map[string]any{"format": "json", "validation_mode": "json_schema", "schema_path": "schema.json"},
|
||||
}},
|
||||
})},
|
||||
}}, promptkit.WithSchemaFS(schemas, "."))
|
||||
if err != nil {
|
||||
t.Fatalf("construct engine before request resources exist: %v", err)
|
||||
}
|
||||
schemas["schema.json"] = &fstest.MapFile{Data: []byte(`{"$schema":"https://json-schema.org/draft/2020-12/schema","type":"object"}`)}
|
||||
artifactPath := filepath.Join(t.TempDir(), "note.txt")
|
||||
if err := os.WriteFile(artifactPath, []byte("request artifact"), 0o600); err != nil {
|
||||
t.Fatalf("write request artifact: %v", err)
|
||||
}
|
||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "structured", Inputs: map[string]promptkit.ArtifactRef{
|
||||
"note": promptkit.File(artifactPath),
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatalf("prepare with resources supplied after construction: %v", err)
|
||||
}
|
||||
if len(prepared.Messages) != 1 || prepared.Messages[0].Content != "request artifact" || prepared.StructuredOutput == nil || prepared.StructuredOutput.JSONSchema == nil {
|
||||
t.Fatalf("runtime artifact or schema was not prepared: %+v", prepared)
|
||||
}
|
||||
}
|
||||
@@ -21,14 +21,14 @@ func TestBuiltInGenerationError(t *testing.T) {
|
||||
engine := newBuiltInGenerationErrorEngine(t, http.StatusUnprocessableEntity,
|
||||
`{"error":{"code":"`+codeMarker+`","type":"`+typeMarker+`","message":"`+messageMarker+`"}}`)
|
||||
|
||||
result, err := engine.Run(context.Background(), generationErrorRunRequest())
|
||||
result, err := engine.Run(context.Background(), simpleGenerationRequest())
|
||||
if result != nil {
|
||||
t.Fatalf("Run result = %#v, want nil", result)
|
||||
}
|
||||
assertGenerationError(t, err, http.StatusUnprocessableEntity, codeMarker, typeMarker, messageMarker)
|
||||
|
||||
preparedEngine := newBuiltInGenerationErrorEngine(t, http.StatusServiceUnavailable, `{"error":{}}`)
|
||||
prepared, err := preparedEngine.PrepareExecution(context.Background(), generationErrorRunRequest())
|
||||
prepared, err := preparedEngine.PrepareExecution(context.Background(), simpleGenerationRequest())
|
||||
if err != nil {
|
||||
t.Fatalf("PrepareExecution: %v", err)
|
||||
}
|
||||
@@ -43,7 +43,7 @@ func TestBuiltInGenerationErrorWithAppendedMessages(t *testing.T) {
|
||||
const messageMarker = "combined-request-provider-marker"
|
||||
engine := newBuiltInGenerationErrorEngine(t, http.StatusUnprocessableEntity,
|
||||
`{"error":{"message":"`+messageMarker+`"}}`)
|
||||
request := generationErrorRunRequest()
|
||||
request := simpleGenerationRequest()
|
||||
request.AppendedMessages = []promptkit.RenderedMessage{
|
||||
{Role: promptkit.RoleAssistant, Content: "previous response"},
|
||||
{Role: promptkit.RoleUser, Content: "consumer correction"},
|
||||
@@ -63,7 +63,7 @@ func TestBuiltInRepairGenerationError(t *testing.T) {
|
||||
messageMarker = "repair-message-marker"
|
||||
)
|
||||
calls := 0
|
||||
config := contractConfig(frameworkSchemaDir)
|
||||
config := simpleConfig("message")
|
||||
config.HTTPClient = &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
||||
calls++
|
||||
if calls == 1 {
|
||||
@@ -77,7 +77,7 @@ func TestBuiltInRepairGenerationError(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("NewEngine: %v", err)
|
||||
}
|
||||
req := generationErrorRunRequest()
|
||||
req := simpleGenerationRequest()
|
||||
req.Validation = &promptkit.OutputContract{
|
||||
Format: promptkit.FormatJSON,
|
||||
ValidationMode: promptkit.ValidationJSON,
|
||||
@@ -96,7 +96,7 @@ func TestBuiltInRepairGenerationError(t *testing.T) {
|
||||
|
||||
func TestBuiltInGenerationFailureCategories(t *testing.T) {
|
||||
transportCause := errors.New("private transport diagnostic")
|
||||
config := contractConfig(frameworkSchemaDir)
|
||||
config := simpleConfig("message")
|
||||
config.HTTPClient = &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
||||
return nil, transportCause
|
||||
})}
|
||||
@@ -104,7 +104,7 @@ func TestBuiltInGenerationFailureCategories(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
result, err := engine.Run(context.Background(), generationErrorRunRequest())
|
||||
result, err := engine.Run(context.Background(), simpleGenerationRequest())
|
||||
if result != nil || !errors.Is(err, promptkit.ErrLLMGenerate) || !errors.Is(err, promptkit.ErrTransport) || !errors.Is(err, transportCause) {
|
||||
t.Fatalf("transport result=%#v, error=%v", result, err)
|
||||
}
|
||||
@@ -122,7 +122,7 @@ func TestBuiltInGenerationFailureCategories(t *testing.T) {
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
engine := newBuiltInGenerationErrorEngine(t, http.StatusOK, tc.body)
|
||||
result, err := engine.Run(context.Background(), generationErrorRunRequest())
|
||||
result, err := engine.Run(context.Background(), simpleGenerationRequest())
|
||||
if result != nil || !errors.Is(err, promptkit.ErrLLMGenerate) || !errors.Is(err, promptkit.ErrMalformedResponse) || errors.Is(err, promptkit.ErrTransport) {
|
||||
t.Fatalf("malformed result=%#v, error=%v", result, err)
|
||||
}
|
||||
@@ -139,7 +139,7 @@ func TestBuiltInGenerationPreservesContextIdentity(t *testing.T) {
|
||||
{name: "deadline", cause: context.DeadlineExceeded},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
config := contractConfig(frameworkSchemaDir)
|
||||
config := simpleConfig("message")
|
||||
config.HTTPClient = &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
||||
return nil, tc.cause
|
||||
})}
|
||||
@@ -147,7 +147,7 @@ func TestBuiltInGenerationPreservesContextIdentity(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
result, err := engine.Run(context.Background(), generationErrorRunRequest())
|
||||
result, err := engine.Run(context.Background(), simpleGenerationRequest())
|
||||
if result != nil || !errors.Is(err, tc.cause) || !errors.Is(err, promptkit.ErrLLMGenerate) || !errors.Is(err, promptkit.ErrTransport) {
|
||||
t.Fatalf("context result=%#v, error=%v", result, err)
|
||||
}
|
||||
@@ -165,8 +165,8 @@ func TestInjectedGenerationFailureCategories(t *testing.T) {
|
||||
{name: "arbitrary error", client: &fakeLLMClient{err: errors.New("injected failure")}},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
engine := newContractEngineWithOptions(t, frameworkSchemaDir, promptkit.WithLLMClient(tc.client))
|
||||
result, err := engine.Run(context.Background(), generationErrorRunRequest())
|
||||
engine := newSimpleEngine(t, "message", promptkit.WithLLMClient(tc.client))
|
||||
result, err := engine.Run(context.Background(), simpleGenerationRequest())
|
||||
if result != nil || !errors.Is(err, promptkit.ErrLLMGenerate) || errors.Is(err, promptkit.ErrTransport) || errors.Is(err, promptkit.ErrMalformedResponse) != tc.malformed {
|
||||
t.Fatalf("injected result=%#v, error=%v", result, err)
|
||||
}
|
||||
@@ -207,7 +207,7 @@ func assertGenerationError(t *testing.T, err error, statusCode int, code, provid
|
||||
func newBuiltInGenerationErrorEngine(t *testing.T, statusCode int, body string) *promptkit.Engine {
|
||||
t.Helper()
|
||||
|
||||
config := contractConfig(frameworkSchemaDir)
|
||||
config := simpleConfig("message")
|
||||
noRetries := 0
|
||||
config.TransientRetries = &noRetries
|
||||
config.HTTPClient = &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
||||
@@ -224,12 +224,6 @@ func newBuiltInGenerationErrorEngine(t *testing.T, statusCode int, body string)
|
||||
return engine
|
||||
}
|
||||
|
||||
func generationErrorRunRequest() promptkit.RunRequest {
|
||||
return promptkit.RunRequest{
|
||||
PromptID: frameworkMarkdownSummaryPromptID,
|
||||
Inputs: map[string]promptkit.ArtifactRef{
|
||||
"transcript": promptkit.Inline("Rin opens the gate."),
|
||||
"glossary": promptkit.Inline("gate: A guarded passage."),
|
||||
},
|
||||
}
|
||||
func simpleGenerationRequest() promptkit.RunRequest {
|
||||
return promptkit.RunRequest{PromptID: "prompt"}
|
||||
}
|
||||
|
||||
16
go.mod
16
go.mod
@@ -1,12 +1,18 @@
|
||||
module gitea.maximumdirect.net/eric/promptkit
|
||||
|
||||
go 1.25.5
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v2 v2.20260926.0
|
||||
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v2 v2.20260926.0
|
||||
gitea.maximumdirect.net/eric/ConfigKit v0.8.0
|
||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
)
|
||||
|
||||
require golang.org/x/text v0.14.0 // indirect
|
||||
require (
|
||||
gitea.maximumdirect.net/eric/ReleaseKit v0.4.1 // indirect
|
||||
github.com/yuin/goldmark/v2 v2.1.6 // indirect
|
||||
go.yaml.in/yaml/v4 v4.0.0-rc.6 // indirect
|
||||
golang.org/x/mod v0.40.0 // indirect
|
||||
golang.org/x/text v0.14.0 // indirect
|
||||
)
|
||||
|
||||
tool gitea.maximumdirect.net/eric/ReleaseKit/cmd/releasekit
|
||||
|
||||
18
go.sum
18
go.sum
@@ -1,14 +1,16 @@
|
||||
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v2 v2.20260926.0 h1:wdab+R+lgMZxzYwhbAuCrW7zdSQ1cW9ovBR6UzLmZR4=
|
||||
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v2 v2.20260926.0/go.mod h1:a3VBbh9oaGXD6ruxhtwHj+AWW6DY/SSyby1QtMmYSUo=
|
||||
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v2 v2.20260926.0 h1:baCCIB6Ee5t0S4Zc++NTBIPcJEv5UPS6DlSrpZfykwg=
|
||||
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v2 v2.20260926.0/go.mod h1:mtgD1E0SGm0e/I8Ffhz+AYX7GN7RKUWJXehNFIKHs4Y=
|
||||
gitea.maximumdirect.net/eric/ConfigKit v0.8.0 h1:xCIhYnOHjw7rFvB+vIremPKGeROOxKTomnUEUipuQno=
|
||||
gitea.maximumdirect.net/eric/ConfigKit v0.8.0/go.mod h1:C82sbyKHYMV1OwaD2GJanFrwTfcZfjlW2JsGze5b0kA=
|
||||
gitea.maximumdirect.net/eric/ReleaseKit v0.4.1 h1:Qn+mdebYOzCVCEXgCbuGSO/WHmJtItwaQ8e3XwY3qtI=
|
||||
gitea.maximumdirect.net/eric/ReleaseKit v0.4.1/go.mod h1:LVodqNQ3OIKv9NBq+fFeyMByd1lHY25YOmmpah/NaNI=
|
||||
github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI=
|
||||
github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
|
||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ=
|
||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU=
|
||||
github.com/yuin/goldmark/v2 v2.1.6 h1:igy0q9JcqovoFXFx/XjtU8sHVhcUHavPj2sA3EoKJUE=
|
||||
github.com/yuin/goldmark/v2 v2.1.6/go.mod h1:G6M4/qOFtfn01/o14BU1UR2Lo5N3S9Qo7xCuz/sHjGQ=
|
||||
go.yaml.in/yaml/v4 v4.0.0-rc.6 h1:1h7H1ohdUh93/FyE4YaDa1Zh64K6VVbjF4K6WUxMtH4=
|
||||
go.yaml.in/yaml/v4 v4.0.0-rc.6/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0=
|
||||
golang.org/x/mod v0.40.0 h1:hUv+3cXcdRHz08UmSiOob7sadHig73uo5bkXxQ/tvUs=
|
||||
golang.org/x/mod v0.40.0/go.mod h1:0/weTWkPWGBikyTWAX3dkjVztMmBA5hM0DH6BElSupE=
|
||||
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
|
||||
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
|
||||
@@ -21,6 +21,7 @@ var (
|
||||
)
|
||||
|
||||
const fileReadChunkSize = 64 * 1024
|
||||
const fileReadInitialCapacityLimit = 1 << 20
|
||||
|
||||
// Reader resolves artifact references into actual artifacts.
|
||||
type Reader interface {
|
||||
@@ -133,7 +134,11 @@ func readFileArtifact(ctx context.Context, path string, open func(string) (artif
|
||||
return nil, fmt.Errorf("%w: %s", ErrUnsupportedFile, path)
|
||||
}
|
||||
|
||||
data := make([]byte, 0)
|
||||
capacity := 0
|
||||
if size := openedInfo.Size(); size > 0 {
|
||||
capacity = int(min(size, int64(fileReadInitialCapacityLimit)))
|
||||
}
|
||||
data := make([]byte, 0, capacity)
|
||||
chunk := make([]byte, fileReadChunkSize)
|
||||
for {
|
||||
if err := ctx.Err(); err != nil {
|
||||
@@ -153,6 +158,13 @@ func readFileArtifact(ctx context.Context, path string, open func(string) (artif
|
||||
return nil, fmt.Errorf("failed to read file %s: %w", path, readErr)
|
||||
}
|
||||
}
|
||||
if cap(data)-len(data) > len(data) {
|
||||
// A stale size hint should not make a small completed artifact retain a
|
||||
// disproportionately large backing array.
|
||||
compact := make([]byte, len(data))
|
||||
copy(compact, data)
|
||||
data = compact
|
||||
}
|
||||
|
||||
contentType := mime.TypeByExtension(filepath.Ext(path))
|
||||
if contentType == "" {
|
||||
|
||||
180
internal/artifact/reader_benchmark_test.go
Normal file
180
internal/artifact/reader_benchmark_test.go
Normal file
@@ -0,0 +1,180 @@
|
||||
package artifact
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"fmt"
|
||||
"mime"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
)
|
||||
|
||||
var artifactFileReadBenchmarkSink *domain.Artifact
|
||||
|
||||
func BenchmarkArtifactFileRead(b *testing.B) {
|
||||
for _, fixture := range makeArtifactReadBenchmarkFiles(b) {
|
||||
b.Run(fixture.name, func(b *testing.B) {
|
||||
reader := &fileReader{open: openArtifactFile}
|
||||
ref := domain.ArtifactRef{Type: domain.ArtifactRefFile, URI: fixture.path}
|
||||
checkArtifactFileRead(b, reader, ref, fixture.body)
|
||||
|
||||
b.ReportAllocs()
|
||||
b.SetBytes(int64(len(fixture.body)))
|
||||
b.ResetTimer()
|
||||
for range b.N {
|
||||
artifact, err := reader.Read(context.Background(), ref)
|
||||
if err != nil {
|
||||
b.Fatalf("read artifact: %v", err)
|
||||
}
|
||||
artifactFileReadBenchmarkSink = artifact
|
||||
}
|
||||
b.StopTimer()
|
||||
runtime.KeepAlive(artifactFileReadBenchmarkSink)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// BenchmarkRetainedArtifactFileRead measures live artifact bodies when an
|
||||
// opened file reports a size larger than the content that reaches EOF.
|
||||
func BenchmarkRetainedArtifactFileRead(b *testing.B) {
|
||||
for _, fixture := range []struct {
|
||||
name string
|
||||
body []byte
|
||||
}{
|
||||
{name: "small", body: bytes.Repeat([]byte("x"), 256<<10)},
|
||||
{name: "empty", body: []byte{}},
|
||||
} {
|
||||
b.Run(fixture.name, func(b *testing.B) {
|
||||
benchmarkRetainedArtifactFileRead(b, fixture.body)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func benchmarkRetainedArtifactFileRead(b *testing.B, body []byte) {
|
||||
path := filepath.Join(b.TempDir(), "overreported.txt")
|
||||
if err := os.WriteFile(path, body, 0o600); err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
mime.TypeByExtension(filepath.Ext(path))
|
||||
|
||||
reader := &fileReader{open: func(path string) (artifactFile, error) {
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &sizeReportingArtifactFile{artifactFile: file, reportedSize: 16 << 20}, nil
|
||||
}}
|
||||
ref := domain.ArtifactRef{Type: domain.ArtifactRefFile, URI: path}
|
||||
checkArtifactFileRead(b, reader, ref, body)
|
||||
|
||||
const retainedBatchSize = 4
|
||||
retained := make([]*domain.Artifact, retainedBatchSize)
|
||||
artifactFileReadBenchmarkSink = nil
|
||||
runtime.GC()
|
||||
var setupMemory runtime.MemStats
|
||||
runtime.ReadMemStats(&setupMemory)
|
||||
runtime.KeepAlive(reader)
|
||||
runtime.KeepAlive(ref)
|
||||
|
||||
b.ReportAllocs()
|
||||
b.SetBytes(int64(len(body)))
|
||||
b.ResetTimer()
|
||||
for iteration := 0; iteration < b.N; iteration++ {
|
||||
artifact, err := reader.Read(context.Background(), ref)
|
||||
if err != nil {
|
||||
b.Fatalf("read retained artifact: %v", err)
|
||||
}
|
||||
retained[iteration%len(retained)] = artifact
|
||||
}
|
||||
b.StopTimer()
|
||||
for i, artifact := range retained {
|
||||
if artifact != nil {
|
||||
continue
|
||||
}
|
||||
artifact, err := reader.Read(context.Background(), ref)
|
||||
if err != nil {
|
||||
b.Fatalf("fill retained artifact batch: %v", err)
|
||||
}
|
||||
retained[i] = artifact
|
||||
}
|
||||
runtime.GC()
|
||||
var retainedMemory runtime.MemStats
|
||||
runtime.ReadMemStats(&retainedMemory)
|
||||
b.ReportMetric(float64(int64(retainedMemory.HeapAlloc)-int64(setupMemory.HeapAlloc)), "retained-B")
|
||||
runtime.KeepAlive(retained)
|
||||
runtime.KeepAlive(reader)
|
||||
runtime.KeepAlive(ref)
|
||||
}
|
||||
|
||||
type artifactReadBenchmarkFile struct {
|
||||
name string
|
||||
path string
|
||||
body []byte
|
||||
}
|
||||
|
||||
func makeArtifactReadBenchmarkFiles(b *testing.B) []artifactReadBenchmarkFile {
|
||||
b.Helper()
|
||||
fixtures := []struct {
|
||||
name string
|
||||
size int
|
||||
}{
|
||||
{name: "small_4KiB", size: 4 << 10},
|
||||
{name: "1MiB", size: 1 << 20},
|
||||
{name: "large_4MiB", size: 4 << 20},
|
||||
}
|
||||
results := make([]artifactReadBenchmarkFile, 0, len(fixtures))
|
||||
for _, fixture := range fixtures {
|
||||
body := bytes.Repeat([]byte("x"), fixture.size)
|
||||
path := filepath.Join(b.TempDir(), fixture.name+".txt")
|
||||
if err := os.WriteFile(path, body, 0o600); err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
mime.TypeByExtension(filepath.Ext(path))
|
||||
results = append(results, artifactReadBenchmarkFile{
|
||||
name: fixture.name,
|
||||
path: path,
|
||||
body: body,
|
||||
})
|
||||
}
|
||||
return results
|
||||
}
|
||||
|
||||
func checkArtifactFileRead(b *testing.B, reader Reader, ref domain.ArtifactRef, want []byte) {
|
||||
b.Helper()
|
||||
artifact, err := reader.Read(context.Background(), ref)
|
||||
if err != nil {
|
||||
b.Fatalf("check artifact read: %v", err)
|
||||
}
|
||||
if !bytes.Equal(artifact.Body, want) || artifact.Size != int64(len(want)) {
|
||||
b.Fatalf("artifact size=%d body length=%d, want %d", artifact.Size, len(artifact.Body), len(want))
|
||||
}
|
||||
wantHash := fmt.Sprintf("%x", sha256.Sum256(want))
|
||||
if artifact.Hash != wantHash {
|
||||
b.Fatalf("artifact hash=%q, want %q", artifact.Hash, wantHash)
|
||||
}
|
||||
}
|
||||
|
||||
type sizeReportingArtifactFile struct {
|
||||
artifactFile
|
||||
reportedSize int64
|
||||
}
|
||||
|
||||
func (file *sizeReportingArtifactFile) Stat() (os.FileInfo, error) {
|
||||
info, err := file.artifactFile.Stat()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return reportedArtifactFileInfo{FileInfo: info, size: file.reportedSize}, nil
|
||||
}
|
||||
|
||||
type reportedArtifactFileInfo struct {
|
||||
os.FileInfo
|
||||
size int64
|
||||
}
|
||||
|
||||
func (info reportedArtifactFileInfo) Size() int64 { return info.size }
|
||||
@@ -3,7 +3,9 @@ package artifact
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
@@ -232,6 +234,64 @@ func TestFileReaderFailuresAndMetadata(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
func TestFileReaderUsesOpenedSizeOnlyAsACapacityHint(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
body []byte
|
||||
reportedSize int64
|
||||
}{
|
||||
{name: "overreported size", body: []byte("actual file content"), reportedSize: 128},
|
||||
{name: "overreported size with empty content", body: []byte{}, reportedSize: 16 << 20},
|
||||
{name: "underreported size", body: bytes.Repeat([]byte("x"), fileReadChunkSize+17), reportedSize: 1},
|
||||
{name: "very large hint with small content", body: []byte("small actual content"), reportedSize: int64(^uint64(0) >> 1)},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
filePath := filepath.Join(t.TempDir(), "artifact.txt")
|
||||
if err := os.WriteFile(filePath, tc.body, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reader := &fileReader{open: func(path string) (artifactFile, error) {
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
info, err := file.Stat()
|
||||
if err != nil {
|
||||
file.Close()
|
||||
return nil, err
|
||||
}
|
||||
return &reportedInfoFile{
|
||||
artifactFile: file,
|
||||
info: reportedFileSizeInfo{FileInfo: info, size: tc.reportedSize},
|
||||
}, nil
|
||||
}}
|
||||
|
||||
artifact, err := reader.Read(context.Background(), domain.ArtifactRef{
|
||||
Type: domain.ArtifactRefFile,
|
||||
URI: filePath,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("read artifact: %v", err)
|
||||
}
|
||||
if !bytes.Equal(artifact.Body, tc.body) {
|
||||
t.Fatalf("artifact body length=%d, want %d", len(artifact.Body), len(tc.body))
|
||||
}
|
||||
if artifact.Size != int64(len(tc.body)) {
|
||||
t.Fatalf("artifact size=%d, want %d", artifact.Size, len(tc.body))
|
||||
}
|
||||
if artifact.Name != filepath.Base(filePath) || artifact.URI != filePath || artifact.ContentType == "" {
|
||||
t.Fatalf("unexpected artifact metadata: name=%q URI=%q content type=%q", artifact.Name, artifact.URI, artifact.ContentType)
|
||||
}
|
||||
wantHash := fmt.Sprintf("%x", sha256.Sum256(tc.body))
|
||||
if artifact.Hash != wantHash {
|
||||
t.Fatalf("artifact hash=%q, want %q", artifact.Hash, wantHash)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileReaderCancelsAfterReadProgress(t *testing.T) {
|
||||
filePath := filepath.Join(t.TempDir(), "artifact.bin")
|
||||
content := bytes.Repeat([]byte("x"), fileReadChunkSize*2)
|
||||
@@ -268,6 +328,13 @@ func (f *reportedInfoFile) Stat() (os.FileInfo, error) {
|
||||
return f.info, nil
|
||||
}
|
||||
|
||||
type reportedFileSizeInfo struct {
|
||||
os.FileInfo
|
||||
size int64
|
||||
}
|
||||
|
||||
func (info reportedFileSizeInfo) Size() int64 { return info.size }
|
||||
|
||||
type cancelAfterProgressFile struct {
|
||||
artifactFile
|
||||
cancel context.CancelFunc
|
||||
|
||||
@@ -5,63 +5,38 @@ package backend
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/jsonvalue"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/llm"
|
||||
)
|
||||
|
||||
const (
|
||||
// OpenRouterID is the reserved ID of Promptkit's built-in OpenRouter
|
||||
// backend.
|
||||
// OpenRouterID is the conventional OpenRouter backend identifier.
|
||||
OpenRouterID = "openrouter"
|
||||
// RakestrawHomeID is the reserved ID of Promptkit's built-in Rakestrawhome
|
||||
// backend.
|
||||
// RakestrawHomeID is the conventional Rakestrawhome backend identifier.
|
||||
RakestrawHomeID = "rakestrawhome"
|
||||
|
||||
defaultQueueCapacity = 1024
|
||||
)
|
||||
|
||||
// ErrBackendNotFound identifies a registry lookup for an unknown backend ID.
|
||||
var ErrBackendNotFound = errors.New("backend not found")
|
||||
|
||||
var environmentVariableName = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`)
|
||||
|
||||
// Registry is an immutable collection of validated backend definitions.
|
||||
type Registry struct {
|
||||
backends map[string]domain.Backend
|
||||
}
|
||||
|
||||
// NewRegistry constructs a registry containing maintained definitions followed
|
||||
// by consumer additions. Every ID must be unique across both groups.
|
||||
func NewRegistry(maintained, additions []domain.Backend) (*Registry, error) {
|
||||
registry := &Registry{
|
||||
backends: make(map[string]domain.Backend, len(maintained)+len(additions)),
|
||||
}
|
||||
|
||||
definitions := make([]domain.Backend, 0, len(maintained)+len(additions))
|
||||
definitions = append(definitions, maintained...)
|
||||
definitions = append(definitions, additions...)
|
||||
|
||||
for _, definition := range definitions {
|
||||
definition.ID = strings.TrimSpace(definition.ID)
|
||||
if definition.ID == "" {
|
||||
return nil, errors.New("backend ID must not be blank")
|
||||
}
|
||||
if _, exists := registry.backends[definition.ID]; exists {
|
||||
return nil, fmt.Errorf("backend ID %q is already registered", definition.ID)
|
||||
}
|
||||
|
||||
normalized, err := NormalizeDefinition(definition)
|
||||
// NewEffectiveRegistry stores already-validated effective backend definitions
|
||||
// under their exact configuration keys. Configuration validation and
|
||||
// normalization precede this ownership boundary.
|
||||
func NewEffectiveRegistry(definitions map[string]domain.Backend) (*Registry, error) {
|
||||
registry := &Registry{backends: make(map[string]domain.Backend, len(definitions))}
|
||||
for id, definition := range definitions {
|
||||
definition.ID = id
|
||||
cloned, err := domain.CloneBackend(definition)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, fmt.Errorf("backend %q: %w", id, err)
|
||||
}
|
||||
registry.backends[normalized.ID] = normalized
|
||||
registry.backends[id] = cloned
|
||||
}
|
||||
|
||||
return registry, nil
|
||||
}
|
||||
|
||||
@@ -74,12 +49,11 @@ func (r *Registry) GetBackend(id string) (domain.Backend, error) {
|
||||
if !ok {
|
||||
return domain.Backend{}, fmt.Errorf("%w: %q", ErrBackendNotFound, id)
|
||||
}
|
||||
extraParams, err := jsonvalue.CopyMap(definition.ExtraParams)
|
||||
cloned, err := domain.CloneBackend(definition)
|
||||
if err != nil {
|
||||
return domain.Backend{}, fmt.Errorf("copy backend %q: %w", id, err)
|
||||
}
|
||||
definition.ExtraParams = extraParams
|
||||
return definition, nil
|
||||
return cloned, nil
|
||||
}
|
||||
|
||||
// CapacityPolicies returns a copy of the normalized policies for limited
|
||||
@@ -100,80 +74,3 @@ func (r *Registry) CapacityPolicies() map[string]domain.BackendCapacityPolicy {
|
||||
}
|
||||
return policies
|
||||
}
|
||||
|
||||
// NormalizeDefinition validates and defensively copies one backend definition.
|
||||
func NormalizeDefinition(definition domain.Backend) (domain.Backend, error) {
|
||||
endpoint, err := domain.NormalizeOpenAICompatibleBaseEndpoint(definition.Endpoint)
|
||||
if err != nil {
|
||||
return domain.Backend{}, fmt.Errorf("backend %q endpoint: %w", definition.ID, err)
|
||||
}
|
||||
definition.Endpoint = endpoint
|
||||
|
||||
definition.APIKeyEnv = strings.TrimSpace(definition.APIKeyEnv)
|
||||
if definition.APIKeyEnv != "" && !environmentVariableName.MatchString(definition.APIKeyEnv) {
|
||||
return domain.Backend{}, fmt.Errorf(
|
||||
"backend %q api key environment variable %q is invalid",
|
||||
definition.ID,
|
||||
definition.APIKeyEnv,
|
||||
)
|
||||
}
|
||||
|
||||
if definition.ConcurrencyLimit < 0 {
|
||||
return domain.Backend{}, fmt.Errorf(
|
||||
"backend %q concurrency limit must not be negative",
|
||||
definition.ID,
|
||||
)
|
||||
}
|
||||
if definition.QueueCapacity < 0 {
|
||||
return domain.Backend{}, fmt.Errorf(
|
||||
"backend %q queue capacity must not be negative",
|
||||
definition.ID,
|
||||
)
|
||||
}
|
||||
if definition.ConcurrencyLimit == 0 {
|
||||
if definition.QueueCapacitySet {
|
||||
return domain.Backend{}, fmt.Errorf(
|
||||
"backend %q queue capacity requires a positive concurrency limit",
|
||||
definition.ID,
|
||||
)
|
||||
}
|
||||
definition.QueueCapacity = 0
|
||||
} else {
|
||||
if !definition.QueueCapacitySet {
|
||||
definition.QueueCapacity = defaultQueueCapacity
|
||||
definition.QueueCapacitySet = true
|
||||
}
|
||||
maxInt := int(^uint(0) >> 1)
|
||||
if definition.QueueCapacity > maxInt-definition.ConcurrencyLimit {
|
||||
return domain.Backend{}, fmt.Errorf(
|
||||
"backend %q total capacity overflows int",
|
||||
definition.ID,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
keys := make([]string, 0, len(definition.ExtraParams))
|
||||
for key := range definition.ExtraParams {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, key := range keys {
|
||||
if key == "" {
|
||||
return domain.Backend{}, fmt.Errorf("backend %q extra parameter key must not be empty", definition.ID)
|
||||
}
|
||||
if llm.IsReservedOpenAIChatRequestField(key) {
|
||||
return domain.Backend{}, fmt.Errorf(
|
||||
"backend %q extra parameter %q collides with a reserved request field",
|
||||
definition.ID,
|
||||
key,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
extraParams, err := jsonvalue.CopyMap(definition.ExtraParams)
|
||||
if err != nil {
|
||||
return domain.Backend{}, fmt.Errorf("backend %q extra parameters: %w", definition.ID, err)
|
||||
}
|
||||
definition.ExtraParams = extraParams
|
||||
return definition, nil
|
||||
}
|
||||
|
||||
@@ -3,374 +3,90 @@ package backend_test
|
||||
import (
|
||||
"errors"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/backend"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
)
|
||||
|
||||
const validEndpoint = "https://backend.example/v1"
|
||||
|
||||
func TestRegistryIncludesMaintainedDefinitions(t *testing.T) {
|
||||
maintained := []domain.Backend{
|
||||
{ID: backend.OpenRouterID, Endpoint: validEndpoint, ConcurrencyLimit: 2, QueueCapacity: 3, QueueCapacitySet: true},
|
||||
{ID: backend.RakestrawHomeID, Endpoint: "https://second.example/v1", ConcurrencyLimit: 4, QueueCapacity: 5, QueueCapacitySet: true},
|
||||
}
|
||||
registry, err := backend.NewRegistry(maintained, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("construct registry: %v", err)
|
||||
}
|
||||
|
||||
for _, expected := range maintained {
|
||||
t.Run(expected.ID, func(t *testing.T) {
|
||||
definition, err := registry.GetBackend(expected.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("look up maintained definition: %v", err)
|
||||
}
|
||||
if !reflect.DeepEqual(definition, expected) {
|
||||
t.Fatalf("unexpected maintained definition: %#v", definition)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
policies := registry.CapacityPolicies()
|
||||
if len(policies) != 2 ||
|
||||
policies[backend.OpenRouterID] != (domain.BackendCapacityPolicy{
|
||||
ConcurrencyLimit: 2,
|
||||
QueueCapacity: 3,
|
||||
}) ||
|
||||
policies[backend.RakestrawHomeID] != (domain.BackendCapacityPolicy{
|
||||
ConcurrencyLimit: 4,
|
||||
QueueCapacity: 5,
|
||||
}) {
|
||||
t.Fatalf("unexpected built-in capacity policies: %#v", policies)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegistryNormalizesUniqueAdditionsAndIsolatesMutations(t *testing.T) {
|
||||
nested := map[string]int{"limit": 2}
|
||||
extraParams := map[string]any{
|
||||
"count": int64(7),
|
||||
"nested": nested,
|
||||
}
|
||||
registry, err := backend.NewRegistry(nil, []domain.Backend{
|
||||
{
|
||||
ID: " custom ",
|
||||
Endpoint: " https://custom.example/openai/v1 ",
|
||||
APIKeyEnv: " CUSTOM_API_KEY ",
|
||||
ExtraParams: extraParams,
|
||||
ConcurrencyLimit: 3,
|
||||
QueueCapacity: 2,
|
||||
QueueCapacitySet: true,
|
||||
func TestEffectiveRegistryOwnsDefinitionsAndCapacityPolicies(t *testing.T) {
|
||||
params := map[string]any{"nested": map[string]any{"enabled": true}}
|
||||
defaultProfile := "catalog-profile"
|
||||
reachabilityEndpoint := "https://probe.example/ready"
|
||||
authenticationKind := "openrouter_key"
|
||||
authenticationEndpoint := "https://probe.example/key"
|
||||
smokeTests := &domain.BackendSmokeTestConfig{
|
||||
DefaultProfile: &defaultProfile,
|
||||
Reachability: &domain.ReachabilityProbeConfig{Endpoint: &reachabilityEndpoint},
|
||||
Authentication: &domain.AuthenticationProbeConfig{
|
||||
Kind: &authenticationKind, Endpoint: &authenticationEndpoint,
|
||||
},
|
||||
{
|
||||
ID: "Custom",
|
||||
Endpoint: validEndpoint,
|
||||
}
|
||||
registry, err := backend.NewEffectiveRegistry(map[string]domain.Backend{
|
||||
" local.v1 ": {
|
||||
Endpoint: "https://backend.example/v1", ExtraParams: params,
|
||||
ConcurrencyLimit: 2, QueueCapacity: 3, QueueCapacitySet: true,
|
||||
SmokeTests: smokeTests,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("construct registry: %v", err)
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
nested["limit"] = 99
|
||||
extraParams["added"] = true
|
||||
|
||||
got, err := registry.GetBackend("custom")
|
||||
if err != nil {
|
||||
t.Fatalf("look up custom backend: %v", err)
|
||||
params["nested"].(map[string]any)["enabled"] = false
|
||||
defaultProfile = "mutated-profile"
|
||||
reachabilityEndpoint = "https://mutated.example/ready"
|
||||
authenticationKind = "none"
|
||||
authenticationEndpoint = "https://mutated.example/key"
|
||||
selected, err := registry.GetBackend(" local.v1 ")
|
||||
if err != nil || selected.ID != " local.v1 " || selected.ExtraParams["nested"].(map[string]any)["enabled"] != true {
|
||||
t.Fatalf("effective backend was not captured under its literal ID: %+v, %v", selected, err)
|
||||
}
|
||||
if got.ID != "custom" ||
|
||||
got.Endpoint != "https://custom.example/openai/v1" ||
|
||||
got.APIKeyEnv != "CUSTOM_API_KEY" ||
|
||||
got.ConcurrencyLimit != 3 ||
|
||||
got.QueueCapacity != 2 ||
|
||||
!got.QueueCapacitySet {
|
||||
t.Fatalf("unexpected normalized definition: %#v", got)
|
||||
if selected.SmokeTests == nil || selected.SmokeTests.DefaultProfile == nil ||
|
||||
*selected.SmokeTests.DefaultProfile != "catalog-profile" || selected.SmokeTests.Reachability == nil ||
|
||||
selected.SmokeTests.Reachability.Endpoint == nil || *selected.SmokeTests.Reachability.Endpoint != "https://probe.example/ready" ||
|
||||
selected.SmokeTests.Authentication == nil || selected.SmokeTests.Authentication.Kind == nil ||
|
||||
*selected.SmokeTests.Authentication.Kind != "openrouter_key" || selected.SmokeTests.Authentication.Endpoint == nil ||
|
||||
*selected.SmokeTests.Authentication.Endpoint != "https://probe.example/key" {
|
||||
t.Fatalf("registry did not capture nested diagnostic metadata: %+v", selected.SmokeTests)
|
||||
}
|
||||
if count, ok := got.ExtraParams["count"].(int64); !ok || count != 7 {
|
||||
t.Fatalf("integer type or value changed: %#v", got.ExtraParams["count"])
|
||||
selected.ExtraParams["nested"].(map[string]any)["enabled"] = false
|
||||
*selected.SmokeTests.DefaultProfile = "mutated-return"
|
||||
*selected.SmokeTests.Reachability.Endpoint = "https://mutated-return.example/ready"
|
||||
*selected.SmokeTests.Authentication.Kind = "none"
|
||||
*selected.SmokeTests.Authentication.Endpoint = "https://mutated-return.example/key"
|
||||
again, err := registry.GetBackend(" local.v1 ")
|
||||
if err != nil || again.ExtraParams["nested"].(map[string]any)["enabled"] != true {
|
||||
t.Fatalf("lookup exposed registry values: %+v, %v", again, err)
|
||||
}
|
||||
gotNested, ok := got.ExtraParams["nested"].(map[string]int)
|
||||
if !ok || gotNested["limit"] != 2 {
|
||||
t.Fatalf("container type or value changed: %#v", got.ExtraParams["nested"])
|
||||
if again.SmokeTests == nil || again.SmokeTests.DefaultProfile == nil ||
|
||||
*again.SmokeTests.DefaultProfile != "catalog-profile" || again.SmokeTests.Reachability == nil ||
|
||||
again.SmokeTests.Reachability.Endpoint == nil || *again.SmokeTests.Reachability.Endpoint != "https://probe.example/ready" ||
|
||||
again.SmokeTests.Authentication == nil || again.SmokeTests.Authentication.Kind == nil ||
|
||||
*again.SmokeTests.Authentication.Kind != "openrouter_key" || again.SmokeTests.Authentication.Endpoint == nil ||
|
||||
*again.SmokeTests.Authentication.Endpoint != "https://probe.example/key" {
|
||||
t.Fatalf("lookup exposed nested diagnostic metadata: %+v", again.SmokeTests)
|
||||
}
|
||||
if _, exists := got.ExtraParams["added"]; exists {
|
||||
t.Fatalf("registry retained caller map: %#v", got.ExtraParams)
|
||||
}
|
||||
|
||||
gotNested["limit"] = 100
|
||||
got.ExtraParams["added"] = true
|
||||
again, err := registry.GetBackend("custom")
|
||||
if err != nil {
|
||||
t.Fatalf("look up custom backend again: %v", err)
|
||||
}
|
||||
if again.ExtraParams["nested"].(map[string]int)["limit"] != 2 {
|
||||
t.Fatalf("lookup exposed registry nested map: %#v", again.ExtraParams)
|
||||
}
|
||||
if _, exists := again.ExtraParams["added"]; exists {
|
||||
t.Fatalf("lookup exposed registry map: %#v", again.ExtraParams)
|
||||
}
|
||||
|
||||
if _, err := registry.GetBackend("Custom"); err != nil {
|
||||
t.Fatalf("backend IDs should be case-sensitive: %v", err)
|
||||
}
|
||||
|
||||
policies := registry.CapacityPolicies()
|
||||
if len(policies) != 1 {
|
||||
t.Fatalf("unexpected capacity policy count: %#v", policies)
|
||||
}
|
||||
policies["custom"] = domain.BackendCapacityPolicy{}
|
||||
delete(policies, "custom")
|
||||
againPolicies := registry.CapacityPolicies()
|
||||
if againPolicies["custom"] != (domain.BackendCapacityPolicy{
|
||||
ConcurrencyLimit: 3,
|
||||
QueueCapacity: 2,
|
||||
if !reflect.DeepEqual(policies, map[string]domain.BackendCapacityPolicy{
|
||||
" local.v1 ": {ConcurrencyLimit: 2, QueueCapacity: 3},
|
||||
}) {
|
||||
t.Fatalf("capacity policy map mutated registry state: %#v", againPolicies)
|
||||
t.Fatalf("capacity policy = %+v", policies)
|
||||
}
|
||||
delete(policies, " local.v1 ")
|
||||
if len(registry.CapacityPolicies()) != 1 {
|
||||
t.Fatal("capacity policy map exposed registry state")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewRegistryNormalizesCapacityPolicy(t *testing.T) {
|
||||
maxInt := int(^uint(0) >> 1)
|
||||
tests := []struct {
|
||||
name string
|
||||
definition domain.Backend
|
||||
want domain.BackendCapacityPolicy
|
||||
wantSet bool
|
||||
wantError bool
|
||||
}{
|
||||
{
|
||||
name: "unlimited when omitted",
|
||||
definition: domain.Backend{},
|
||||
},
|
||||
{
|
||||
name: "default queue",
|
||||
definition: domain.Backend{
|
||||
ConcurrencyLimit: 2,
|
||||
},
|
||||
want: domain.BackendCapacityPolicy{
|
||||
ConcurrencyLimit: 2,
|
||||
QueueCapacity: 1024,
|
||||
},
|
||||
wantSet: true,
|
||||
},
|
||||
{
|
||||
name: "explicit zero queue",
|
||||
definition: domain.Backend{
|
||||
ConcurrencyLimit: 2,
|
||||
QueueCapacitySet: true,
|
||||
},
|
||||
want: domain.BackendCapacityPolicy{
|
||||
ConcurrencyLimit: 2,
|
||||
},
|
||||
wantSet: true,
|
||||
},
|
||||
{
|
||||
name: "negative concurrency limit",
|
||||
definition: domain.Backend{
|
||||
ConcurrencyLimit: -1,
|
||||
},
|
||||
wantError: true,
|
||||
},
|
||||
{
|
||||
name: "negative queue capacity",
|
||||
definition: domain.Backend{
|
||||
ConcurrencyLimit: 1,
|
||||
QueueCapacity: -1,
|
||||
QueueCapacitySet: true,
|
||||
},
|
||||
wantError: true,
|
||||
},
|
||||
{
|
||||
name: "queue without limit",
|
||||
definition: domain.Backend{
|
||||
QueueCapacitySet: true,
|
||||
},
|
||||
wantError: true,
|
||||
},
|
||||
{
|
||||
name: "total overflow",
|
||||
definition: domain.Backend{
|
||||
ConcurrencyLimit: maxInt,
|
||||
QueueCapacity: 1,
|
||||
QueueCapacitySet: true,
|
||||
},
|
||||
wantError: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
tc.definition.ID = "custom"
|
||||
tc.definition.Endpoint = validEndpoint
|
||||
registry, err := backend.NewRegistry(nil, []domain.Backend{tc.definition})
|
||||
if tc.wantError {
|
||||
if err == nil {
|
||||
t.Fatal("expected invalid capacity policy error")
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("construct registry: %v", err)
|
||||
}
|
||||
|
||||
definition, err := registry.GetBackend("custom")
|
||||
if err != nil {
|
||||
t.Fatalf("look up custom backend: %v", err)
|
||||
}
|
||||
if definition.ConcurrencyLimit != tc.want.ConcurrencyLimit ||
|
||||
definition.QueueCapacity != tc.want.QueueCapacity ||
|
||||
definition.QueueCapacitySet != tc.wantSet {
|
||||
t.Fatalf("normalized capacity=(%d, %d, %t), want (%d, %d, %t)",
|
||||
definition.ConcurrencyLimit,
|
||||
definition.QueueCapacity,
|
||||
definition.QueueCapacitySet,
|
||||
tc.want.ConcurrencyLimit,
|
||||
tc.want.QueueCapacity,
|
||||
tc.wantSet,
|
||||
)
|
||||
}
|
||||
policies := registry.CapacityPolicies()
|
||||
got, ok := policies["custom"]
|
||||
if ok != tc.wantSet || got != tc.want {
|
||||
t.Fatalf("capacity policy=(%#v, %t), want (%#v, %t)", got, ok, tc.want, tc.wantSet)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewRegistryRejectsDuplicateIDs(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
additions []domain.Backend
|
||||
wantID string
|
||||
}{
|
||||
{
|
||||
name: "OpenRouter collision after normalization",
|
||||
additions: []domain.Backend{{ID: " openrouter "}},
|
||||
wantID: backend.OpenRouterID,
|
||||
},
|
||||
{
|
||||
name: "Rakestrawhome collision after normalization",
|
||||
additions: []domain.Backend{{ID: " rakestrawhome "}},
|
||||
wantID: backend.RakestrawHomeID,
|
||||
},
|
||||
{
|
||||
name: "consumer collision after normalization",
|
||||
additions: []domain.Backend{
|
||||
{ID: "custom", Endpoint: validEndpoint},
|
||||
{ID: " custom ", Endpoint: "https://other.example/v1"},
|
||||
},
|
||||
wantID: "custom",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
_, err := backend.NewRegistry(nil, tc.additions)
|
||||
if err == nil {
|
||||
t.Fatal("expected duplicate ID error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), tc.wantID) {
|
||||
t.Fatalf("expected error to identify %q, got %v", tc.wantID, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewRegistryValidatesIDs(t *testing.T) {
|
||||
for _, id := range []string{"", " \t\n "} {
|
||||
t.Run(id, func(t *testing.T) {
|
||||
_, err := backend.NewRegistry(nil, []domain.Backend{{
|
||||
ID: id,
|
||||
Endpoint: validEndpoint,
|
||||
}})
|
||||
if err == nil {
|
||||
t.Fatal("expected blank ID error")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewRegistryValidatesEndpoints(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
endpoint string
|
||||
}{
|
||||
{name: "blank", endpoint: ""},
|
||||
{name: "relative", endpoint: "/v1"},
|
||||
{name: "missing host", endpoint: "https:///v1"},
|
||||
{name: "unsupported scheme", endpoint: "ftp://backend.example/v1"},
|
||||
{name: "user information", endpoint: "https://user@backend.example/v1"},
|
||||
{name: "query", endpoint: "https://backend.example/v1?mode=chat"},
|
||||
{name: "empty query", endpoint: "https://backend.example/v1?"},
|
||||
{name: "fragment", endpoint: "https://backend.example/v1#chat"},
|
||||
{name: "empty fragment", endpoint: "https://backend.example/v1#"},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
_, err := backend.NewRegistry(nil, []domain.Backend{{
|
||||
ID: "custom",
|
||||
Endpoint: tc.endpoint,
|
||||
}})
|
||||
if err == nil {
|
||||
t.Fatal("expected invalid endpoint error")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewRegistryValidatesEnvironmentVariableNames(t *testing.T) {
|
||||
for _, name := range []string{"1API_KEY", "API-KEY", "API KEY", "ÅPI_KEY"} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
_, err := backend.NewRegistry(nil, []domain.Backend{{
|
||||
ID: "custom",
|
||||
Endpoint: validEndpoint,
|
||||
APIKeyEnv: name,
|
||||
}})
|
||||
if err == nil {
|
||||
t.Fatal("expected invalid environment-variable name error")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewRegistryRejectsInvalidAndReservedExtraParameters(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
extraParams map[string]any
|
||||
}{
|
||||
{name: "unsupported value", extraParams: map[string]any{"value": make(chan int)}},
|
||||
{name: "reserved key", extraParams: map[string]any{"model": "override"}},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
_, err := backend.NewRegistry(nil, []domain.Backend{{
|
||||
ID: "custom",
|
||||
Endpoint: validEndpoint,
|
||||
ExtraParams: tc.extraParams,
|
||||
}})
|
||||
if err == nil {
|
||||
t.Fatal("expected invalid extra parameters error")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegistryLookupReportsNotFound(t *testing.T) {
|
||||
registry, err := backend.NewRegistry(nil, nil)
|
||||
func TestEffectiveRegistryMissingBackend(t *testing.T) {
|
||||
registry, err := backend.NewEffectiveRegistry(nil)
|
||||
if err != nil {
|
||||
t.Fatalf("construct registry: %v", err)
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
_, err = registry.GetBackend("missing")
|
||||
if !errors.Is(err, backend.ErrBackendNotFound) {
|
||||
t.Fatalf("expected ErrBackendNotFound, got %v", err)
|
||||
if _, err := registry.GetBackend("missing"); !errors.Is(err, backend.ErrBackendNotFound) {
|
||||
t.Fatalf("missing backend error = %v", err)
|
||||
}
|
||||
if !strings.Contains(err.Error(), "missing") {
|
||||
t.Fatalf("expected error to identify backend, got %v", err)
|
||||
if _, err := registry.GetBackend("local.v1"); !errors.Is(err, backend.ErrBackendNotFound) {
|
||||
t.Fatalf("trimmed lookup unexpectedly matched: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,249 +0,0 @@
|
||||
// Package catalog validates immutable maintained backend catalog assets.
|
||||
package catalog
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"path"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/backend"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/profile"
|
||||
)
|
||||
|
||||
// Source identifies one immutable backend catalog asset tree.
|
||||
type Source struct {
|
||||
Name string
|
||||
ExpectedBackendID string
|
||||
FS fs.FS
|
||||
Root string
|
||||
}
|
||||
|
||||
// Set is the validated maintained backend and raw profile catalog.
|
||||
type Set struct {
|
||||
Backends []domain.Backend
|
||||
Profiles profile.Repository
|
||||
profileIDs []string
|
||||
}
|
||||
|
||||
// Load validates and combines immutable catalog sources in source order.
|
||||
func Load(sources ...Source) (Set, error) {
|
||||
if len(sources) == 0 {
|
||||
return Set{}, errors.New("at least one catalog source is required")
|
||||
}
|
||||
loaded := Set{Backends: make([]domain.Backend, 0, len(sources))}
|
||||
names := map[string]bool{}
|
||||
backendIDs := map[string]bool{}
|
||||
profileIDs := map[string]bool{}
|
||||
for _, source := range sources {
|
||||
if err := validateSource(source, names); err != nil {
|
||||
return Set{}, err
|
||||
}
|
||||
names[source.Name] = true
|
||||
definition, err := loadBackend(source)
|
||||
if err != nil {
|
||||
return Set{}, err
|
||||
}
|
||||
if backendIDs[definition.ID] {
|
||||
return Set{}, fmt.Errorf("catalog %s: backend ID duplicates an earlier catalog", source.Name)
|
||||
}
|
||||
backendIDs[definition.ID] = true
|
||||
repository, metadata, err := profile.LoadFSRepository(context.Background(), source.FS, path.Join(source.Root, "profiles"))
|
||||
if err != nil {
|
||||
return Set{}, fmt.Errorf("catalog %s: %w", source.Name, err)
|
||||
}
|
||||
if len(metadata) == 0 {
|
||||
return Set{}, fmt.Errorf("catalog %s: profiles must not be empty", source.Name)
|
||||
}
|
||||
resolvingRepository := profile.NewResolvingRepository(repository)
|
||||
for _, entry := range metadata {
|
||||
if strings.HasPrefix(entry.ID, "promptkit-") {
|
||||
return Set{}, fmt.Errorf("catalog %s: %s uses the reserved promptkit- profile ID prefix", source.Name, entry.Path)
|
||||
}
|
||||
if profileIDs[entry.ID] {
|
||||
return Set{}, fmt.Errorf("catalog %s: %s duplicates an earlier profile ID", source.Name, entry.Path)
|
||||
}
|
||||
if containsField(entry.ExplicitFields, "endpoint") || containsField(entry.ExplicitFields, "api_key_env") {
|
||||
return Set{}, fmt.Errorf("catalog %s: %s contains connection metadata", source.Name, entry.Path)
|
||||
}
|
||||
value, err := repository.GetProfile(context.Background(), entry.ID)
|
||||
if err != nil {
|
||||
return Set{}, fmt.Errorf("catalog %s: %s: %w", source.Name, entry.Path, err)
|
||||
}
|
||||
if err := rejectSecretKeys(value.ExtraParams); err != nil {
|
||||
return Set{}, fmt.Errorf("catalog %s: %s: prohibited extra parameter key", source.Name, entry.Path)
|
||||
}
|
||||
resolved, err := resolvingRepository.GetProfile(context.Background(), entry.ID)
|
||||
if err != nil {
|
||||
return Set{}, fmt.Errorf("catalog %s: %s has invalid profile inheritance", source.Name, entry.Path)
|
||||
}
|
||||
if resolved.BackendID != definition.ID {
|
||||
return Set{}, fmt.Errorf("catalog %s: %s selects a different backend", source.Name, entry.Path)
|
||||
}
|
||||
profileIDs[entry.ID] = true
|
||||
loaded.profileIDs = append(loaded.profileIDs, entry.ID)
|
||||
}
|
||||
if loaded.Profiles == nil {
|
||||
loaded.Profiles = repository
|
||||
} else {
|
||||
loaded.Profiles = profile.NewOverlayRepository(loaded.Profiles, repository)
|
||||
}
|
||||
loaded.Backends = append(loaded.Backends, definition)
|
||||
}
|
||||
sort.Strings(loaded.profileIDs)
|
||||
return loaded, nil
|
||||
}
|
||||
|
||||
func validateSource(source Source, names map[string]bool) error {
|
||||
if strings.TrimSpace(source.Name) == "" || source.Name != strings.TrimSpace(source.Name) {
|
||||
return errors.New("catalog source name must not be blank")
|
||||
}
|
||||
if names[source.Name] {
|
||||
return fmt.Errorf("duplicate catalog source name %q", source.Name)
|
||||
}
|
||||
if source.FS == nil {
|
||||
return fmt.Errorf("catalog %s: filesystem is nil", source.Name)
|
||||
}
|
||||
if source.Root == "." || source.Root == "" || !fs.ValidPath(source.Root) {
|
||||
return fmt.Errorf("catalog %s: asset root is invalid", source.Name)
|
||||
}
|
||||
if strings.TrimSpace(source.ExpectedBackendID) == "" {
|
||||
return fmt.Errorf("catalog %s: expected backend ID is blank", source.Name)
|
||||
}
|
||||
return validateLayout(source)
|
||||
}
|
||||
|
||||
func validateLayout(source Source) error {
|
||||
manifestPath := path.Join(source.Root, "backend.json")
|
||||
profilesRoot := path.Join(source.Root, "profiles")
|
||||
return fs.WalkDir(source.FS, source.Root, func(assetPath string, entry fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if assetPath == source.Root {
|
||||
if !entry.IsDir() {
|
||||
return fmt.Errorf("catalog %s: invalid asset path %s", source.Name, assetPath)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if entry.IsDir() {
|
||||
if assetPath == profilesRoot || strings.HasPrefix(assetPath, profilesRoot+"/") {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("catalog %s: invalid asset path %s", source.Name, assetPath)
|
||||
}
|
||||
if assetPath == manifestPath && entry.Type().IsRegular() {
|
||||
return nil
|
||||
}
|
||||
if strings.HasPrefix(assetPath, profilesRoot+"/") && entry.Type().IsRegular() && strings.HasSuffix(assetPath, ".yml") {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("catalog %s: invalid asset path %s", source.Name, assetPath)
|
||||
})
|
||||
}
|
||||
|
||||
func loadBackend(source Source) (domain.Backend, error) {
|
||||
data, err := fs.ReadFile(source.FS, path.Join(source.Root, "backend.json"))
|
||||
if err != nil {
|
||||
return domain.Backend{}, fmt.Errorf("catalog %s: backend.json: %w", source.Name, err)
|
||||
}
|
||||
type manifest struct {
|
||||
SchemaVersion *int `json:"schema_version"`
|
||||
ID *string `json:"id"`
|
||||
Endpoint *string `json:"endpoint"`
|
||||
APIKeyEnv *string `json:"api_key_env"`
|
||||
ConcurrencyLimit *int `json:"concurrency_limit"`
|
||||
QueueCapacity *int `json:"queue_capacity"`
|
||||
ExtraParams json.RawMessage `json:"extra_params"`
|
||||
}
|
||||
decoder := json.NewDecoder(bytes.NewReader(data))
|
||||
decoder.DisallowUnknownFields()
|
||||
var value manifest
|
||||
if err := decoder.Decode(&value); err != nil {
|
||||
return domain.Backend{}, fmt.Errorf("catalog %s: backend.json: invalid manifest", source.Name)
|
||||
}
|
||||
var trailing any
|
||||
if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) {
|
||||
return domain.Backend{}, fmt.Errorf("catalog %s: backend.json: invalid manifest", source.Name)
|
||||
}
|
||||
if value.SchemaVersion == nil || *value.SchemaVersion != 1 || value.ID == nil || value.Endpoint == nil || value.APIKeyEnv == nil || value.ConcurrencyLimit == nil || value.QueueCapacity == nil || value.ExtraParams == nil {
|
||||
return domain.Backend{}, fmt.Errorf("catalog %s: backend.json: required field is missing or unsupported", source.Name)
|
||||
}
|
||||
if *value.ID != source.ExpectedBackendID {
|
||||
return domain.Backend{}, fmt.Errorf("catalog %s: backend ID does not match expected ID", source.Name)
|
||||
}
|
||||
if strings.TrimSpace(*value.APIKeyEnv) == "" {
|
||||
return domain.Backend{}, fmt.Errorf("catalog %s: backend.json: api key environment variable must not be blank", source.Name)
|
||||
}
|
||||
extraParams, err := decodeExtraParams(value.ExtraParams)
|
||||
if err != nil {
|
||||
return domain.Backend{}, fmt.Errorf("catalog %s: backend.json: invalid extra parameters", source.Name)
|
||||
}
|
||||
if err := rejectSecretKeys(extraParams); err != nil {
|
||||
return domain.Backend{}, fmt.Errorf("catalog %s: backend.json: prohibited extra parameter key", source.Name)
|
||||
}
|
||||
normalized, err := backend.NormalizeDefinition(domain.Backend{ID: *value.ID, Endpoint: *value.Endpoint, APIKeyEnv: *value.APIKeyEnv, ExtraParams: extraParams, ConcurrencyLimit: *value.ConcurrencyLimit, QueueCapacity: *value.QueueCapacity, QueueCapacitySet: true})
|
||||
if err != nil {
|
||||
return domain.Backend{}, fmt.Errorf("catalog %s: backend.json: invalid backend definition", source.Name)
|
||||
}
|
||||
return normalized, nil
|
||||
}
|
||||
|
||||
func decodeExtraParams(data []byte) (map[string]any, error) {
|
||||
if bytes.Equal(bytes.TrimSpace(data), []byte("null")) {
|
||||
return nil, nil
|
||||
}
|
||||
decoder := json.NewDecoder(bytes.NewReader(data))
|
||||
decoder.UseNumber()
|
||||
var value map[string]any
|
||||
if err := decoder.Decode(&value); err != nil || value == nil {
|
||||
return nil, errors.New("extra parameters must be an object or null")
|
||||
}
|
||||
var trailing any
|
||||
if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) {
|
||||
return nil, errors.New("extra parameters must contain exactly one JSON value")
|
||||
}
|
||||
return value, nil
|
||||
}
|
||||
|
||||
func containsField(fields []string, target string) bool {
|
||||
for _, field := range fields {
|
||||
if field == target {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func rejectSecretKeys(value map[string]any) error {
|
||||
return rejectSecretValue(value)
|
||||
}
|
||||
|
||||
func rejectSecretValue(value any) error {
|
||||
switch value := value.(type) {
|
||||
case map[string]any:
|
||||
for key, child := range value {
|
||||
switch strings.ToLower(key) {
|
||||
case "api_key", "apikey", "authorization", "credential", "credentials", "password", "secret", "token", "access_token":
|
||||
return errors.New("prohibited key")
|
||||
}
|
||||
if err := rejectSecretValue(child); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
case []any:
|
||||
for _, child := range value {
|
||||
if err := rejectSecretValue(child); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -1,356 +0,0 @@
|
||||
package catalog
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"strings"
|
||||
"testing"
|
||||
"testing/fstest"
|
||||
|
||||
openrouter "gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v2"
|
||||
rakestrawhome "gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v2"
|
||||
)
|
||||
|
||||
func TestLoadPublishedCatalogs(t *testing.T) {
|
||||
loaded, err := Load(
|
||||
Source{Name: "OpenRouter", ExpectedBackendID: "openrouter", FS: openrouter.FS(), Root: openrouter.Root},
|
||||
Source{Name: "Rakestrawhome", ExpectedBackendID: "rakestrawhome", FS: rakestrawhome.FS(), Root: rakestrawhome.Root},
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("load published catalogs: %v", err)
|
||||
}
|
||||
if loaded.Profiles == nil || len(loaded.profileIDs) == 0 {
|
||||
t.Fatalf("published catalogs have no usable profiles: %#v", loaded)
|
||||
}
|
||||
if _, err := loaded.Profiles.GetProfile(context.Background(), "gemini-flash-latest"); err != nil {
|
||||
t.Fatalf("load maintained standard-profile base: %v", err)
|
||||
}
|
||||
|
||||
expectedBackends := map[string]struct {
|
||||
endpoint string
|
||||
apiKeyEnv string
|
||||
concurrencyLimit int
|
||||
queueCapacity int
|
||||
}{
|
||||
"openrouter": {
|
||||
endpoint: "https://openrouter.ai/api/v1",
|
||||
apiKeyEnv: "OPENROUTER_API_KEY",
|
||||
concurrencyLimit: 16,
|
||||
queueCapacity: 1024,
|
||||
},
|
||||
"rakestrawhome": {
|
||||
endpoint: "https://inference.ai.rakestrawhome.com/v1",
|
||||
apiKeyEnv: "RAKESTRAWHOME_INFERENCE_API_KEY",
|
||||
concurrencyLimit: 4,
|
||||
queueCapacity: 1024,
|
||||
},
|
||||
}
|
||||
if len(loaded.Backends) != len(expectedBackends) {
|
||||
t.Fatalf("published backend count = %d, want %d", len(loaded.Backends), len(expectedBackends))
|
||||
}
|
||||
for _, definition := range loaded.Backends {
|
||||
expected, found := expectedBackends[definition.ID]
|
||||
if !found {
|
||||
t.Fatalf("unexpected published backend %q", definition.ID)
|
||||
}
|
||||
if definition.Endpoint != expected.endpoint ||
|
||||
definition.APIKeyEnv != expected.apiKeyEnv ||
|
||||
definition.ConcurrencyLimit != expected.concurrencyLimit ||
|
||||
definition.QueueCapacity != expected.queueCapacity ||
|
||||
!definition.QueueCapacitySet ||
|
||||
definition.ExtraParams != nil {
|
||||
t.Fatalf("unexpected published backend definition: %#v", definition)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadRejectsInvalidSources(t *testing.T) {
|
||||
for name, sources := range map[string][]Source{
|
||||
"none": nil,
|
||||
"blank name": {{Name: " ", ExpectedBackendID: "openrouter", FS: openrouter.FS(), Root: openrouter.Root}},
|
||||
"duplicate name": {
|
||||
{Name: "same", ExpectedBackendID: "openrouter", FS: openrouter.FS(), Root: openrouter.Root},
|
||||
{Name: "same", ExpectedBackendID: "rakestrawhome", FS: rakestrawhome.FS(), Root: rakestrawhome.Root},
|
||||
},
|
||||
"nil filesystem": {{Name: "missing", ExpectedBackendID: "openrouter", Root: openrouter.Root}},
|
||||
"invalid root": {{Name: "invalid-root", ExpectedBackendID: "openrouter", FS: openrouter.FS(), Root: "."}},
|
||||
"blank expected backend": {{Name: "blank-backend", ExpectedBackendID: " ", FS: openrouter.FS(), Root: openrouter.Root}},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
if _, err := Load(sources...); err == nil {
|
||||
t.Fatal("expected error")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadRejectsInvalidLayouts(t *testing.T) {
|
||||
tests := map[string]func(fstest.MapFS){
|
||||
"unexpected file": func(fsys fstest.MapFS) {
|
||||
fsys["catalog/notes.txt"] = &fstest.MapFile{Data: []byte("unexpected")}
|
||||
},
|
||||
"unexpected directory": func(fsys fstest.MapFS) {
|
||||
fsys["catalog/unexpected"] = &fstest.MapFile{Mode: fs.ModeDir}
|
||||
},
|
||||
"nonregular manifest": func(fsys fstest.MapFS) {
|
||||
fsys["catalog/backend.json"].Mode = fs.ModeSymlink
|
||||
},
|
||||
"wrong profile extension": func(fsys fstest.MapFS) {
|
||||
fsys["catalog/profiles/extra.yaml"] = &fstest.MapFile{Data: []byte(validProfile("extra", "one"))}
|
||||
},
|
||||
"nonregular profile": func(fsys fstest.MapFS) {
|
||||
fsys["catalog/profiles/one-profile.yml"].Mode = fs.ModeSymlink
|
||||
},
|
||||
}
|
||||
for name, mutate := range tests {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
fsys := validCatalogFS("one")
|
||||
mutate(fsys)
|
||||
if _, err := Load(testSource("one", fsys)); err == nil {
|
||||
t.Fatal("expected invalid layout error")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadRejectsInvalidManifests(t *testing.T) {
|
||||
tests := map[string]string{
|
||||
"malformed": `{`,
|
||||
"trailing value": validManifest("one", "TEST_API_KEY", "null") + `{}`,
|
||||
"missing fields": `{"schema_version":1,"id":"one"}`,
|
||||
"unsupported version": strings.Replace(validManifest("one", "TEST_API_KEY", "null"), `"schema_version":1`, `"schema_version":2`, 1),
|
||||
"unknown field": strings.Replace(validManifest("one", "TEST_API_KEY", "null"), `"extra_params":null`, `"extra_params":null,"unknown":true`, 1),
|
||||
"blank API key env": validManifest("one", " ", "null"),
|
||||
"invalid API key env": validManifest("one", "LEAK-MARKER", "null"),
|
||||
"invalid endpoint": strings.Replace(validManifest("one", "TEST_API_KEY", "null"), `https://one.example/v1`, `ftp://leak-marker.invalid/v1`, 1),
|
||||
"zero concurrency": strings.Replace(validManifest("one", "TEST_API_KEY", "null"), `"concurrency_limit":2`, `"concurrency_limit":0`, 1),
|
||||
"non-object parameters": validManifest("one", "TEST_API_KEY", `[]`),
|
||||
"secret parameter": validManifest("one", "TEST_API_KEY", `{"nested":{"token":"leak-marker"}}`),
|
||||
}
|
||||
for name, manifest := range tests {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
fsys := validCatalogFS("one")
|
||||
fsys["catalog/backend.json"].Data = []byte(manifest)
|
||||
_, err := Load(testSource("one", fsys))
|
||||
if err == nil {
|
||||
t.Fatal("expected invalid manifest error")
|
||||
}
|
||||
if strings.Contains(strings.ToLower(err.Error()), "leak-marker") {
|
||||
t.Fatalf("catalog error exposed manifest content: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadPreservesManifestJSONNumbers(t *testing.T) {
|
||||
fsys := validCatalogFS("one")
|
||||
fsys["catalog/backend.json"].Data = []byte(validManifest(
|
||||
"one",
|
||||
"TEST_API_KEY",
|
||||
`{"large":9007199254740993,"nested":[1.25]}`,
|
||||
))
|
||||
loaded, err := Load(testSource("one", fsys))
|
||||
if err != nil {
|
||||
t.Fatalf("load catalog: %v", err)
|
||||
}
|
||||
if got := loaded.Backends[0].ExtraParams["large"]; got != json.Number("9007199254740993") {
|
||||
t.Fatalf("large JSON integer = %#v, want preserved json.Number", got)
|
||||
}
|
||||
nested := loaded.Backends[0].ExtraParams["nested"].([]any)
|
||||
if nested[0] != json.Number("1.25") {
|
||||
t.Fatalf("nested JSON number = %#v, want preserved json.Number", nested[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadRejectsInvalidCatalogProfiles(t *testing.T) {
|
||||
tests := map[string]func(fstest.MapFS){
|
||||
"empty": func(fsys fstest.MapFS) {
|
||||
delete(fsys, "catalog/profiles/one-profile.yml")
|
||||
fsys["catalog/profiles"] = &fstest.MapFile{Mode: fs.ModeDir}
|
||||
},
|
||||
"malformed": func(fsys fstest.MapFS) {
|
||||
fsys["catalog/profiles/one-profile.yml"].Data = []byte("id: [")
|
||||
},
|
||||
"raw API key": func(fsys fstest.MapFS) {
|
||||
fsys["catalog/profiles/one-profile.yml"].Data = []byte(validProfile("one-profile", "one") + "api_key: leak-marker\n")
|
||||
},
|
||||
"endpoint field": func(fsys fstest.MapFS) {
|
||||
fsys["catalog/profiles/one-profile.yml"].Data = []byte(validProfile("one-profile", "one") + "endpoint: ''\n")
|
||||
},
|
||||
"API key environment field": func(fsys fstest.MapFS) {
|
||||
fsys["catalog/profiles/one-profile.yml"].Data = []byte(validProfile("one-profile", "one") + "api_key_env: ''\n")
|
||||
},
|
||||
"owner mismatch": func(fsys fstest.MapFS) {
|
||||
fsys["catalog/profiles/one-profile.yml"].Data = []byte(validProfile("one-profile", "other"))
|
||||
},
|
||||
"missing base": func(fsys fstest.MapFS) {
|
||||
fsys["catalog/profiles/one-profile.yml"].Data = []byte("id: one-profile\nbase_profile: leak-marker\n")
|
||||
},
|
||||
"cyclic base": func(fsys fstest.MapFS) {
|
||||
fsys["catalog/profiles/one-profile.yml"].Data = []byte("id: one-profile\nbase_profile: second\n")
|
||||
fsys["catalog/profiles/second.yml"] = &fstest.MapFile{Data: []byte("id: second\nbase_profile: one-profile\n")}
|
||||
},
|
||||
"secret profile parameter": func(fsys fstest.MapFS) {
|
||||
fsys["catalog/profiles/one-profile.yml"].Data = []byte(validProfile("one-profile", "one") + "extra_params:\n nested:\n password: leak-marker\n")
|
||||
},
|
||||
"unknown field is redacted": func(fsys fstest.MapFS) {
|
||||
fsys["catalog/profiles/one-profile.yml"].Data = []byte(validProfile("one-profile", "one") + "leak_marker: leak-marker\n")
|
||||
},
|
||||
}
|
||||
for name, mutate := range tests {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
fsys := validCatalogFS("one")
|
||||
mutate(fsys)
|
||||
_, err := Load(testSource("one", fsys))
|
||||
if err == nil {
|
||||
t.Fatal("expected invalid profile error")
|
||||
}
|
||||
if strings.Contains(strings.ToLower(err.Error()), "leak-marker") {
|
||||
t.Fatalf("catalog error exposed profile content: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadReservesPromptkitProfilePrefixForCatalogs(t *testing.T) {
|
||||
tests := map[string]struct {
|
||||
profileID string
|
||||
wantError bool
|
||||
}{
|
||||
"maintained name": {
|
||||
profileID: "promptkit-openrouter-standard",
|
||||
wantError: true,
|
||||
},
|
||||
"other reserved name": {
|
||||
profileID: "promptkit-example",
|
||||
wantError: true,
|
||||
},
|
||||
"ordinary name": {
|
||||
profileID: "ordinary-profile",
|
||||
},
|
||||
}
|
||||
for name, test := range tests {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
fsys := validCatalogFS("one")
|
||||
fsys["catalog/profiles/one-profile.yml"].Data = []byte(validProfile(test.profileID, "one"))
|
||||
|
||||
_, err := Load(testSource("one", fsys))
|
||||
if (err != nil) != test.wantError {
|
||||
t.Fatalf("Load() error = %v, want error %t", err, test.wantError)
|
||||
}
|
||||
if err != nil && !strings.Contains(err.Error(), "catalog one: one-profile.yml") {
|
||||
t.Fatalf("Load() error = %v, want catalog source and path context", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadRejectsCrossCatalogConflicts(t *testing.T) {
|
||||
t.Run("duplicate backend", func(t *testing.T) {
|
||||
second := catalogFS("one", map[string]string{
|
||||
"catalog/profiles/second.yml": validProfile("second", "one"),
|
||||
}, "null")
|
||||
_, err := Load(
|
||||
Source{Name: "first", ExpectedBackendID: "one", FS: validCatalogFS("one"), Root: "catalog"},
|
||||
Source{Name: "second", ExpectedBackendID: "one", FS: second, Root: "catalog"},
|
||||
)
|
||||
if err == nil {
|
||||
t.Fatal("expected duplicate backend error")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("duplicate profile", func(t *testing.T) {
|
||||
first := catalogFS("one", map[string]string{
|
||||
"catalog/profiles/shared.yml": validProfile("shared", "one"),
|
||||
}, "null")
|
||||
second := catalogFS("two", map[string]string{
|
||||
"catalog/profiles/shared.yml": validProfile("shared", "two"),
|
||||
}, "null")
|
||||
_, err := Load(testSource("one", first), testSource("two", second))
|
||||
if err == nil {
|
||||
t.Fatal("expected duplicate profile error")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("cross-catalog base", func(t *testing.T) {
|
||||
first := catalogFS("one", map[string]string{
|
||||
"catalog/profiles/base.yml": validProfile("base", "one"),
|
||||
}, "null")
|
||||
second := catalogFS("two", map[string]string{
|
||||
"catalog/profiles/child.yml": "id: child\nbase_profile: base\n",
|
||||
}, "null")
|
||||
_, err := Load(testSource("one", first), testSource("two", second))
|
||||
if err == nil {
|
||||
t.Fatal("expected cross-catalog base error")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestLoadReturnsDefensiveCatalogValues(t *testing.T) {
|
||||
fsys := catalogFS("one", map[string]string{
|
||||
"catalog/profiles/one-profile.yml": validProfile("one-profile", "one") + "extra_params:\n nested:\n value: profile\n",
|
||||
}, `{"nested":{"value":"backend"}}`)
|
||||
loaded, err := Load(testSource("one", fsys))
|
||||
if err != nil {
|
||||
t.Fatalf("load catalog: %v", err)
|
||||
}
|
||||
loaded.Backends[0].ExtraParams["nested"].(map[string]any)["value"] = "changed"
|
||||
profileValue, err := loaded.Profiles.GetProfile(context.Background(), "one-profile")
|
||||
if err != nil {
|
||||
t.Fatalf("load profile: %v", err)
|
||||
}
|
||||
profileValue.ExtraParams["nested"].(map[string]any)["value"] = "changed"
|
||||
|
||||
again, err := Load(testSource("one", fsys))
|
||||
if err != nil {
|
||||
t.Fatalf("reload catalog: %v", err)
|
||||
}
|
||||
if got := again.Backends[0].ExtraParams["nested"].(map[string]any)["value"]; got != "backend" {
|
||||
t.Fatalf("backend mutation escaped returned set: %#v", got)
|
||||
}
|
||||
againProfile, err := loaded.Profiles.GetProfile(context.Background(), "one-profile")
|
||||
if err != nil {
|
||||
t.Fatalf("reload profile: %v", err)
|
||||
}
|
||||
if got := againProfile.ExtraParams["nested"].(map[string]any)["value"]; got != "profile" {
|
||||
t.Fatalf("profile mutation escaped returned value: %#v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func testSource(id string, fsys fs.FS) Source {
|
||||
return Source{Name: id, ExpectedBackendID: id, FS: fsys, Root: "catalog"}
|
||||
}
|
||||
|
||||
func validCatalogFS(id string) fstest.MapFS {
|
||||
return catalogFS(id, map[string]string{
|
||||
"catalog/profiles/" + id + "-profile.yml": validProfile(id+"-profile", id),
|
||||
}, "null")
|
||||
}
|
||||
|
||||
func catalogFS(id string, profiles map[string]string, extraParams string) fstest.MapFS {
|
||||
fsys := fstest.MapFS{
|
||||
"catalog/backend.json": &fstest.MapFile{Data: []byte(validManifest(id, "TEST_API_KEY", extraParams))},
|
||||
}
|
||||
for name, content := range profiles {
|
||||
fsys[name] = &fstest.MapFile{Data: []byte(content)}
|
||||
}
|
||||
return fsys
|
||||
}
|
||||
|
||||
func validManifest(id, apiKeyEnv, extraParams string) string {
|
||||
return fmt.Sprintf(
|
||||
`{"schema_version":1,"id":%q,"endpoint":%q,"api_key_env":%q,"concurrency_limit":2,"queue_capacity":3,"extra_params":%s}`,
|
||||
id,
|
||||
"https://"+id+".example/v1",
|
||||
apiKeyEnv,
|
||||
extraParams,
|
||||
)
|
||||
}
|
||||
|
||||
func validProfile(id, backendID string) string {
|
||||
return fmt.Sprintf("id: %s\nbackend: %s\nmodel: test-model\n", id, backendID)
|
||||
}
|
||||
|
||||
var _ fs.FS = openrouter.FS()
|
||||
330
internal/configuration/benchmark_test.go
Normal file
330
internal/configuration/benchmark_test.go
Normal file
@@ -0,0 +1,330 @@
|
||||
package configuration
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
"testing/fstest"
|
||||
|
||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
)
|
||||
|
||||
const benchmarkDefinitionCount = 128
|
||||
|
||||
var (
|
||||
constructionResult *loaded
|
||||
selectedPrompt *domain.PromptDefinition
|
||||
selectedPromptHash string
|
||||
selectedPromptInspection *domain.PromptInspection
|
||||
selectedProfile *domain.ExecutionProfile
|
||||
selectedProfileIdentity string
|
||||
)
|
||||
|
||||
type configurationShape struct {
|
||||
main configkit.Input
|
||||
underlays []configkit.Underlay
|
||||
inputEntries int
|
||||
effectiveEntries int
|
||||
tiers int
|
||||
}
|
||||
|
||||
func benchmarkPrompt(profileID string) map[string]any {
|
||||
return map[string]any{
|
||||
"version": "1",
|
||||
"default_profile": profileID,
|
||||
"messages": []any{map[string]any{"role": "user", "content": "Hello {{.name}}"}},
|
||||
"output": map[string]any{"format": "text", "validation_mode": "none"},
|
||||
}
|
||||
}
|
||||
|
||||
func independentDefinitions(count int) configurationShape {
|
||||
profiles := make(map[string]any, count)
|
||||
prompts := make(map[string]any, count)
|
||||
for index := range count {
|
||||
profileID := fmt.Sprintf("profile-%03d", index)
|
||||
promptID := fmt.Sprintf("prompt-%03d", index)
|
||||
profiles[profileID] = map[string]any{"endpoint": "https://example.test/v1", "model": profileID}
|
||||
prompts[promptID] = benchmarkPrompt(profileID)
|
||||
}
|
||||
return configurationShape{
|
||||
main: configkit.Input{Source: configkit.Mapping("independent definitions", map[string]any{"profiles": profiles, "prompts": prompts})},
|
||||
inputEntries: count * 2,
|
||||
effectiveEntries: count * 2,
|
||||
tiers: 1,
|
||||
}
|
||||
}
|
||||
|
||||
func fileBackedDefinitions(count int) configurationShape {
|
||||
profiles := make(map[string]any, count)
|
||||
prompts := make(map[string]any, count)
|
||||
for index := range count {
|
||||
profileID := fmt.Sprintf("profile-%03d", index)
|
||||
promptID := fmt.Sprintf("prompt-%03d", index)
|
||||
profiles[profileID] = map[string]any{"endpoint": "https://example.test/v1", "model": profileID}
|
||||
prompt := benchmarkPrompt(profileID)
|
||||
prompt["messages"] = []any{map[string]any{"role": "user", "content_file": "body.md"}}
|
||||
prompts[promptID] = prompt
|
||||
}
|
||||
namespace := configkit.NewFilesystem("benchmark content", fstest.MapFS{
|
||||
"assets/body.md": &fstest.MapFile{Data: []byte("Hello {{.name}}")},
|
||||
})
|
||||
return configurationShape{
|
||||
main: configkit.Input{
|
||||
Source: configkit.Mapping("file-backed definitions", map[string]any{"profiles": profiles, "prompts": prompts}),
|
||||
Filesystem: &configkit.FilesystemContext{FS: &namespace, Root: ".", BaseDir: "assets"},
|
||||
},
|
||||
inputEntries: count * 2, effectiveEntries: count * 2, tiers: 1,
|
||||
}
|
||||
}
|
||||
|
||||
func sharedBaseDefinitions(count int) configurationShape {
|
||||
profiles := make(map[string]any, count+1)
|
||||
profiles["base"] = map[string]any{
|
||||
"endpoint": "https://example.test/v1", "model": "shared-model",
|
||||
"extra_params": map[string]any{"provider": map[string]any{"mode": "shared", "limit": 4}},
|
||||
}
|
||||
for index := range count {
|
||||
profiles[fmt.Sprintf("child-%03d", index)] = map[string]any{"extends": "base", "temperature": float64(index%10) / 10}
|
||||
}
|
||||
return configurationShape{
|
||||
main: configkit.Input{Source: configkit.Mapping("inherited definitions", map[string]any{
|
||||
"profiles": profiles,
|
||||
"prompts": map[string]any{"shared": benchmarkPrompt(fmt.Sprintf("child-%03d", count-1))},
|
||||
})},
|
||||
inputEntries: count + 2,
|
||||
effectiveEntries: count + 2,
|
||||
tiers: 1,
|
||||
}
|
||||
}
|
||||
|
||||
func layeredDefinitions(count int) configurationShape {
|
||||
base := make(map[string]any, count)
|
||||
operator := make(map[string]any, count)
|
||||
programmatic := make(map[string]any, count)
|
||||
final := make(map[string]any, count)
|
||||
for index := range count {
|
||||
id := fmt.Sprintf("profile-%03d", index)
|
||||
base[id] = map[string]any{"endpoint": "https://example.test/v1", "model": "base-model"}
|
||||
operator[id] = map[string]any{"model": "operator-model"}
|
||||
programmatic[id] = map[string]any{"extra_params": map[string]any{"provider": map[string]any{"mode": "programmatic"}}}
|
||||
final[id] = map[string]any{"temperature": float64(index%10) / 10}
|
||||
}
|
||||
return configurationShape{
|
||||
underlays: []configkit.Underlay{
|
||||
{Inputs: []configkit.Input{{Source: configkit.Mapping("fallback", map[string]any{"profiles": base})}}},
|
||||
{Inputs: []configkit.Input{{Source: configkit.Mapping("operator", map[string]any{"profiles": operator})}}},
|
||||
{Inputs: []configkit.Input{{Source: configkit.Mapping("programmatic", map[string]any{"profiles": programmatic})}}},
|
||||
},
|
||||
main: configkit.Input{Source: configkit.Mapping("main", map[string]any{
|
||||
"profiles": final,
|
||||
"prompts": map[string]any{"layered": benchmarkPrompt(fmt.Sprintf("profile-%03d", count-1))},
|
||||
})},
|
||||
inputEntries: count*4 + 1,
|
||||
effectiveEntries: count + 1,
|
||||
tiers: 4,
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkCollectionFileBackedConstruction(b *testing.B) {
|
||||
for _, count := range []int{4, 16, 64} {
|
||||
b.Run(fmt.Sprintf("collections_%d", count), func(b *testing.B) {
|
||||
collections := make(map[string]any, count)
|
||||
for index := range count {
|
||||
collections[fmt.Sprintf("collection-%03d", index)] = map[string]any{
|
||||
"prompts": map[string]any{
|
||||
"example": map[string]any{
|
||||
"messages": []any{map[string]any{"role": "user", "content_file": "body.md"}},
|
||||
"output": map[string]any{"format": "text", "validation_mode": "none"},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
namespace := configkit.NewFilesystem("collection benchmark", fstest.MapFS{
|
||||
"body.md": {Data: []byte("Hello")},
|
||||
})
|
||||
main := configkit.Input{
|
||||
Source: configkit.Mapping("collections", map[string]any{"collections": collections}),
|
||||
Filesystem: &configkit.FilesystemContext{FS: &namespace, Root: ".", BaseDir: "."},
|
||||
}
|
||||
b.ReportAllocs()
|
||||
for b.Loop() {
|
||||
result, err := load(main, nil)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
constructionResult = result
|
||||
}
|
||||
b.ReportMetric(float64(count), "collections/op")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkConfigurationConstruction(b *testing.B) {
|
||||
for _, scenario := range []struct {
|
||||
name string
|
||||
shape configurationShape
|
||||
}{
|
||||
{name: "independent", shape: independentDefinitions(benchmarkDefinitionCount)},
|
||||
{name: "file_backed", shape: fileBackedDefinitions(64)},
|
||||
{name: "shared_base", shape: sharedBaseDefinitions(benchmarkDefinitionCount)},
|
||||
{name: "layered", shape: layeredDefinitions(benchmarkDefinitionCount)},
|
||||
} {
|
||||
b.Run(scenario.name, func(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
for b.Loop() {
|
||||
result, err := load(scenario.shape.main, scenario.shape.underlays)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
constructionResult = result
|
||||
}
|
||||
b.ReportMetric(float64(scenario.shape.inputEntries), "input-entries/op")
|
||||
b.ReportMetric(float64(scenario.shape.effectiveEntries), "effective-entries/op")
|
||||
b.ReportMetric(float64(scenario.shape.tiers), "tiers/op")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkConfigurationSelection(b *testing.B) {
|
||||
shape := independentDefinitions(benchmarkDefinitionCount)
|
||||
result, err := load(shape.main, nil)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
b.Run("prompt", func(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
for b.Loop() {
|
||||
definition, _, err := result.prompt("prompt-127")
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
selectedPrompt = definition
|
||||
}
|
||||
b.ReportMetric(float64(shape.effectiveEntries), "effective-entries/op")
|
||||
})
|
||||
b.Run("profile", func(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
for b.Loop() {
|
||||
profile, err := result.GetProfile(context.Background(), "profile-127")
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
selectedProfile = profile
|
||||
}
|
||||
b.ReportMetric(float64(shape.effectiveEntries), "effective-entries/op")
|
||||
})
|
||||
}
|
||||
|
||||
func inspectionExtraParams(entries int) map[string]any {
|
||||
providers := make(map[string]any, entries)
|
||||
for index := range entries {
|
||||
providers[fmt.Sprintf("provider-%03d", index)] = map[string]any{
|
||||
"enabled": true,
|
||||
"labels": []any{fmt.Sprintf("tier-%d", index%4), fmt.Sprintf("region-%d", index%3)},
|
||||
"limits": map[string]any{"burst": index + 1, "period": "minute"},
|
||||
"routes": []any{map[string]any{"weight": index + 1, "fallback": true}},
|
||||
}
|
||||
}
|
||||
return map[string]any{"provider_options": providers}
|
||||
}
|
||||
|
||||
func inspectionDefinitions(extraParams map[string]any) configurationShape {
|
||||
profiles := map[string]any{
|
||||
"small": map[string]any{"model": "inspection-model", "extra_params": extraParams},
|
||||
}
|
||||
prompt := benchmarkPrompt("small")
|
||||
prompt["inputs"] = []any{map[string]any{"name": "transcript", "required": true}}
|
||||
prompts := map[string]any{
|
||||
"inspection": prompt,
|
||||
}
|
||||
return configurationShape{
|
||||
main: configkit.Input{Source: configkit.Mapping("inspection definitions", map[string]any{
|
||||
"profiles": profiles,
|
||||
"prompts": prompts,
|
||||
})},
|
||||
inputEntries: 2, effectiveEntries: 2, tiers: 1,
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkInspectionSelection(b *testing.B) {
|
||||
for _, scenario := range []struct {
|
||||
name string
|
||||
entries int
|
||||
}{
|
||||
{name: "small_extra", entries: 2},
|
||||
{name: "nested_extra", entries: 64},
|
||||
} {
|
||||
shape := inspectionDefinitions(inspectionExtraParams(scenario.entries))
|
||||
result, err := load(shape.main, nil)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
ctx := context.Background()
|
||||
definition, hash, err := result.GetPromptSnapshot(ctx, "inspection")
|
||||
if err != nil || definition.ID != "inspection" || definition.Version == nil || *definition.Version != "1" || hash == "" ||
|
||||
len(definition.Inputs) != 1 || definition.Inputs[0].Name != "transcript" || !definition.Inputs[0].Required {
|
||||
b.Fatalf("prompt lookup fixture is invalid: definition=%+v hash=%q err=%v", definition, hash, err)
|
||||
}
|
||||
profile, err := result.GetProfile(ctx, "small")
|
||||
if err != nil || profile.ID != "small" || profile.ResolvedTarget.Model != "inspection-model" ||
|
||||
len(profile.ResolvedTarget.ExtraParams["provider_options"].(map[string]any)) != scenario.entries {
|
||||
b.Fatalf("profile lookup fixture is invalid: profile=%+v err=%v", profile, err)
|
||||
}
|
||||
inspection, err := result.GetPromptInspection(ctx, "inspection")
|
||||
if err != nil || inspection.PromptID != "inspection" || inspection.PromptVersion == nil || *inspection.PromptVersion != "1" ||
|
||||
len(inspection.Inputs) != 1 || inspection.Inputs[0].Name != "transcript" {
|
||||
b.Fatalf("prompt inspection fixture is invalid: inspection=%+v err=%v", inspection, err)
|
||||
}
|
||||
identity, err := result.GetProfileIdentity(ctx, "small")
|
||||
if err != nil || identity != "small" {
|
||||
b.Fatalf("profile identity fixture is invalid: identity=%q err=%v", identity, err)
|
||||
}
|
||||
|
||||
b.Run("prompt/"+scenario.name, func(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
for b.Loop() {
|
||||
definition, hash, err := result.GetPromptSnapshot(ctx, "inspection")
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
selectedPrompt = definition
|
||||
selectedPromptHash = hash
|
||||
}
|
||||
b.ReportMetric(float64(scenario.entries), "extra-entries/op")
|
||||
})
|
||||
b.Run("prompt_inspection/"+scenario.name, func(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
for b.Loop() {
|
||||
inspection, err := result.GetPromptInspection(ctx, "inspection")
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
selectedPromptInspection = inspection
|
||||
}
|
||||
b.ReportMetric(float64(scenario.entries), "extra-entries/op")
|
||||
})
|
||||
b.Run("profile/"+scenario.name, func(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
for b.Loop() {
|
||||
profile, err := result.GetProfile(ctx, "small")
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
selectedProfile = profile
|
||||
}
|
||||
b.ReportMetric(float64(scenario.entries), "extra-entries/op")
|
||||
})
|
||||
b.Run("profile_identity/"+scenario.name, func(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
for b.Loop() {
|
||||
identity, err := result.GetProfileIdentity(ctx, "small")
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
selectedProfileIdentity = identity
|
||||
}
|
||||
b.ReportMetric(float64(scenario.entries), "extra-entries/op")
|
||||
})
|
||||
}
|
||||
}
|
||||
76
internal/configuration/inspection.go
Normal file
76
internal/configuration/inspection.go
Normal file
@@ -0,0 +1,76 @@
|
||||
package configuration
|
||||
|
||||
import (
|
||||
"io"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
||||
)
|
||||
|
||||
// Inspection is the read-only subset of the retained ConfigKit result used by
|
||||
// the facade. It does not expose typed configuration or resource contexts.
|
||||
type Inspection interface {
|
||||
Metadata() configkit.Metadata
|
||||
Sources() []configkit.SourceInfo
|
||||
Diagnostics() []configkit.Diagnostic
|
||||
Explain(configkit.Path) (configkit.Explanation, bool)
|
||||
WriteEffective(io.Writer) error
|
||||
WriteExplanation(io.Writer, configkit.Path) error
|
||||
capturedResult() *configkit.Result[document]
|
||||
}
|
||||
|
||||
type inspectionResult struct {
|
||||
*configkit.Result[document]
|
||||
}
|
||||
|
||||
func (inspection inspectionResult) capturedResult() *configkit.Result[document] {
|
||||
return inspection.Result
|
||||
}
|
||||
|
||||
// Inspection returns only the retained result's inspection operations.
|
||||
func (state *loaded) Inspection() Inspection {
|
||||
if state == nil {
|
||||
return nil
|
||||
}
|
||||
return inspectionResult{Result: state.result}
|
||||
}
|
||||
|
||||
// Diff compares two captured ConfigKit results without reconstructing them.
|
||||
func Diff(left, right Inspection) ([]configkit.Difference, error) {
|
||||
var leftResult, rightResult *configkit.Result[document]
|
||||
if left != nil {
|
||||
leftResult = left.capturedResult()
|
||||
}
|
||||
if right != nil {
|
||||
rightResult = right.capturedResult()
|
||||
}
|
||||
return configkit.Diff(leftResult, rightResult)
|
||||
}
|
||||
|
||||
func secretProviderPath(path configkit.Path) bool {
|
||||
pointer := path.String()
|
||||
if pointer == "" {
|
||||
return false
|
||||
}
|
||||
parts := strings.Split(strings.TrimPrefix(pointer, "/"), "/")
|
||||
if !providerExtraParamsPath(parts) {
|
||||
return false
|
||||
}
|
||||
key := strings.ToLower(strings.ReplaceAll(strings.ReplaceAll(parts[len(parts)-1], "~1", "/"), "~0", "~"))
|
||||
switch key {
|
||||
case "api_key", "apikey", "authorization", "credential", "credentials", "password", "secret", "token", "access_token":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func providerExtraParamsPath(parts []string) bool {
|
||||
if len(parts) >= 4 && parts[1] != "" &&
|
||||
(parts[0] == "backends" || parts[0] == "profiles") && parts[2] == "extra_params" {
|
||||
return true
|
||||
}
|
||||
return len(parts) >= 6 && parts[0] == "collections" && parts[1] != "" &&
|
||||
(parts[2] == "backends" || parts[2] == "profiles") &&
|
||||
parts[3] != "" && parts[4] == "extra_params"
|
||||
}
|
||||
32
internal/configuration/inspection_test.go
Normal file
32
internal/configuration/inspection_test.go
Normal file
@@ -0,0 +1,32 @@
|
||||
package configuration
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
||||
)
|
||||
|
||||
func TestSecretProviderPathRecognizesResourceExtraParams(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
path configkit.Path
|
||||
want bool
|
||||
}{
|
||||
{name: "backend nested secret", path: configkit.Keys("backends", "provider", "extra_params", "nested", "Token"), want: true},
|
||||
{name: "profile nested secret", path: configkit.Keys("profiles", "assistant", "extra_params", "authorization"), want: true},
|
||||
{name: "collection backend nested secret", path: configkit.Keys("collections", "provider", "backends", "default", "extra_params", "nested", "Token"), want: true},
|
||||
{name: "collection profile secret", path: configkit.Keys("collections", "assistant", "profiles", "model", "extra_params", "authorization"), want: true},
|
||||
{name: "unrelated nested mapping", path: configkit.Keys("application", "backends", "provider", "extra_params", "token"), want: false},
|
||||
{name: "extra params outside resource map", path: configkit.Keys("backends", "provider", "options", "extra_params", "token"), want: false},
|
||||
{name: "collection extra params outside resource map", path: configkit.Keys("collections", "provider", "settings", "backends", "default", "extra_params", "token"), want: false},
|
||||
{name: "missing resource key", path: configkit.Keys("backends", "", "extra_params", "token"), want: false},
|
||||
{name: "ordinary provider value", path: configkit.Keys("profiles", "assistant", "extra_params", "provider", "mode"), want: false},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
if got := secretProviderPath(test.path); got != test.want {
|
||||
t.Fatalf("secretProviderPath(%q) = %t, want %t", test.path, got, test.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
87
internal/configuration/listing.go
Normal file
87
internal/configuration/listing.go
Normal file
@@ -0,0 +1,87 @@
|
||||
package configuration
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
)
|
||||
|
||||
// CollectionIdentity is the captured metadata for one final collection entry.
|
||||
type CollectionIdentity struct {
|
||||
Namespace string
|
||||
ReleaseVersion *string
|
||||
}
|
||||
|
||||
func (state *loaded) ListProfiles(ctx context.Context) ([]string, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
profiles := append([]string(nil), state.profileIDs...)
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return profiles, nil
|
||||
}
|
||||
|
||||
func (state *loaded) ListBackends(ctx context.Context) ([]string, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
backends := append([]string(nil), state.backendIDs...)
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return backends, nil
|
||||
}
|
||||
|
||||
func (state *loaded) ListNamespaces(ctx context.Context, kinds []string) ([]string, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
selected := kinds
|
||||
if len(selected) == 0 {
|
||||
selected = []string{"backend", "profile", "prompt"}
|
||||
}
|
||||
set := make(map[string]struct{})
|
||||
for _, kind := range selected {
|
||||
for _, namespace := range state.namespaces[kind] {
|
||||
set[namespace] = struct{}{}
|
||||
}
|
||||
}
|
||||
namespaces := sortedKeys(set)
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return namespaces, nil
|
||||
}
|
||||
|
||||
func (state *loaded) ListCollections(ctx context.Context) ([]CollectionIdentity, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
collections := make([]CollectionIdentity, len(state.collections))
|
||||
for index, collection := range state.collections {
|
||||
collections[index] = CollectionIdentity{
|
||||
Namespace: collection.Namespace,
|
||||
ReleaseVersion: stringPointerValue(collection.ReleaseVersion),
|
||||
}
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return collections, nil
|
||||
}
|
||||
|
||||
func (state *loaded) ListPrompts(ctx context.Context) ([]domain.PromptIdentity, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
prompts := append([]domain.PromptIdentity(nil), state.promptIDs...)
|
||||
for index := range prompts {
|
||||
prompts[index].Version = stringPointerValue(prompts[index].Version)
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return prompts, nil
|
||||
}
|
||||
160
internal/configuration/listing_test.go
Normal file
160
internal/configuration/listing_test.go
Normal file
@@ -0,0 +1,160 @@
|
||||
package configuration
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
)
|
||||
|
||||
func TestLoadedListsSortedEffectiveIdentities(t *testing.T) {
|
||||
state, err := Load(mapping("identities", map[string]any{
|
||||
"backends": map[string]any{
|
||||
"local": map[string]any{"endpoint": "http://localhost:8080/v1"},
|
||||
},
|
||||
"collections": map[string]any{
|
||||
"z": map[string]any{"profiles": map[string]any{"profile": map[string]any{"backend": "local", "model": "z-model"}}, "prompts": map[string]any{"prompt": listingPromptDefinition()}},
|
||||
"partial": map[string]any{"profiles": map[string]any{"profile": map[string]any{"backend": "local"}}},
|
||||
"a": map[string]any{"profiles": map[string]any{"profile": map[string]any{"backend": "local", "model": "a-model"}}, "prompts": map[string]any{"prompt": listingPromptDefinition()}},
|
||||
" z": map[string]any{"profiles": map[string]any{"profile ": map[string]any{"backend": "local", "model": "spaced-model"}}, "prompts": map[string]any{"prompt ": listingPromptDefinition()}},
|
||||
},
|
||||
}), nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
backends, err := state.ListBackends(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wantBackends := []string{"local"}
|
||||
if !reflect.DeepEqual(backends, wantBackends) {
|
||||
t.Fatalf("backend IDs = %#v, want %#v", backends, wantBackends)
|
||||
}
|
||||
|
||||
profiles, err := state.ListProfiles(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wantProfiles := []string{" z/profile ", "a/profile", "partial/profile", "z/profile"}
|
||||
if !reflect.DeepEqual(profiles, wantProfiles) {
|
||||
t.Fatalf("profile IDs = %#v, want %#v", profiles, wantProfiles)
|
||||
}
|
||||
|
||||
prompts, err := state.ListPrompts(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wantPrompts := []domain.PromptIdentity{
|
||||
{ID: " z/prompt ", Version: stringPointer("v1")},
|
||||
{ID: "a/prompt", Version: stringPointer("v1")},
|
||||
{ID: "z/prompt", Version: stringPointer("v1")},
|
||||
}
|
||||
if !reflect.DeepEqual(prompts, wantPrompts) {
|
||||
t.Fatalf("prompt identities = %#v, want %#v", prompts, wantPrompts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadedListsAreIndependent(t *testing.T) {
|
||||
state, err := Load(mapping("identities", map[string]any{
|
||||
"backends": map[string]any{"backend": map[string]any{"endpoint": "https://example.test/v1"}},
|
||||
"profiles": map[string]any{"profile": map[string]any{}},
|
||||
"prompts": map[string]any{"example": listingPromptDefinition()},
|
||||
}), nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
profiles, err := state.ListProfiles(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
profiles = append(profiles, "caller profile")
|
||||
profiles[0] = "changed profile"
|
||||
backends, err := state.ListBackends(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
backends[0] = "changed backend"
|
||||
|
||||
prompts, err := state.ListPrompts(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
prompts[0].ID = "changed prompt"
|
||||
*prompts[0].Version = "changed version"
|
||||
|
||||
profiles, err = state.ListProfiles(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(profiles, []string{"profile"}) {
|
||||
t.Fatalf("caller mutation changed profile inventory: %#v", profiles)
|
||||
}
|
||||
backends, err = state.ListBackends(context.Background())
|
||||
if err != nil || !reflect.DeepEqual(backends, []string{"backend"}) {
|
||||
t.Fatalf("caller mutation changed backend inventory: %#v, %v", backends, err)
|
||||
}
|
||||
prompts, err = state.ListPrompts(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wantPrompts := []domain.PromptIdentity{{ID: "example", Version: stringPointer("v1")}}
|
||||
if !reflect.DeepEqual(prompts, wantPrompts) {
|
||||
t.Fatalf("caller mutation changed prompt inventory: %#v", prompts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadedListsEmptyInventory(t *testing.T) {
|
||||
state, err := Load(configkit.Input{Source: configkit.NoSource()}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
profiles, err := state.ListProfiles(context.Background())
|
||||
if err != nil || len(profiles) != 0 {
|
||||
t.Fatalf("empty profile listing = %#v, %v", profiles, err)
|
||||
}
|
||||
backends, err := state.ListBackends(context.Background())
|
||||
if err != nil || len(backends) != 0 {
|
||||
t.Fatalf("empty backend listing = %#v, %v", backends, err)
|
||||
}
|
||||
prompts, err := state.ListPrompts(context.Background())
|
||||
if err != nil || len(prompts) != 0 {
|
||||
t.Fatalf("empty prompt listing = %#v, %v", prompts, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadedListsHonorCanceledContext(t *testing.T) {
|
||||
state, err := Load(mapping("identities", map[string]any{
|
||||
"profiles": map[string]any{"partial": map[string]any{}},
|
||||
"prompts": map[string]any{"example": listingPromptDefinition()},
|
||||
}), nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
|
||||
profiles, err := state.ListProfiles(ctx)
|
||||
if !errors.Is(err, context.Canceled) || len(profiles) != 0 {
|
||||
t.Fatalf("canceled profile listing = %#v, %v", profiles, err)
|
||||
}
|
||||
backends, err := state.ListBackends(ctx)
|
||||
if !errors.Is(err, context.Canceled) || len(backends) != 0 {
|
||||
t.Fatalf("canceled backend listing = %#v, %v", backends, err)
|
||||
}
|
||||
prompts, err := state.ListPrompts(ctx)
|
||||
if !errors.Is(err, context.Canceled) || len(prompts) != 0 {
|
||||
t.Fatalf("canceled prompt listing = %#v, %v", prompts, err)
|
||||
}
|
||||
}
|
||||
|
||||
func listingPromptDefinition() map[string]any {
|
||||
return map[string]any{
|
||||
"version": "v1",
|
||||
"messages": []any{map[string]any{"role": "user", "content": "Hello."}},
|
||||
"output": map[string]any{"format": "text", "validation_mode": "basic"},
|
||||
}
|
||||
}
|
||||
587
internal/configuration/load.go
Normal file
587
internal/configuration/load.go
Normal file
@@ -0,0 +1,587 @@
|
||||
// Package configuration adapts ConfigKit's effective configuration to
|
||||
// Promptkit's definition and execution boundaries.
|
||||
package configuration
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/backend"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/jsonvalue"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/llm"
|
||||
)
|
||||
|
||||
type document struct {
|
||||
Version *string `yaml:"version"`
|
||||
Backends map[string]backendConfig `yaml:"backends"`
|
||||
Profiles map[string]profileConfig `yaml:"profiles"`
|
||||
Prompts map[string]promptConfig `yaml:"prompts"`
|
||||
Collections map[string]collectionConfig `yaml:"collections"`
|
||||
}
|
||||
|
||||
type collectionConfig struct {
|
||||
ReleaseVersion *string `yaml:"release_version"`
|
||||
Backends map[string]backendConfig `yaml:"backends"`
|
||||
Profiles map[string]profileConfig `yaml:"profiles"`
|
||||
Prompts map[string]promptConfig `yaml:"prompts"`
|
||||
}
|
||||
|
||||
type backendConfig struct {
|
||||
Endpoint *string `yaml:"endpoint"`
|
||||
APIKeyEnv *string `yaml:"api_key_env"`
|
||||
ConcurrencyLimit *int `yaml:"concurrency_limit"`
|
||||
QueueCapacity *int `yaml:"queue_capacity"`
|
||||
ExtraParams map[string]any `yaml:"extra_params"`
|
||||
SmokeTests *backendSmokeTestConfig `yaml:"smoke_tests"`
|
||||
}
|
||||
|
||||
type backendSmokeTestConfig struct {
|
||||
DefaultProfile *string `yaml:"default_profile"`
|
||||
Reachability *reachabilityProbeConfig `yaml:"reachability"`
|
||||
Authentication *authenticationProbeConfig `yaml:"authentication"`
|
||||
}
|
||||
|
||||
type reachabilityProbeConfig struct {
|
||||
Endpoint *string `yaml:"endpoint"`
|
||||
}
|
||||
|
||||
type authenticationProbeConfig struct {
|
||||
Kind *string `yaml:"kind"`
|
||||
Endpoint *string `yaml:"endpoint"`
|
||||
}
|
||||
|
||||
type profileConfig struct {
|
||||
Backend *string `yaml:"backend"`
|
||||
Endpoint *string `yaml:"endpoint"`
|
||||
Model *string `yaml:"model"`
|
||||
Temperature *float64 `yaml:"temperature"`
|
||||
TopP *float64 `yaml:"top_p"`
|
||||
MaxTokens *int `yaml:"max_tokens"`
|
||||
TimeoutSeconds *int `yaml:"timeout_seconds"`
|
||||
ServiceTier *string `yaml:"service_tier"`
|
||||
ReasoningEffort *string `yaml:"reasoning_effort"`
|
||||
APIKeyEnv *string `yaml:"api_key_env"`
|
||||
APIKeyRequired *bool `yaml:"api_key_required"`
|
||||
ExtraParams map[string]any `yaml:"extra_params"`
|
||||
}
|
||||
|
||||
type promptConfig struct {
|
||||
Version *string `yaml:"version"`
|
||||
DefaultProfile *string `yaml:"default_profile"`
|
||||
Description *string `yaml:"description"`
|
||||
SessionID *string `yaml:"session_id"`
|
||||
Inputs []promptInputConfig `yaml:"inputs"`
|
||||
Messages []promptMessageConfig `yaml:"messages"`
|
||||
Output *promptOutputConfig `yaml:"output"`
|
||||
}
|
||||
|
||||
type promptInputConfig struct {
|
||||
Name string `yaml:"name"`
|
||||
Required bool `yaml:"required"`
|
||||
ContentType *string `yaml:"content_type"`
|
||||
Description *string `yaml:"description"`
|
||||
}
|
||||
|
||||
type promptMessageConfig struct {
|
||||
Role *string `yaml:"role"`
|
||||
Content *string `yaml:"content"`
|
||||
ContentFile *string `yaml:"content_file"`
|
||||
CacheControl *cacheControlConfig `yaml:"cache_control"`
|
||||
}
|
||||
|
||||
type cacheControlConfig struct {
|
||||
Type *string `yaml:"type"`
|
||||
TTL *string `yaml:"ttl"`
|
||||
}
|
||||
|
||||
type promptOutputConfig struct {
|
||||
Format *string `yaml:"format"`
|
||||
ValidationMode *string `yaml:"validation_mode"`
|
||||
SchemaPath *string `yaml:"schema_path"`
|
||||
RepairAttempts int `yaml:"repair_attempts"`
|
||||
}
|
||||
|
||||
// fieldState distinguishes omission, an explicit value, and an explicit null.
|
||||
// Inherited values are explicit because ConfigKit has already expanded them.
|
||||
type fieldState uint8
|
||||
|
||||
const (
|
||||
fieldAbsent fieldState = iota
|
||||
fieldValue
|
||||
fieldNull
|
||||
)
|
||||
|
||||
type loaded struct {
|
||||
result *configkit.Result[document]
|
||||
backends map[string]map[string]fieldState
|
||||
profiles map[string]map[string]fieldState
|
||||
promptMetadata map[string][]promptInputPresence
|
||||
prompts map[string]*promptEntry
|
||||
registry *backend.Registry
|
||||
indexedProfiles map[string]domain.ExecutionProfile
|
||||
backendIDs []string
|
||||
profileIDs []string
|
||||
promptIDs []domain.PromptIdentity
|
||||
namespaces map[string][]string
|
||||
collections []CollectionIdentity
|
||||
}
|
||||
|
||||
type resourceBases struct {
|
||||
backends configkit.Path
|
||||
profiles configkit.Path
|
||||
prompts configkit.Path
|
||||
}
|
||||
|
||||
type resourceSet struct {
|
||||
isCollection bool
|
||||
namespace string
|
||||
bases resourceBases
|
||||
backends map[string]backendConfig
|
||||
profiles map[string]profileConfig
|
||||
prompts map[string]promptConfig
|
||||
}
|
||||
|
||||
func rootResourceBases() resourceBases {
|
||||
return resourceBases{
|
||||
backends: configkit.Keys("backends"),
|
||||
profiles: configkit.Keys("profiles"),
|
||||
prompts: configkit.Keys("prompts"),
|
||||
}
|
||||
}
|
||||
|
||||
func resourceSets(config *document) []resourceSet {
|
||||
sets := make([]resourceSet, 0, len(config.Collections)+1)
|
||||
sets = append(sets, resourceSet{
|
||||
bases: rootResourceBases(), backends: config.Backends, profiles: config.Profiles, prompts: config.Prompts,
|
||||
})
|
||||
for _, namespace := range sortedKeys(config.Collections) {
|
||||
collection := config.Collections[namespace]
|
||||
base := configkit.Keys("collections", namespace)
|
||||
sets = append(sets, resourceSet{
|
||||
isCollection: true,
|
||||
namespace: namespace,
|
||||
bases: resourceBases{
|
||||
backends: base.Key("backends"),
|
||||
profiles: base.Key("profiles"),
|
||||
prompts: base.Key("prompts"),
|
||||
},
|
||||
backends: collection.Backends,
|
||||
profiles: collection.Profiles,
|
||||
prompts: collection.Prompts,
|
||||
})
|
||||
}
|
||||
return sets
|
||||
}
|
||||
|
||||
func indexedResourceID(namespace, localKey string) string {
|
||||
qualified, valid := domain.QualifyLocalResourceID(namespace, localKey)
|
||||
if valid {
|
||||
return qualified
|
||||
}
|
||||
if namespace == "" {
|
||||
return localKey
|
||||
}
|
||||
return namespace + "/" + localKey
|
||||
}
|
||||
|
||||
func indexedBackends(config *document) map[string]backendConfig {
|
||||
backends := make(map[string]backendConfig)
|
||||
for _, resources := range resourceSets(config) {
|
||||
for localKey, definition := range resources.backends {
|
||||
backends[indexedResourceID(resources.namespace, localKey)] = definition
|
||||
}
|
||||
}
|
||||
return backends
|
||||
}
|
||||
|
||||
func indexedProfiles(config *document) map[string]profileConfig {
|
||||
profiles := make(map[string]profileConfig)
|
||||
for _, resources := range resourceSets(config) {
|
||||
for localKey, definition := range resources.profiles {
|
||||
profiles[indexedResourceID(resources.namespace, localKey)] = definition
|
||||
}
|
||||
}
|
||||
return profiles
|
||||
}
|
||||
|
||||
var envName = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`)
|
||||
|
||||
var backendFields = []string{"endpoint", "api_key_env", "concurrency_limit", "queue_capacity", "extra_params", "smoke_tests"}
|
||||
var profileFields = []string{"backend", "endpoint", "model", "temperature", "top_p", "max_tokens", "timeout_seconds", "service_tier", "reasoning_effort", "api_key_env", "api_key_required", "extra_params"}
|
||||
|
||||
// Load captures one immutable effective configuration for internal consumers.
|
||||
func Load(main configkit.Input, underlays []configkit.Underlay) (*loaded, error) {
|
||||
return load(main, underlays)
|
||||
}
|
||||
|
||||
// load performs one ConfigKit lifecycle over the selected source tiers.
|
||||
func load(main configkit.Input, underlays []configkit.Underlay) (*loaded, error) {
|
||||
state := &loaded{
|
||||
backends: make(map[string]map[string]fieldState),
|
||||
profiles: make(map[string]map[string]fieldState),
|
||||
promptMetadata: make(map[string][]promptInputPresence),
|
||||
}
|
||||
result, err := configkit.LoadLayered(main, configkit.LayeredOptions[document]{
|
||||
Underlays: underlays,
|
||||
SourceVersionMode: configkit.SourceVersionModeIfPresent,
|
||||
Definition: configkit.Definition[document]{
|
||||
Version: &configkit.VersionRule{
|
||||
Path: configkit.Keys("version"),
|
||||
Check: checkConfigurationVersion,
|
||||
},
|
||||
Secret: secretProviderPath,
|
||||
Resolve: func(config *document, resolution *configkit.Resolution) error {
|
||||
for _, resources := range resourceSets(config) {
|
||||
captureFields(resources.backends, resources.bases.backends, resources.namespace, backendFields, resolution, state.backends)
|
||||
captureFields(resources.profiles, resources.bases.profiles, resources.namespace, profileFields, resolution, state.profiles)
|
||||
capturePromptMetadata(resources.prompts, resources.bases.prompts, resources.namespace, resolution, state.promptMetadata)
|
||||
resolveBackends(resources.backends, resources.bases.backends, resolution)
|
||||
resolveProfiles(resources.profiles, resources.bases.profiles, resolution)
|
||||
resolvePrompts(resources.prompts, resources.bases.prompts, resolution)
|
||||
}
|
||||
return nil
|
||||
},
|
||||
Validate: func(config *document, validation *configkit.Validation) {
|
||||
validateEffective(config, validation)
|
||||
},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("load configuration: %w", err)
|
||||
}
|
||||
state.result = result
|
||||
if err := state.materializePrompts(); err != nil {
|
||||
return nil, fmt.Errorf("load configuration: %w", err)
|
||||
}
|
||||
if err := state.buildExecutionProfiles(); err != nil {
|
||||
return nil, fmt.Errorf("load configuration: %w", err)
|
||||
}
|
||||
state.backendIDs = sortedKeys(indexedBackends(&result.Config))
|
||||
state.profileIDs = sortedKeys(state.indexedProfiles)
|
||||
for _, id := range sortedKeys(state.prompts) {
|
||||
state.promptIDs = append(state.promptIDs, domain.PromptIdentity{ID: id, Version: stringPointerValue(state.prompts[id].definition.Version)})
|
||||
}
|
||||
state.namespaces = map[string][]string{
|
||||
"backend": resourceNamespaces(state.backendIDs),
|
||||
"profile": resourceNamespaces(state.profileIDs),
|
||||
"prompt": promptNamespaces(state.promptIDs),
|
||||
}
|
||||
for _, namespace := range sortedKeys(result.Config.Collections) {
|
||||
collection := result.Config.Collections[namespace]
|
||||
state.collections = append(state.collections, CollectionIdentity{
|
||||
Namespace: namespace, ReleaseVersion: stringPointerValue(collection.ReleaseVersion),
|
||||
})
|
||||
}
|
||||
return state, nil
|
||||
}
|
||||
|
||||
func resourceNamespaces(ids []string) []string {
|
||||
namespaces := make(map[string]struct{})
|
||||
for _, id := range ids {
|
||||
if namespace, _, qualified := strings.Cut(id, "/"); qualified {
|
||||
namespaces[namespace] = struct{}{}
|
||||
}
|
||||
}
|
||||
return sortedKeys(namespaces)
|
||||
}
|
||||
|
||||
func promptNamespaces(prompts []domain.PromptIdentity) []string {
|
||||
ids := make([]string, len(prompts))
|
||||
for index := range prompts {
|
||||
ids[index] = prompts[index].ID
|
||||
}
|
||||
return resourceNamespaces(ids)
|
||||
}
|
||||
|
||||
func checkConfigurationVersion(version configkit.Version, validation *configkit.Validation) error {
|
||||
if !version.Present {
|
||||
return nil
|
||||
}
|
||||
if version.Null || version.Text != "1" {
|
||||
validation.Error(configkit.Keys("version"), "unsupported configuration specification version")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type promptInputPresence struct {
|
||||
contentType fieldState
|
||||
description fieldState
|
||||
}
|
||||
|
||||
func capturePromptMetadata(prompts map[string]promptConfig, basePath configkit.Path, namespace string, resolution *configkit.Resolution, destination map[string][]promptInputPresence) {
|
||||
for _, id := range sortedKeys(prompts) {
|
||||
inputs := prompts[id].Inputs
|
||||
states := make([]promptInputPresence, len(inputs))
|
||||
base := basePath.Key(id).Key("inputs")
|
||||
for index := range inputs {
|
||||
path := base.Index(index)
|
||||
states[index] = promptInputPresence{
|
||||
contentType: captureFieldState(path.Key("content_type"), resolution),
|
||||
description: captureFieldState(path.Key("description"), resolution),
|
||||
}
|
||||
}
|
||||
destination[indexedResourceID(namespace, id)] = states
|
||||
}
|
||||
}
|
||||
|
||||
func captureFieldState(path configkit.Path, resolution *configkit.Resolution) fieldState {
|
||||
info, exists := resolution.InputInfo(path)
|
||||
switch {
|
||||
case !exists || !info.Explicit:
|
||||
return fieldAbsent
|
||||
case info.Null:
|
||||
return fieldNull
|
||||
default:
|
||||
return fieldValue
|
||||
}
|
||||
}
|
||||
|
||||
func captureFields[T any](entries map[string]T, basePath configkit.Path, namespace string, fields []string, resolution *configkit.Resolution, destination map[string]map[string]fieldState) {
|
||||
for id := range entries {
|
||||
states := make(map[string]fieldState, len(fields))
|
||||
for _, field := range fields {
|
||||
info, exists := resolution.InputInfo(basePath.Key(id).Key(field))
|
||||
switch {
|
||||
case !exists || !info.Explicit:
|
||||
states[field] = fieldAbsent
|
||||
case info.Null:
|
||||
states[field] = fieldNull
|
||||
default:
|
||||
states[field] = fieldValue
|
||||
}
|
||||
}
|
||||
destination[indexedResourceID(namespace, id)] = states
|
||||
}
|
||||
}
|
||||
|
||||
func sortedKeys[T any](entries map[string]T) []string {
|
||||
keys := make([]string, 0, len(entries))
|
||||
for key := range entries {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
return keys
|
||||
}
|
||||
|
||||
func normalizeString(value **string, path configkit.Path, resolution *configkit.Resolution, normalized string) bool {
|
||||
if value != nil && *value != nil && **value != normalized {
|
||||
configkit.Normalize(resolution, path, value, &normalized, "normalize configuration value")
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func resolveBackends(backends map[string]backendConfig, basePath configkit.Path, resolution *configkit.Resolution) {
|
||||
for _, id := range sortedKeys(backends) {
|
||||
backend := backends[id]
|
||||
base := basePath.Key(id)
|
||||
if backend.Endpoint != nil {
|
||||
endpoint, err := domain.NormalizeOpenAICompatibleBaseEndpoint(*backend.Endpoint)
|
||||
if err != nil {
|
||||
resolution.Error(base.Key("endpoint"), err.Error())
|
||||
} else {
|
||||
normalizeString(&backend.Endpoint, base.Key("endpoint"), resolution, endpoint)
|
||||
}
|
||||
}
|
||||
if backend.APIKeyEnv != nil {
|
||||
normalizeString(&backend.APIKeyEnv, base.Key("api_key_env"), resolution, strings.TrimSpace(*backend.APIKeyEnv))
|
||||
}
|
||||
if smokeTests := backend.SmokeTests; smokeTests != nil {
|
||||
smokePath := base.Key("smoke_tests")
|
||||
if smokeTests.Reachability != nil {
|
||||
normalizeDiagnosticEndpoint(&smokeTests.Reachability.Endpoint, smokePath.Key("reachability").Key("endpoint"), resolution)
|
||||
}
|
||||
if smokeTests.Authentication != nil {
|
||||
if smokeTests.Authentication.Kind != nil {
|
||||
normalizeString(&smokeTests.Authentication.Kind,
|
||||
smokePath.Key("authentication").Key("kind"), resolution,
|
||||
strings.ToLower(strings.TrimSpace(*smokeTests.Authentication.Kind)))
|
||||
}
|
||||
normalizeDiagnosticEndpoint(&smokeTests.Authentication.Endpoint,
|
||||
smokePath.Key("authentication").Key("endpoint"), resolution)
|
||||
}
|
||||
}
|
||||
if backend.ConcurrencyLimit != nil && *backend.ConcurrencyLimit > 0 && backend.QueueCapacity == nil {
|
||||
capacity := 1024
|
||||
if resolution.IsExplicit(base.Key("queue_capacity")) {
|
||||
configkit.Normalize(resolution, base.Key("queue_capacity"), &backend.QueueCapacity, &capacity, "select default queue capacity")
|
||||
} else {
|
||||
configkit.Default(resolution, base.Key("queue_capacity"), &backend.QueueCapacity, &capacity, "select default queue capacity")
|
||||
}
|
||||
}
|
||||
backends[id] = backend
|
||||
}
|
||||
}
|
||||
|
||||
func normalizeDiagnosticEndpoint(value **string, path configkit.Path, resolution *configkit.Resolution) {
|
||||
if value == nil || *value == nil {
|
||||
return
|
||||
}
|
||||
normalized := strings.TrimSpace(**value)
|
||||
request, err := http.NewRequest(http.MethodGet, normalized, nil)
|
||||
if normalized == "" || err != nil || request.URL == nil ||
|
||||
(strings.ToLower(request.URL.Scheme) != "http" && strings.ToLower(request.URL.Scheme) != "https") ||
|
||||
request.URL.Hostname() == "" {
|
||||
resolution.Error(path, "diagnostic endpoint must be an absolute HTTP(S) URL with a host")
|
||||
return
|
||||
}
|
||||
normalizeString(value, path, resolution, normalized)
|
||||
}
|
||||
|
||||
func resolveProfiles(profiles map[string]profileConfig, basePath configkit.Path, resolution *configkit.Resolution) {
|
||||
for _, id := range sortedKeys(profiles) {
|
||||
profile := profiles[id]
|
||||
base := basePath.Key(id)
|
||||
if profile.Endpoint != nil && *profile.Endpoint != "" {
|
||||
endpoint, err := domain.NormalizeOpenAICompatibleBaseEndpoint(*profile.Endpoint)
|
||||
if err != nil {
|
||||
resolution.Error(base.Key("endpoint"), err.Error())
|
||||
} else {
|
||||
normalizeString(&profile.Endpoint, base.Key("endpoint"), resolution, endpoint)
|
||||
}
|
||||
}
|
||||
for _, field := range []struct {
|
||||
name string
|
||||
value **string
|
||||
}{
|
||||
{"service_tier", &profile.ServiceTier},
|
||||
{"reasoning_effort", &profile.ReasoningEffort},
|
||||
{"api_key_env", &profile.APIKeyEnv},
|
||||
} {
|
||||
if *field.value != nil {
|
||||
normalizeString(field.value, base.Key(field.name), resolution, strings.TrimSpace(**field.value))
|
||||
}
|
||||
}
|
||||
profiles[id] = profile
|
||||
}
|
||||
}
|
||||
|
||||
func validateEffective(config *document, validation *configkit.Validation) {
|
||||
sets := resourceSets(config)
|
||||
backendIDs := make(map[string]struct{})
|
||||
for _, resources := range sets {
|
||||
if resources.isCollection && !domain.IsValidResourceNamespace(resources.namespace) {
|
||||
validation.Error(configkit.Keys("collections", resources.namespace), "collection namespace must be nonblank and contain no slash")
|
||||
}
|
||||
for localKey := range resources.backends {
|
||||
backendIDs[indexedResourceID(resources.namespace, localKey)] = struct{}{}
|
||||
}
|
||||
}
|
||||
for _, resources := range sets {
|
||||
for _, id := range sortedKeys(resources.backends) {
|
||||
backend := resources.backends[id]
|
||||
base := resources.bases.backends.Key(id)
|
||||
if !domain.IsValidLocalResourceKey(id) {
|
||||
validation.Error(base, "backend key must be a nonblank local key without a slash")
|
||||
}
|
||||
if backend.Endpoint == nil || *backend.Endpoint == "" {
|
||||
validation.Error(base.Key("endpoint"), "endpoint is required")
|
||||
}
|
||||
if backend.APIKeyEnv != nil && *backend.APIKeyEnv != "" && !envName.MatchString(*backend.APIKeyEnv) {
|
||||
validation.Error(base.Key("api_key_env"), "environment variable name is invalid")
|
||||
}
|
||||
if backend.ConcurrencyLimit != nil && *backend.ConcurrencyLimit < 0 {
|
||||
validation.Error(base.Key("concurrency_limit"), "concurrency limit must not be negative")
|
||||
}
|
||||
if backend.QueueCapacity != nil {
|
||||
if *backend.QueueCapacity < 0 {
|
||||
validation.Error(base.Key("queue_capacity"), "queue capacity must not be negative")
|
||||
}
|
||||
if backend.ConcurrencyLimit == nil || *backend.ConcurrencyLimit <= 0 {
|
||||
validation.Error(base.Key("queue_capacity"), "queue capacity requires a positive concurrency limit")
|
||||
} else if *backend.QueueCapacity > int(^uint(0)>>1)-*backend.ConcurrencyLimit {
|
||||
validation.Error(base.Key("queue_capacity"), "total capacity overflows int")
|
||||
}
|
||||
}
|
||||
validateExtraParams(backend.ExtraParams, base.Key("extra_params"), true, validation)
|
||||
if smokeTests := backend.SmokeTests; smokeTests != nil {
|
||||
smokePath := base.Key("smoke_tests")
|
||||
if smokeTests.DefaultProfile != nil {
|
||||
if strings.TrimSpace(*smokeTests.DefaultProfile) == "" {
|
||||
validation.Error(smokePath.Key("default_profile"), "default profile must not be blank")
|
||||
} else if !domain.IsValidResourceID(*smokeTests.DefaultProfile) {
|
||||
validation.Error(smokePath.Key("default_profile"), "default profile reference is malformed")
|
||||
}
|
||||
}
|
||||
if authentication := smokeTests.Authentication; authentication != nil {
|
||||
kindPath := smokePath.Key("authentication").Key("kind")
|
||||
if authentication.Kind == nil {
|
||||
validation.Error(kindPath, "authentication kind is required")
|
||||
} else if *authentication.Kind != "none" && *authentication.Kind != "openrouter_key" {
|
||||
validation.Error(kindPath, "authentication kind must be none or openrouter_key")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, id := range sortedKeys(resources.profiles) {
|
||||
profile := resources.profiles[id]
|
||||
base := resources.bases.profiles.Key(id)
|
||||
if !domain.IsValidLocalResourceKey(id) {
|
||||
validation.Error(base, "profile key must be a nonblank local key without a slash")
|
||||
}
|
||||
if profile.Backend != nil && *profile.Backend != "" {
|
||||
if !domain.IsValidResourceID(*profile.Backend) {
|
||||
validation.Error(base.Key("backend"), "backend reference is malformed")
|
||||
} else if _, exists := backendIDs[*profile.Backend]; !exists {
|
||||
validation.Error(base.Key("backend"), "backend is not defined")
|
||||
}
|
||||
}
|
||||
if profile.Model != nil && *profile.Model != "" && strings.TrimSpace(*profile.Model) == "" {
|
||||
validation.Error(base.Key("model"), "model must not be blank")
|
||||
}
|
||||
if profile.Temperature != nil {
|
||||
if err := domain.ValidateExecutionTargetSettings(domain.ExecutionTarget{Temperature: *profile.Temperature}); err != nil {
|
||||
validation.Error(base.Key("temperature"), err.Error())
|
||||
}
|
||||
}
|
||||
if profile.TopP != nil {
|
||||
if err := domain.ValidateExecutionTargetSettings(domain.ExecutionTarget{TopP: *profile.TopP}); err != nil {
|
||||
validation.Error(base.Key("top_p"), err.Error())
|
||||
}
|
||||
}
|
||||
if profile.MaxTokens != nil {
|
||||
if err := domain.ValidateExecutionTargetSettings(domain.ExecutionTarget{MaxTokens: *profile.MaxTokens}); err != nil {
|
||||
validation.Error(base.Key("max_tokens"), err.Error())
|
||||
}
|
||||
}
|
||||
if profile.TimeoutSeconds != nil {
|
||||
if err := domain.ValidateExecutionTargetSettings(domain.ExecutionTarget{TimeoutSeconds: *profile.TimeoutSeconds}); err != nil {
|
||||
validation.Error(base.Key("timeout_seconds"), err.Error())
|
||||
}
|
||||
}
|
||||
validateExtraParams(profile.ExtraParams, base.Key("extra_params"), false, validation)
|
||||
}
|
||||
for _, id := range sortedKeys(resources.prompts) {
|
||||
prompt := resources.prompts[id]
|
||||
base := resources.bases.prompts.Key(id)
|
||||
if !domain.IsValidLocalResourceKey(id) {
|
||||
validation.Error(base, "prompt key must be a nonblank local key without a slash")
|
||||
}
|
||||
if prompt.DefaultProfile != nil && *prompt.DefaultProfile != "" && !domain.IsValidResourceID(*prompt.DefaultProfile) {
|
||||
validation.Error(base.Key("default_profile"), "default profile reference is malformed")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func validateExtraParams(values map[string]any, path configkit.Path, backend bool, validation *configkit.Validation) {
|
||||
if values == nil {
|
||||
return
|
||||
}
|
||||
if _, err := jsonvalue.CopyMap(values); err != nil {
|
||||
validation.Error(path, "extra parameters must be a JSON-compatible map with nonempty keys")
|
||||
return
|
||||
}
|
||||
if backend {
|
||||
for key := range values {
|
||||
if llm.IsReservedOpenAIChatRequestField(key) {
|
||||
validation.Error(path.Key(key), "extra parameter conflicts with a request field")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
474
internal/configuration/load_test.go
Normal file
474
internal/configuration/load_test.go
Normal file
@@ -0,0 +1,474 @@
|
||||
package configuration
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
||||
)
|
||||
|
||||
func mapping(label string, values map[string]any) configkit.Input {
|
||||
return configkit.Input{Source: configkit.Mapping(label, values)}
|
||||
}
|
||||
|
||||
func TestLoadChecksOptionalConfigurationVersion(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
version any
|
||||
include bool
|
||||
valid bool
|
||||
}{
|
||||
{name: "string one", version: "1", include: true, valid: true},
|
||||
{name: "integer one", version: int64(1), include: true, valid: true},
|
||||
{name: "omitted", valid: true},
|
||||
{name: "explicit null", include: true},
|
||||
{name: "unsupported string", version: "2", include: true},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
values := map[string]any{}
|
||||
if test.include {
|
||||
values["version"] = test.version
|
||||
}
|
||||
loaded, err := load(mapping("main", values), nil)
|
||||
if test.valid {
|
||||
if err != nil {
|
||||
t.Fatalf("load accepted version: %v", err)
|
||||
}
|
||||
if test.include && (loaded.result.Config.Version == nil || *loaded.result.Config.Version != "1") {
|
||||
t.Fatalf("typed version = %v, want 1", loaded.result.Config.Version)
|
||||
}
|
||||
if !test.include && loaded.result.Config.Version != nil {
|
||||
t.Fatalf("omitted version = %q, want nil", *loaded.result.Config.Version)
|
||||
}
|
||||
return
|
||||
}
|
||||
var loadErr *configkit.LoadError
|
||||
if err == nil || !errors.As(err, &loadErr) {
|
||||
t.Fatalf("load error = %v, want ConfigKit load error", err)
|
||||
}
|
||||
if strings.Contains(err.Error(), "2") {
|
||||
t.Fatalf("load error disclosed the unsupported value: %v", err)
|
||||
}
|
||||
found := false
|
||||
for _, diagnostic := range loadErr.Diagnostics() {
|
||||
if diagnostic.Path.String() == "/version" && diagnostic.Message == "unsupported configuration specification version" {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatalf("load diagnostics = %+v, want safe version diagnostic", loadErr.Diagnostics())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadConsumesAcceptedUnderlayVersionAndValidatesHeaderlessPayload(t *testing.T) {
|
||||
underlay := mapping("declared underlay", map[string]any{
|
||||
"version": int64(1),
|
||||
})
|
||||
loaded, err := load(mapping("headerless main", map[string]any{}), []configkit.Underlay{{Inputs: []configkit.Input{underlay}}})
|
||||
if err != nil {
|
||||
t.Fatalf("load accepted non-main declaration: %v", err)
|
||||
}
|
||||
if loaded.result.Config.Version != nil {
|
||||
t.Fatalf("consumed non-main declaration remained in effective payload: %q", *loaded.result.Config.Version)
|
||||
}
|
||||
|
||||
invalidFragment := mapping("headerless prompt fragment", map[string]any{
|
||||
"prompts": map[string]any{"incomplete": map[string]any{"description": "still validated"}},
|
||||
})
|
||||
if _, err := load(configkit.Input{Source: configkit.NoSource()}, []configkit.Underlay{{Inputs: []configkit.Input{invalidFragment}}}); err == nil || !strings.Contains(err.Error(), "at least one message is required") {
|
||||
t.Fatalf("headerless fragment validation error = %v, want final prompt validation", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadComposesLiteralKeysAndEffectivePresence(t *testing.T) {
|
||||
base := mapping("defaults", map[string]any{
|
||||
"version": int64(1),
|
||||
"backends": map[string]any{
|
||||
"local.v1": map[string]any{"endpoint": "http://localhost:8080/v1", "concurrency_limit": 2},
|
||||
},
|
||||
"collections": map[string]any{"base": map[string]any{
|
||||
"release_version": "",
|
||||
"backends": map[string]any{
|
||||
"local.v1": map[string]any{"endpoint": "http://localhost:8080/v1", "concurrency_limit": 3},
|
||||
},
|
||||
"profiles": map[string]any{
|
||||
"one": map[string]any{"backend": "local.v1", "temperature": 0.8, "top_p": 0.9, "api_key_required": true,
|
||||
"extra_params": map[string]any{"provider": map[string]any{"first": 1, "second": 2}}},
|
||||
"child.v1": map[string]any{"extends": "one"},
|
||||
"namespaced": map[string]any{"backend": "base/local.v1", "model": "collection-model"},
|
||||
},
|
||||
}, "invoice": map[string]any{
|
||||
"release_version": "r7",
|
||||
"prompts": map[string]any{
|
||||
"summary": map[string]any{
|
||||
"version": "v1.0",
|
||||
"messages": []any{map[string]any{"role": "user", "content": "Summarize."}},
|
||||
"output": map[string]any{"format": "text", "validation_mode": "basic"},
|
||||
},
|
||||
"version": map[string]any{
|
||||
"messages": []any{map[string]any{"role": "user", "content": "Nested ID."}},
|
||||
"output": map[string]any{"format": "text", "validation_mode": "basic"},
|
||||
},
|
||||
},
|
||||
}},
|
||||
})
|
||||
overlay := mapping("operator", map[string]any{
|
||||
"backends": map[string]any{"local.v1": map[string]any{"queue_capacity": 0}},
|
||||
"collections": map[string]any{"base": map[string]any{"profiles": map[string]any{"child.v1": map[string]any{
|
||||
"temperature": 0, "top_p": nil, "api_key_required": false,
|
||||
"extra_params": map[string]any{"provider": map[string]any{"second": 3}},
|
||||
}}}},
|
||||
})
|
||||
loaded, err := load(mapping("main", map[string]any{"version": "1"}), []configkit.Underlay{
|
||||
{Inputs: []configkit.Input{base}}, {Inputs: []configkit.Input{overlay}},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
profile := loaded.result.Config.Collections["base"].Profiles["child.v1"]
|
||||
if profile.Temperature == nil || *profile.Temperature != 0 || profile.TopP != nil || profile.APIKeyRequired == nil || *profile.APIKeyRequired {
|
||||
t.Fatalf("effective values were not retained: %+v", profile)
|
||||
}
|
||||
if loaded.profiles["base/child.v1"]["temperature"] != fieldValue || loaded.profiles["base/child.v1"]["top_p"] != fieldNull || loaded.profiles["base/child.v1"]["api_key_required"] != fieldValue {
|
||||
t.Fatalf("effective presence was lost: %+v", loaded.profiles["base/child.v1"])
|
||||
}
|
||||
provider := profile.ExtraParams["provider"].(map[string]any)
|
||||
if provider["first"] != int64(1) || provider["second"] != int64(3) {
|
||||
t.Fatalf("recursive overlay failed: %+v", provider)
|
||||
}
|
||||
backend := loaded.result.Config.Backends["local.v1"]
|
||||
if backend.QueueCapacity == nil || *backend.QueueCapacity != 0 || loaded.backends["local.v1"]["queue_capacity"] != fieldValue {
|
||||
t.Fatalf("explicit zero queue capacity was lost: %+v", backend)
|
||||
}
|
||||
if _, err := loaded.registry.GetBackend("base/local.v1"); err != nil {
|
||||
t.Fatalf("collection backend was not indexed by its full ID: %v", err)
|
||||
}
|
||||
collectionProfile, exists := loaded.indexedProfiles["base/namespaced"]
|
||||
if !exists || collectionProfile.ID != "base/namespaced" || collectionProfile.ResolvedTarget == nil || collectionProfile.ResolvedTarget.BackendID != "base/local.v1" {
|
||||
t.Fatalf("collection profile snapshot = %+v, exists %t", collectionProfile, exists)
|
||||
}
|
||||
if loaded.result.Config.Collections["invoice"].Prompts["summary"].Messages[0].Content == nil || loaded.result.Config.Collections["invoice"].Prompts["summary"].Version == nil || *loaded.result.Config.Collections["invoice"].Prompts["summary"].Version != "v1.0" {
|
||||
t.Fatal("literal prompt ID and version were not decoded")
|
||||
}
|
||||
if loaded.result.Config.Version == nil || *loaded.result.Config.Version != "1" ||
|
||||
loaded.result.Config.Collections["base"].ReleaseVersion == nil || *loaded.result.Config.Collections["base"].ReleaseVersion != "" ||
|
||||
loaded.result.Config.Collections["invoice"].ReleaseVersion == nil || *loaded.result.Config.Collections["invoice"].ReleaseVersion != "r7" {
|
||||
t.Fatalf("root or collection version metadata was not retained: %+v", loaded.result.Config)
|
||||
}
|
||||
if _, _, err := loaded.prompt("invoice/version"); err != nil {
|
||||
t.Fatalf("resource named version was not indexed under its qualified identity: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadRejectsInvalidCollectionAndResourceKeys(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
values map[string]any
|
||||
message string
|
||||
}{
|
||||
{
|
||||
name: "namespace with separator",
|
||||
values: map[string]any{"collections": map[string]any{"bad/name": map[string]any{}}},
|
||||
message: "collection namespace must be nonblank and contain no slash",
|
||||
},
|
||||
{
|
||||
name: "root resource with separator",
|
||||
values: map[string]any{"backends": map[string]any{
|
||||
"bad/name": map[string]any{"endpoint": "https://example.test/v1"},
|
||||
}},
|
||||
message: "backend key must be a nonblank local key without a slash",
|
||||
},
|
||||
{
|
||||
name: "collection resource with separator",
|
||||
values: map[string]any{"collections": map[string]any{"good": map[string]any{
|
||||
"prompts": map[string]any{"bad/name": map[string]any{
|
||||
"messages": []any{map[string]any{"role": "user", "content": "Hello."}},
|
||||
"output": map[string]any{"format": "text", "validation_mode": "basic"},
|
||||
}},
|
||||
}}},
|
||||
message: "prompt key must be a nonblank local key without a slash",
|
||||
},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
_, err := load(mapping("invalid names", test.values), nil)
|
||||
var loadErr *configkit.LoadError
|
||||
if err == nil || !errors.As(err, &loadErr) {
|
||||
t.Fatalf("load error = %v, want ConfigKit validation error", err)
|
||||
}
|
||||
for _, diagnostic := range loadErr.Diagnostics() {
|
||||
if diagnostic.Message == test.message {
|
||||
return
|
||||
}
|
||||
}
|
||||
t.Fatalf("load diagnostics = %+v, want message %q", loadErr.Diagnostics(), test.message)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadRecordsBackendNormalizationAndNullDefault(t *testing.T) {
|
||||
loaded, err := load(mapping("backend", map[string]any{"backends": map[string]any{
|
||||
"local": map[string]any{"endpoint": " HTTP://localhost:8080/v1 ", "api_key_env": " TOKEN_NAME ",
|
||||
"concurrency_limit": 2, "queue_capacity": nil},
|
||||
}}), nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
backend := loaded.result.Config.Backends["local"]
|
||||
if backend.Endpoint == nil || *backend.Endpoint != "http://localhost:8080/v1" ||
|
||||
backend.APIKeyEnv == nil || *backend.APIKeyEnv != "TOKEN_NAME" ||
|
||||
backend.QueueCapacity == nil || *backend.QueueCapacity != 1024 {
|
||||
t.Fatalf("backend normalization/default was not applied: %+v", backend)
|
||||
}
|
||||
if loaded.backends["local"]["queue_capacity"] != fieldNull {
|
||||
t.Fatalf("explicit null was not retained: %+v", loaded.backends["local"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadBackendSmokeTestsOmissionAndNull(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
smokeTests any
|
||||
include bool
|
||||
inherited bool
|
||||
}{
|
||||
{name: "omitted"},
|
||||
{name: "null", smokeTests: nil, include: true, inherited: true},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
backend := map[string]any{"endpoint": "https://backend.example/v1"}
|
||||
if test.include {
|
||||
backend["smoke_tests"] = test.smokeTests
|
||||
}
|
||||
main := mapping("backend", map[string]any{
|
||||
"backends": map[string]any{"local": backend},
|
||||
})
|
||||
var underlays []configkit.Underlay
|
||||
if test.inherited {
|
||||
underlays = []configkit.Underlay{{Inputs: []configkit.Input{mapping("defaults", map[string]any{
|
||||
"backends": map[string]any{"local": map[string]any{
|
||||
"endpoint": "https://backend.example/v1",
|
||||
"smoke_tests": map[string]any{"default_profile": "inherited-profile"},
|
||||
}},
|
||||
})}}}
|
||||
}
|
||||
loaded, err := load(main, underlays)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := loaded.result.Config.Backends["local"].SmokeTests; got != nil {
|
||||
t.Fatalf("smoke_tests = %+v, want nil", got)
|
||||
}
|
||||
selected, err := loaded.registry.GetBackend("local")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if selected.SmokeTests != nil {
|
||||
t.Fatalf("indexed smoke_tests = %+v, want nil", selected.SmokeTests)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadMergesBackendSmokeTestsAndRetainsFallbacks(t *testing.T) {
|
||||
base := mapping("defaults", map[string]any{
|
||||
"backends": map[string]any{"local": map[string]any{
|
||||
"endpoint": "https://backend.example/v1",
|
||||
"smoke_tests": map[string]any{
|
||||
"default_profile": "catalog-profile",
|
||||
"reachability": map[string]any{"endpoint": " https://probe.example/ready?check=1 "},
|
||||
"authentication": map[string]any{
|
||||
"kind": " OPENROUTER_KEY ",
|
||||
"endpoint": "https://probe.example/key?scope=current",
|
||||
},
|
||||
},
|
||||
}},
|
||||
})
|
||||
overlay := mapping("operator", map[string]any{
|
||||
"backends": map[string]any{"local": map[string]any{
|
||||
"smoke_tests": map[string]any{
|
||||
"default_profile": "not-yet-listed",
|
||||
"authentication": map[string]any{"kind": " NONE ", "endpoint": nil},
|
||||
},
|
||||
}},
|
||||
})
|
||||
loaded, err := load(configkit.Input{Source: configkit.NoSource()}, []configkit.Underlay{
|
||||
{Inputs: []configkit.Input{base}},
|
||||
{Inputs: []configkit.Input{overlay}},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("load effective diagnostic metadata: %v", err)
|
||||
}
|
||||
selected, err := loaded.registry.GetBackend("local")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
metadata := selected.SmokeTests
|
||||
if selected.Endpoint != "https://backend.example/v1" {
|
||||
t.Fatalf("effective backend endpoint = %q", selected.Endpoint)
|
||||
}
|
||||
if metadata == nil || metadata.DefaultProfile == nil || *metadata.DefaultProfile != "not-yet-listed" {
|
||||
t.Fatalf("default profile overlay was not retained: %+v", metadata)
|
||||
}
|
||||
if metadata.Reachability == nil || metadata.Reachability.Endpoint == nil ||
|
||||
*metadata.Reachability.Endpoint != "https://probe.example/ready?check=1" {
|
||||
t.Fatalf("full reachability destination was not normalized: %+v", metadata.Reachability)
|
||||
}
|
||||
if metadata.Authentication == nil || metadata.Authentication.Kind == nil || *metadata.Authentication.Kind != "none" ||
|
||||
metadata.Authentication.Endpoint != nil {
|
||||
t.Fatalf("authentication overlay or null fallback was not retained: %+v", metadata.Authentication)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadRejectsInvalidFinalBackendSmokeTests(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
data map[string]any
|
||||
path string
|
||||
}{
|
||||
{
|
||||
name: "blank default profile",
|
||||
data: map[string]any{"default_profile": " \t "},
|
||||
path: "/backends/local/smoke_tests/default_profile",
|
||||
},
|
||||
{
|
||||
name: "authentication kind required",
|
||||
data: map[string]any{"authentication": map[string]any{}},
|
||||
path: "/backends/local/smoke_tests/authentication/kind",
|
||||
},
|
||||
{
|
||||
name: "unknown authentication kind",
|
||||
data: map[string]any{"authentication": map[string]any{"kind": "bearer"}},
|
||||
path: "/backends/local/smoke_tests/authentication/kind",
|
||||
},
|
||||
{
|
||||
name: "non-http reachability URL",
|
||||
data: map[string]any{"reachability": map[string]any{"endpoint": "ftp://probe.example/ready"}},
|
||||
path: "/backends/local/smoke_tests/reachability/endpoint",
|
||||
},
|
||||
{
|
||||
name: "blank authentication URL",
|
||||
data: map[string]any{"authentication": map[string]any{"kind": "none", "endpoint": " "}},
|
||||
path: "/backends/local/smoke_tests/authentication/endpoint",
|
||||
},
|
||||
{
|
||||
name: "relative authentication URL",
|
||||
data: map[string]any{"authentication": map[string]any{"kind": "none", "endpoint": "/key"}},
|
||||
path: "/backends/local/smoke_tests/authentication/endpoint",
|
||||
},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
_, err := load(mapping("backend", map[string]any{
|
||||
"backends": map[string]any{"local": map[string]any{
|
||||
"endpoint": "https://backend.example/v1",
|
||||
"smoke_tests": test.data,
|
||||
}},
|
||||
}), nil)
|
||||
var loadError *configkit.LoadError
|
||||
if !errors.As(err, &loadError) {
|
||||
t.Fatalf("expected final-value validation error, got %v", err)
|
||||
}
|
||||
for _, diagnostic := range loadError.Diagnostics() {
|
||||
if diagnostic.Path.String() == test.path {
|
||||
return
|
||||
}
|
||||
}
|
||||
t.Fatalf("expected diagnostic at %s, got %+v", test.path, loadError.Diagnostics())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadDefaultsAbsentQueueCapacity(t *testing.T) {
|
||||
loaded, err := load(mapping("backend", map[string]any{"backends": map[string]any{
|
||||
"local": map[string]any{"endpoint": "http://localhost:8080/v1", "concurrency_limit": 1},
|
||||
}}), nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
backend := loaded.result.Config.Backends["local"]
|
||||
if backend.QueueCapacity == nil || *backend.QueueCapacity != 1024 || loaded.backends["local"]["queue_capacity"] != fieldAbsent {
|
||||
t.Fatalf("absent queue capacity did not receive its default: %+v", backend)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadRejectsStrictGroupConflict(t *testing.T) {
|
||||
_, err := load(configkit.Input{Source: configkit.NoSource()}, []configkit.Underlay{{Inputs: []configkit.Input{
|
||||
mapping("first", map[string]any{"profiles": map[string]any{"shared": map[string]any{"model": "one"}}}),
|
||||
mapping("second", map[string]any{"profiles": map[string]any{"shared": map[string]any{"model": "one"}}}),
|
||||
}}})
|
||||
var loadError *configkit.LoadError
|
||||
if !errors.As(err, &loadError) {
|
||||
t.Fatalf("expected ConfigKit load error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadValidatesOnlyFinalDomainValues(t *testing.T) {
|
||||
underlay := mapping("defaults", map[string]any{
|
||||
"backends": map[string]any{"local": map[string]any{"endpoint": "not a URL"}},
|
||||
"profiles": map[string]any{"partial": map[string]any{"backend": "local"}},
|
||||
})
|
||||
main := mapping("operator", map[string]any{"backends": map[string]any{"local": map[string]any{"endpoint": "http://localhost:8080/v1"}}})
|
||||
loaded, err := load(main, []configkit.Underlay{{Inputs: []configkit.Input{underlay}}})
|
||||
if err != nil {
|
||||
t.Fatalf("replaced lower-layer domain value should not fail: %v", err)
|
||||
}
|
||||
if loaded.result.Config.Profiles["partial"].Model != nil {
|
||||
t.Fatal("unused partial profile unexpectedly gained a model")
|
||||
}
|
||||
|
||||
_, err = load(configkit.Input{Source: configkit.NoSource()}, []configkit.Underlay{{Inputs: []configkit.Input{
|
||||
mapping("invalid", map[string]any{"backends": map[string]any{"unused": map[string]any{"endpoint": "not a URL"}}}),
|
||||
}}})
|
||||
var loadError *configkit.LoadError
|
||||
if !errors.As(err, &loadError) {
|
||||
t.Fatalf("expected path-aware effective backend failure, got %v", err)
|
||||
}
|
||||
if diagnostics := loadError.Diagnostics(); len(diagnostics) != 1 || diagnostics[0].Path.String() != "/backends/unused/endpoint" {
|
||||
t.Fatalf("expected endpoint diagnostic, got %+v", diagnostics)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadValidatesOnlyFinalBackendSmokeTestValues(t *testing.T) {
|
||||
underlay := mapping("defaults", map[string]any{
|
||||
"backends": map[string]any{"local": map[string]any{
|
||||
"endpoint": "https://backend.example/v1",
|
||||
"smoke_tests": map[string]any{
|
||||
"default_profile": " ",
|
||||
"reachability": map[string]any{"endpoint": "ftp://invalid.example/"},
|
||||
"authentication": map[string]any{"kind": "unsupported"},
|
||||
},
|
||||
}},
|
||||
})
|
||||
main := mapping("operator", map[string]any{
|
||||
"backends": map[string]any{"local": map[string]any{
|
||||
"smoke_tests": map[string]any{
|
||||
"default_profile": "dangling-profile",
|
||||
"reachability": map[string]any{"endpoint": nil},
|
||||
"authentication": map[string]any{"kind": "none"},
|
||||
},
|
||||
}},
|
||||
})
|
||||
loaded, err := load(main, []configkit.Underlay{{Inputs: []configkit.Input{underlay}}})
|
||||
if err != nil {
|
||||
t.Fatalf("replaced lower metadata should not fail: %v", err)
|
||||
}
|
||||
selected, err := loaded.registry.GetBackend("local")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if selected.SmokeTests == nil || selected.SmokeTests.DefaultProfile == nil ||
|
||||
*selected.SmokeTests.DefaultProfile != "dangling-profile" {
|
||||
t.Fatalf("final dangling default profile was not retained: %+v", selected.SmokeTests)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadKeepsSyntaxErrorsFromLowerTiers(t *testing.T) {
|
||||
_, err := load(mapping("operator", map[string]any{"backends": map[string]any{"local": map[string]any{"endpoint": "http://localhost:8080"}}}),
|
||||
[]configkit.Underlay{{Inputs: []configkit.Input{{Source: configkit.Bytes("broken", []byte("backends: ["))}}}})
|
||||
var loadError *configkit.LoadError
|
||||
if !errors.As(err, &loadError) {
|
||||
t.Fatalf("expected lower-tier parsing error, got %v", err)
|
||||
}
|
||||
}
|
||||
413
internal/configuration/prompts.go
Normal file
413
internal/configuration/prompts.go
Normal file
@@ -0,0 +1,413 @@
|
||||
package configuration
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
"text/template"
|
||||
|
||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/filecatalog"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/prompt"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/promptdef"
|
||||
)
|
||||
|
||||
var (
|
||||
errPromptNotFound = errors.New("prompt definition not found")
|
||||
)
|
||||
|
||||
type promptEntry struct {
|
||||
definition domain.PromptDefinition
|
||||
inspection domain.PromptInspection
|
||||
requirementInputs []prompt.InputMetadata
|
||||
hash string
|
||||
session *template.Template
|
||||
messages []*template.Template
|
||||
}
|
||||
|
||||
func resolvePrompts(prompts map[string]promptConfig, basePath configkit.Path, resolution *configkit.Resolution) {
|
||||
for _, id := range sortedKeys(prompts) {
|
||||
definition := prompts[id]
|
||||
base := basePath.Key(id)
|
||||
for _, field := range []struct {
|
||||
name string
|
||||
value **string
|
||||
}{
|
||||
{"description", &definition.Description},
|
||||
{"session_id", &definition.SessionID},
|
||||
} {
|
||||
if *field.value != nil {
|
||||
normalizeString(field.value, base.Key(field.name), resolution, strings.TrimSpace(**field.value))
|
||||
}
|
||||
}
|
||||
inputsChanged := false
|
||||
for index := range definition.Inputs {
|
||||
input := &definition.Inputs[index]
|
||||
inputPath := base.Key("inputs").Index(index)
|
||||
normalizedName := strings.TrimSpace(input.Name)
|
||||
if normalizedName != input.Name {
|
||||
configkit.Normalize(resolution, inputPath.Key("name"), &input.Name, normalizedName, "normalize input metadata")
|
||||
inputsChanged = true
|
||||
}
|
||||
for _, field := range []struct {
|
||||
name string
|
||||
value **string
|
||||
}{
|
||||
{"content_type", &input.ContentType},
|
||||
{"description", &input.Description},
|
||||
} {
|
||||
if *field.value != nil {
|
||||
normalized := strings.TrimSpace(**field.value)
|
||||
if normalized != **field.value {
|
||||
configkit.Normalize(resolution, inputPath.Key(field.name), field.value, &normalized, "normalize input metadata")
|
||||
inputsChanged = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if inputsChanged {
|
||||
resolution.Record(base.Key("inputs"), configkit.ChangeNormalization, "normalize input metadata")
|
||||
}
|
||||
// Normalize message metadata on execution-owned copies below. ConfigKit
|
||||
// records sequence mutations as whole-value replacements, which would
|
||||
// discard the effective source contexts of unchanged content_file fields.
|
||||
if definition.Output != nil && definition.Output.SchemaPath != nil {
|
||||
normalizeString(&definition.Output.SchemaPath, base.Key("output").Key("schema_path"), resolution, strings.TrimSpace(*definition.Output.SchemaPath))
|
||||
}
|
||||
prompts[id] = definition
|
||||
}
|
||||
}
|
||||
|
||||
func (state *loaded) materializePrompts() error {
|
||||
state.prompts = make(map[string]*promptEntry)
|
||||
var metadata *configkit.Metadata
|
||||
lookupOrigin := func(path configkit.Path) (configkit.FieldInfo, bool) {
|
||||
if metadata == nil {
|
||||
captured := state.result.Metadata()
|
||||
metadata = &captured
|
||||
}
|
||||
return metadata.Lookup(path)
|
||||
}
|
||||
for _, resources := range resourceSets(&state.result.Config) {
|
||||
for _, id := range sortedKeys(resources.prompts) {
|
||||
promptBase := resources.bases.prompts.Key(id)
|
||||
if strings.TrimSpace(id) == "" {
|
||||
return fmt.Errorf("%s: prompt ID must not be blank", promptBase)
|
||||
}
|
||||
fullID := indexedResourceID(resources.namespace, id)
|
||||
raw := resources.prompts[id]
|
||||
entry, err := state.materializePrompt(fullID, raw.Version, raw, promptBase, lookupOrigin)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
state.prompts[fullID] = entry
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (state *loaded) materializePrompt(id string, version *string, raw promptConfig, base configkit.Path, lookupOrigin func(configkit.Path) (configkit.FieldInfo, bool)) (*promptEntry, error) {
|
||||
if len(raw.Messages) == 0 {
|
||||
return nil, fmt.Errorf("%s: at least one message is required", base.Key("messages"))
|
||||
}
|
||||
definition := domain.PromptDefinition{ID: id, Version: stringPointerValue(version)}
|
||||
if raw.DefaultProfile != nil {
|
||||
if strings.TrimSpace(*raw.DefaultProfile) == "" {
|
||||
return nil, fmt.Errorf("%s: default profile must not be blank", base.Key("default_profile"))
|
||||
}
|
||||
definition.DefaultProfile = *raw.DefaultProfile
|
||||
}
|
||||
if raw.Description != nil {
|
||||
definition.Description = *raw.Description
|
||||
}
|
||||
if raw.SessionID != nil {
|
||||
definition.SessionID = *raw.SessionID
|
||||
}
|
||||
seenInputs := make(map[string]bool, len(raw.Inputs))
|
||||
definition.Inputs = make([]domain.PromptInput, 0, len(raw.Inputs))
|
||||
entry := &promptEntry{
|
||||
messages: make([]*template.Template, 0, len(raw.Messages)),
|
||||
requirementInputs: make([]prompt.InputMetadata, 0, len(raw.Inputs)),
|
||||
}
|
||||
for index, input := range raw.Inputs {
|
||||
path := base.Key("inputs").Index(index)
|
||||
if input.Name == "" {
|
||||
return nil, fmt.Errorf("%s: input name is required", path.Key("name"))
|
||||
}
|
||||
if seenInputs[input.Name] {
|
||||
return nil, fmt.Errorf("%s: input name is duplicated", path.Key("name"))
|
||||
}
|
||||
seenInputs[input.Name] = true
|
||||
contentType := ""
|
||||
if input.ContentType != nil {
|
||||
contentType = *input.ContentType
|
||||
}
|
||||
description := ""
|
||||
if input.Description != nil {
|
||||
description = *input.Description
|
||||
}
|
||||
definition.Inputs = append(definition.Inputs, domain.PromptInput{
|
||||
Name: input.Name, Required: input.Required, ContentType: contentType, Description: description,
|
||||
})
|
||||
presence := state.promptMetadata[id][index]
|
||||
metadata := prompt.InputMetadata{Name: input.Name, Required: input.Required}
|
||||
if presence.contentType == fieldValue {
|
||||
metadata.ContentType = stringPointer(contentType)
|
||||
}
|
||||
if presence.description == fieldValue {
|
||||
metadata.Description = stringPointer(description)
|
||||
}
|
||||
entry.requirementInputs = append(entry.requirementInputs, metadata)
|
||||
}
|
||||
definition.Templates = make([]domain.PromptMessageTemplate, 0, len(raw.Messages))
|
||||
for index, message := range raw.Messages {
|
||||
path := base.Key("messages").Index(index)
|
||||
if message.Role == nil {
|
||||
return nil, fmt.Errorf("%s: role is required", path.Key("role"))
|
||||
}
|
||||
role, err := domain.NormalizeMessageRole(*message.Role)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s: %w", path.Key("role"), err)
|
||||
}
|
||||
if (message.Content != nil) == (message.ContentFile != nil) {
|
||||
return nil, fmt.Errorf("%s: exactly one of content or content_file is required", path)
|
||||
}
|
||||
content := ""
|
||||
if message.Content != nil {
|
||||
content = *message.Content
|
||||
} else {
|
||||
var err error
|
||||
content, err = state.readContent(path.Key("content_file"), *message.ContentFile, lookupOrigin)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
var control *domain.CacheControl
|
||||
if message.CacheControl != nil {
|
||||
if message.CacheControl.Type == nil {
|
||||
return nil, fmt.Errorf("%s: type is required", path.Key("cache_control").Key("type"))
|
||||
}
|
||||
control = &domain.CacheControl{Type: domain.CacheControlType(*message.CacheControl.Type)}
|
||||
if message.CacheControl.TTL != nil {
|
||||
control.TTL = *message.CacheControl.TTL
|
||||
}
|
||||
var err error
|
||||
control, err = domain.NormalizeCacheControl(control)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s: %w", path.Key("cache_control"), err)
|
||||
}
|
||||
}
|
||||
compiled, err := prompt.CompileTemplate(fmt.Sprintf("msg_%d", index), content)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s: invalid template syntax: %w", path, err)
|
||||
}
|
||||
entry.messages = append(entry.messages, compiled)
|
||||
definition.Templates = append(definition.Templates, domain.PromptMessageTemplate{
|
||||
Role: role, Content: content, CacheControl: control,
|
||||
})
|
||||
}
|
||||
if raw.Output == nil {
|
||||
return nil, fmt.Errorf("%s: output is required", base.Key("output"))
|
||||
}
|
||||
if raw.Output.Format == nil || raw.Output.ValidationMode == nil {
|
||||
return nil, fmt.Errorf("%s: format and validation_mode are required", base.Key("output"))
|
||||
}
|
||||
contract := domain.OutputContract{
|
||||
Format: domain.OutputFormat(*raw.Output.Format),
|
||||
ValidationMode: domain.ValidationMode(*raw.Output.ValidationMode),
|
||||
RepairAttempts: raw.Output.RepairAttempts,
|
||||
}
|
||||
if raw.Output.SchemaPath != nil {
|
||||
contract.SchemaPath = *raw.Output.SchemaPath
|
||||
}
|
||||
if err := domain.ValidateOutputContract(contract); err != nil {
|
||||
return nil, fmt.Errorf("%s: invalid output contract: %w", base.Key("output"), err)
|
||||
}
|
||||
definition.OutputFormat = contract.Format
|
||||
definition.Validation = contract
|
||||
if raw.SessionID != nil {
|
||||
var err error
|
||||
entry.session, err = prompt.CompileTemplate("session_id", definition.SessionID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s: invalid template syntax: %w", base.Key("session_id"), err)
|
||||
}
|
||||
}
|
||||
hash, err := promptdef.HashDefinition(&definition)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s: hash prompt definition: %w", base, err)
|
||||
}
|
||||
entry.definition = definition
|
||||
entry.hash = hash
|
||||
entry.inspection = domain.PromptInspection{
|
||||
PromptID: id, PromptVersion: stringPointerValue(version), PromptHash: hash,
|
||||
DefaultProfileID: definition.DefaultProfile, Inputs: append([]domain.PromptInput(nil), definition.Inputs...),
|
||||
OutputContract: contract,
|
||||
}
|
||||
return entry, nil
|
||||
}
|
||||
|
||||
func stringPointer(value string) *string {
|
||||
copy := value
|
||||
return ©
|
||||
}
|
||||
|
||||
func stringPointerValue(value *string) *string {
|
||||
if value == nil {
|
||||
return nil
|
||||
}
|
||||
return stringPointer(*value)
|
||||
}
|
||||
|
||||
func (state *loaded) readContent(fieldPath configkit.Path, name string, lookupOrigin func(configkit.Path) (configkit.FieldInfo, bool)) (string, error) {
|
||||
if strings.TrimSpace(name) == "" {
|
||||
return "", fmt.Errorf("%s: content_file path must not be blank", fieldPath)
|
||||
}
|
||||
field, found := lookupOrigin(fieldPath)
|
||||
if !found || len(field.Origin.Locations) != 1 {
|
||||
return "", fmt.Errorf("%s: content_file needs one contributing source with a filesystem context", fieldPath)
|
||||
}
|
||||
sourceID := field.Origin.Locations[0].SourceID
|
||||
context, found := state.result.SourceContext(sourceID)
|
||||
if !found || context.FS == nil {
|
||||
return "", fmt.Errorf("%s: content_file source %s has no filesystem context", fieldPath, sourceID)
|
||||
}
|
||||
resolved, _, err := filecatalog.ResolveFSPath(".", context.BaseDir, name)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("%s: content_file path is outside its source root or invalid: %w", fieldPath, err)
|
||||
}
|
||||
file, err := context.FS.Open(resolved)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("%s: open content_file from source %s: %w", fieldPath, sourceID, err)
|
||||
}
|
||||
content, readErr := io.ReadAll(file)
|
||||
closeErr := file.Close()
|
||||
if readErr != nil {
|
||||
return "", fmt.Errorf("%s: read content_file from source %s: %w", fieldPath, sourceID, readErr)
|
||||
}
|
||||
if closeErr != nil {
|
||||
return "", fmt.Errorf("%s: close content_file from source %s: %w", fieldPath, sourceID, closeErr)
|
||||
}
|
||||
return string(content), nil
|
||||
}
|
||||
|
||||
func (state *loaded) lookupPromptEntry(id string) (*promptEntry, error) {
|
||||
if state == nil || strings.TrimSpace(id) == "" {
|
||||
return nil, errPromptNotFound
|
||||
}
|
||||
entry := state.prompts[id]
|
||||
if entry == nil {
|
||||
return nil, errPromptNotFound
|
||||
}
|
||||
return entry, nil
|
||||
}
|
||||
|
||||
func ownPromptDefinition(definition domain.PromptDefinition) *domain.PromptDefinition {
|
||||
definition.Version = stringPointerValue(definition.Version)
|
||||
definition.Inputs = append([]domain.PromptInput(nil), definition.Inputs...)
|
||||
definition.Templates = append([]domain.PromptMessageTemplate(nil), definition.Templates...)
|
||||
for index := range definition.Templates {
|
||||
if control := definition.Templates[index].CacheControl; control != nil {
|
||||
copy := *control
|
||||
definition.Templates[index].CacheControl = ©
|
||||
}
|
||||
}
|
||||
return &definition
|
||||
}
|
||||
|
||||
func ownPromptInspection(inspection domain.PromptInspection) *domain.PromptInspection {
|
||||
inspection.PromptVersion = stringPointerValue(inspection.PromptVersion)
|
||||
inspection.Inputs = append([]domain.PromptInput(nil), inspection.Inputs...)
|
||||
return &inspection
|
||||
}
|
||||
|
||||
func promptLookupError(err error) error {
|
||||
if errors.Is(err, errPromptNotFound) {
|
||||
return promptdef.ErrPromptDefinitionNotFound
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (state *loaded) prompt(id string) (*domain.PromptDefinition, domain.PromptInspection, error) {
|
||||
entry, err := state.lookupPromptEntry(id)
|
||||
if err != nil {
|
||||
return nil, domain.PromptInspection{}, err
|
||||
}
|
||||
return ownPromptDefinition(entry.definition), *ownPromptInspection(entry.inspection), nil
|
||||
}
|
||||
|
||||
// GetPromptSnapshot returns an independently owned captured definition and its
|
||||
// construction-time hash.
|
||||
func (state *loaded) GetPromptSnapshot(ctx context.Context, id string) (*domain.PromptDefinition, string, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
entry, err := state.lookupPromptEntry(id)
|
||||
if err != nil {
|
||||
return nil, "", promptLookupError(err)
|
||||
}
|
||||
return ownPromptDefinition(entry.definition), entry.hash, nil
|
||||
}
|
||||
|
||||
// GetPromptInspection returns an independently owned inspection of one
|
||||
// captured prompt without copying its execution templates.
|
||||
func (state *loaded) GetPromptInspection(ctx context.Context, id string) (*domain.PromptInspection, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
entry, err := state.lookupPromptEntry(id)
|
||||
if err != nil {
|
||||
return nil, promptLookupError(err)
|
||||
}
|
||||
return ownPromptInspection(entry.inspection), nil
|
||||
}
|
||||
|
||||
// GetPromptDefinition implements the runner's prompt lookup boundary.
|
||||
func (state *loaded) GetPromptDefinition(ctx context.Context, id string) (*domain.PromptDefinition, error) {
|
||||
definition, _, err := state.GetPromptSnapshot(ctx, id)
|
||||
return definition, err
|
||||
}
|
||||
|
||||
// GetCompiledTemplates returns construction-time parses for one prompt ID.
|
||||
func (state *loaded) GetCompiledTemplates(id string) (prompt.CompiledTemplates, bool) {
|
||||
if state == nil {
|
||||
return prompt.CompiledTemplates{}, false
|
||||
}
|
||||
entry := state.prompts[id]
|
||||
if entry == nil {
|
||||
return prompt.CompiledTemplates{}, false
|
||||
}
|
||||
return prompt.CompiledTemplates{Session: entry.session, Messages: entry.messages}, true
|
||||
}
|
||||
|
||||
// GetRequirementSnapshot returns captured templates and input metadata for
|
||||
// one prompt ID. Returned metadata and slices are independently
|
||||
// owned; the compiled templates are shared immutable construction-time parses.
|
||||
func (state *loaded) GetRequirementSnapshot(id string) (prompt.RequirementSnapshot, bool) {
|
||||
if state == nil {
|
||||
return prompt.RequirementSnapshot{}, false
|
||||
}
|
||||
entry := state.prompts[id]
|
||||
if entry == nil {
|
||||
return prompt.RequirementSnapshot{}, false
|
||||
}
|
||||
snapshot := prompt.RequirementSnapshot{
|
||||
Templates: prompt.CompiledTemplates{
|
||||
Session: entry.session,
|
||||
Messages: append([]*template.Template(nil), entry.messages...),
|
||||
},
|
||||
Inputs: make([]prompt.InputMetadata, len(entry.requirementInputs)),
|
||||
}
|
||||
for index, input := range entry.requirementInputs {
|
||||
snapshot.Inputs[index] = input
|
||||
if input.ContentType != nil {
|
||||
snapshot.Inputs[index].ContentType = stringPointer(*input.ContentType)
|
||||
}
|
||||
if input.Description != nil {
|
||||
snapshot.Inputs[index].Description = stringPointer(*input.Description)
|
||||
}
|
||||
}
|
||||
return snapshot, true
|
||||
}
|
||||
501
internal/configuration/prompts_test.go
Normal file
501
internal/configuration/prompts_test.go
Normal file
@@ -0,0 +1,501 @@
|
||||
package configuration
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"testing/fstest"
|
||||
|
||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/promptdef"
|
||||
)
|
||||
|
||||
func TestLoadCapturesInheritedImportedContent(t *testing.T) {
|
||||
files := fstest.MapFS{
|
||||
"app/main.yaml": {Data: []byte("configkit:\n imports:\n - path: ../catalog/base.yaml\n - path: ../other/definition.yaml\ncollections:\n docs:\n prompts:\n child:\n extends: base\n")},
|
||||
"catalog/base.yaml": {Data: []byte("collections:\n docs:\n release_version: r1\n prompts:\n base:\n version: \"v1\"\n messages:\n - role: ' USER '\n cache_control:\n type: ' ephemeral '\n ttl: ' 1h '\n content_file: ./body.md\n output:\n format: text\n validation_mode: basic\n")},
|
||||
"catalog/body.md": {Data: []byte("Captured {{.name}}")},
|
||||
"other/definition.yaml": {Data: []byte("collections:\n other:\n prompts:\n child:\n messages:\n - role: user\n content_file: ./body.md\n output:\n format: text\n validation_mode: none\n")},
|
||||
"other/body.md": {Data: []byte("Other collection {{.name}}")},
|
||||
}
|
||||
namespace := configkit.NewFilesystem("embedded", files)
|
||||
state, err := load(configkit.Input{
|
||||
Source: namespace.File("app/main.yaml"), Imports: true,
|
||||
Filesystem: &configkit.FilesystemContext{FS: &namespace, Root: "."},
|
||||
}, nil)
|
||||
if err != nil {
|
||||
var loadError *configkit.LoadError
|
||||
if errors.As(err, &loadError) {
|
||||
t.Fatalf("%v: %+v", err, loadError.Diagnostics())
|
||||
}
|
||||
t.Fatal(err)
|
||||
}
|
||||
definition, inspection, err := state.prompt("docs/child")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if definition.Templates[0].Content != "Captured {{.name}}" || inspection.PromptHash == "" {
|
||||
t.Fatalf("inherited file content was not captured: %+v, %+v", definition, inspection)
|
||||
}
|
||||
message := definition.Templates[0]
|
||||
if message.Role != "user" || message.CacheControl == nil || message.CacheControl.Type != "ephemeral" || message.CacheControl.TTL != "1h" {
|
||||
t.Fatalf("message metadata was not normalized: %+v", message)
|
||||
}
|
||||
field, ok := state.result.Metadata().Lookup(configkit.Keys("collections", "docs", "prompts", "child", "messages").Index(0).Key("content_file"))
|
||||
if !ok || len(field.Origin.Locations) != 1 {
|
||||
t.Fatalf("content origin was lost during normalization: %+v", field)
|
||||
}
|
||||
var originPath string
|
||||
for _, source := range state.result.Sources() {
|
||||
if source.ID == field.Origin.Locations[0].SourceID {
|
||||
originPath = source.Path
|
||||
}
|
||||
}
|
||||
if originPath != "catalog/base.yaml" {
|
||||
t.Fatalf("content origin = %q, want catalog/base.yaml", originPath)
|
||||
}
|
||||
other, otherInspection, err := state.prompt("other/child")
|
||||
if err != nil || other.Templates[0].Content != "Other collection {{.name}}" {
|
||||
t.Fatalf("other collection did not retain its source context: %+v, %v", other, err)
|
||||
}
|
||||
files["catalog/body.md"].Data = []byte("Changed")
|
||||
files["other/body.md"].Data = []byte("Changed other collection")
|
||||
again, againInspection, err := state.prompt("docs/child")
|
||||
if err != nil || again.Templates[0].Content != "Captured {{.name}}" || againInspection.PromptHash != inspection.PromptHash {
|
||||
t.Fatalf("captured definition changed after source mutation: %+v, %+v, %v", again, againInspection, err)
|
||||
}
|
||||
definition.Templates[0].Content = "caller mutation"
|
||||
third, _, err := state.prompt("docs/child")
|
||||
if err != nil || third.Templates[0].Content != "Captured {{.name}}" {
|
||||
t.Fatalf("caller mutation escaped into index: %+v, %v", third, err)
|
||||
}
|
||||
otherAgain, otherAgainInspection, err := state.prompt("other/child")
|
||||
if err != nil || otherAgain.Templates[0].Content != "Other collection {{.name}}" || otherAgainInspection.PromptHash != otherInspection.PromptHash {
|
||||
t.Fatalf("other captured definition changed after source mutation: %+v, %+v, %v", otherAgain, otherAgainInspection, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadUsesExplicitMappingContextAndClosesContent(t *testing.T) {
|
||||
filesystem := &countingFS{FS: fstest.MapFS{
|
||||
"assets/body.md": {Data: []byte("From mapping")},
|
||||
}}
|
||||
namespace := configkit.NewFilesystem("mapping-context", filesystem)
|
||||
source := configkit.Mapping("application", promptWithFile("body.md"))
|
||||
state, err := load(configkit.Input{
|
||||
Source: source,
|
||||
Filesystem: &configkit.FilesystemContext{FS: &namespace, Root: ".", BaseDir: "assets"},
|
||||
}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
definition, _, err := state.prompt("example")
|
||||
if err != nil || definition.Templates[0].Content != "From mapping" || filesystem.contentOpened != 1 || filesystem.contentClosed != 1 {
|
||||
t.Fatalf("mapping context or handle ownership failed: %+v, opens=%d closes=%d, %v", definition, filesystem.contentOpened, filesystem.contentClosed, err)
|
||||
}
|
||||
_, err = load(mapping("contextless", promptWithFile("body.md")), nil)
|
||||
if err == nil || !strings.Contains(err.Error(), "/prompts/example/messages/0/content_file") {
|
||||
t.Fatalf("expected semantic path for contextless resource, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadCapturesNativeContentFile(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
definitionPath := filepath.Join(root, "definition.yaml")
|
||||
contentPath := filepath.Join(root, "body.md")
|
||||
if err := os.WriteFile(definitionPath, []byte("prompts:\n example:\n version: \"v1\"\n messages:\n - role: user\n content_file: body.md\n output:\n format: text\n validation_mode: basic\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(contentPath, []byte("Native body"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
state, err := load(configkit.Input{Source: configkit.File(definitionPath), Filesystem: &configkit.FilesystemContext{Root: root}}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(contentPath, []byte("Modified"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
definition, _, err := state.prompt("example")
|
||||
if err != nil || definition.Templates[0].Content != "Native body" {
|
||||
t.Fatalf("native content was not captured: %+v, %v", definition, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadRejectsMalformedStaticSyntaxAndCapturesOptionalVersion(t *testing.T) {
|
||||
_, err := load(mapping("bad-template", map[string]any{"prompts": map[string]any{"example": map[string]any{
|
||||
"messages": []any{map[string]any{"role": "user", "content": "{{if}}"}},
|
||||
"output": map[string]any{"format": "text", "validation_mode": "basic"},
|
||||
}}}), nil)
|
||||
if err == nil || !strings.Contains(err.Error(), "invalid template syntax") {
|
||||
t.Fatalf("expected construction-time syntax error, got %v", err)
|
||||
}
|
||||
state, err := load(mapping("version metadata", map[string]any{"prompts": map[string]any{"example": map[string]any{
|
||||
"version": "release label",
|
||||
"messages": []any{map[string]any{"role": "user", "content": "one"}},
|
||||
"output": map[string]any{"format": "text", "validation_mode": "basic"},
|
||||
}}}), nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
definition, inspection, err := state.prompt("example")
|
||||
if err != nil || definition.Version == nil || *definition.Version != "release label" ||
|
||||
inspection.PromptVersion == nil || *inspection.PromptVersion != "release label" {
|
||||
t.Fatalf("optional version metadata was not retained: definition=%+v inspection=%+v err=%v", definition, inspection, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPromptVersionLabelPresenceAndOwnership(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
version any
|
||||
include bool
|
||||
wantPresent bool
|
||||
wantValue string
|
||||
}{
|
||||
{name: "absent"},
|
||||
{name: "null", include: true},
|
||||
{name: "empty", version: "", include: true, wantPresent: true},
|
||||
{name: "exact text", version: " Release/v1 ", include: true, wantPresent: true, wantValue: " Release/v1 "},
|
||||
}
|
||||
|
||||
hashes := make(map[string]string, len(tests))
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
definition := map[string]any{
|
||||
"messages": []any{map[string]any{"role": "user", "content": "hello"}},
|
||||
"output": map[string]any{"format": "text", "validation_mode": "basic"},
|
||||
}
|
||||
if test.include {
|
||||
definition["version"] = test.version
|
||||
}
|
||||
state, err := load(mapping("prompt version label", map[string]any{
|
||||
"prompts": map[string]any{"example": definition},
|
||||
}), nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
captured, inspection, err := state.prompt("example")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
identities, err := state.ListPrompts(context.Background())
|
||||
if err != nil || len(identities) != 1 {
|
||||
t.Fatalf("list prompt labels: %#v, %v", identities, err)
|
||||
}
|
||||
check := func(name string, value *string) {
|
||||
t.Helper()
|
||||
if !test.wantPresent {
|
||||
if value != nil {
|
||||
t.Errorf("%s label = %q, want absent", name, *value)
|
||||
}
|
||||
return
|
||||
}
|
||||
if value == nil || *value != test.wantValue {
|
||||
t.Errorf("%s label = %v, want %q", name, value, test.wantValue)
|
||||
}
|
||||
}
|
||||
check("definition", captured.Version)
|
||||
check("inspection", inspection.PromptVersion)
|
||||
check("listing", identities[0].Version)
|
||||
if captured.Version != nil && (captured.Version == inspection.PromptVersion || captured.Version == identities[0].Version || inspection.PromptVersion == identities[0].Version) {
|
||||
t.Fatal("returned prompt labels share mutable pointers")
|
||||
}
|
||||
hashes[test.name] = inspection.PromptHash
|
||||
|
||||
if captured.Version != nil {
|
||||
*captured.Version = "mutated definition"
|
||||
}
|
||||
if inspection.PromptVersion != nil {
|
||||
*inspection.PromptVersion = "mutated inspection"
|
||||
}
|
||||
if identities[0].Version != nil {
|
||||
*identities[0].Version = "mutated listing"
|
||||
}
|
||||
again, againInspection, err := state.prompt("example")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
check("captured definition after mutation", again.Version)
|
||||
check("captured inspection after mutation", againInspection.PromptVersion)
|
||||
})
|
||||
}
|
||||
if hashes["absent"] == hashes["empty"] {
|
||||
t.Fatal("absent and declared-empty labels produced the same definition hash")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPromptVersionNullClearsInheritedLabel(t *testing.T) {
|
||||
prompt := map[string]any{
|
||||
"version": "base-label",
|
||||
"messages": []any{map[string]any{"role": "user", "content": "hello"}},
|
||||
"output": map[string]any{"format": "text", "validation_mode": "basic"},
|
||||
}
|
||||
state, err := load(
|
||||
mapping("clear inherited label", map[string]any{"prompts": map[string]any{"example": map[string]any{"version": nil}}}),
|
||||
[]configkit.Underlay{{Inputs: []configkit.Input{mapping("base prompt", map[string]any{"prompts": map[string]any{"example": prompt}})}}},
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
definition, inspection, err := state.prompt("example")
|
||||
if err != nil || definition.Version != nil || inspection.PromptVersion != nil {
|
||||
t.Fatalf("null overlay retained inherited label: definition=%+v inspection=%+v err=%v", definition, inspection, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPromptInspectionLookupMatchesSnapshotSelectionAndOwnsInputs(t *testing.T) {
|
||||
makePrompt := func(content string) map[string]any {
|
||||
return map[string]any{
|
||||
"default_profile": "partial",
|
||||
"inputs": []any{map[string]any{
|
||||
"name": "document", "required": true,
|
||||
"content_type": " text/plain ", "description": " source document ",
|
||||
}},
|
||||
"messages": []any{map[string]any{"role": "user", "content": content}},
|
||||
"output": map[string]any{"format": "text", "validation_mode": "none"},
|
||||
}
|
||||
}
|
||||
selectedPrompt := makePrompt("selected")
|
||||
selectedPrompt["version"] = "v2"
|
||||
state, err := load(mapping("prompt inspection lookup", map[string]any{
|
||||
"profiles": map[string]any{"partial": map[string]any{}},
|
||||
"prompts": map[string]any{
|
||||
"selected": selectedPrompt,
|
||||
},
|
||||
}), nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
{
|
||||
inspection, err := state.GetPromptInspection(context.Background(), "selected")
|
||||
if err != nil {
|
||||
t.Fatalf("inspect prompt ID: %v", err)
|
||||
}
|
||||
definition, hash, err := state.GetPromptSnapshot(context.Background(), "selected")
|
||||
if err != nil {
|
||||
t.Fatalf("snapshot prompt ID: %v", err)
|
||||
}
|
||||
if inspection.PromptID != definition.ID || inspection.PromptVersion == nil || definition.Version == nil ||
|
||||
*inspection.PromptVersion != *definition.Version ||
|
||||
inspection.PromptHash != hash || inspection.DefaultProfileID != "partial" ||
|
||||
len(inspection.Inputs) != 1 || inspection.Inputs[0].Name != "document" ||
|
||||
inspection.Inputs[0].ContentType != "text/plain" || inspection.Inputs[0].Description != "source document" {
|
||||
t.Fatalf("inspection did not match the selected snapshot: inspection=%+v definition=%+v hash=%q", inspection, definition, hash)
|
||||
}
|
||||
if len(definition.Inputs) != len(inspection.Inputs) || definition.Inputs[0].Name != inspection.Inputs[0].Name {
|
||||
t.Fatalf("definition and inspection inputs diverged: definition=%+v inspection=%+v", definition.Inputs, inspection.Inputs)
|
||||
}
|
||||
}
|
||||
|
||||
owned, err := state.GetPromptInspection(context.Background(), "selected")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
owned.Inputs[0].Name = "caller mutation"
|
||||
owned.Inputs[0].Description = "caller mutation"
|
||||
again, err := state.GetPromptInspection(context.Background(), "selected")
|
||||
if err != nil || again.Inputs[0].Name != "document" || again.Inputs[0].Description != "source document" {
|
||||
t.Fatalf("inspection mutation escaped into captured state: inspection=%+v err=%v", again, err)
|
||||
}
|
||||
|
||||
for _, id := range []string{"missing", " "} {
|
||||
inspection, err := state.GetPromptInspection(context.Background(), id)
|
||||
if inspection != nil || !errors.Is(err, promptdef.ErrPromptDefinitionNotFound) {
|
||||
t.Errorf("inspection lookup %q=(%+v, %v), want not found", id, inspection, err)
|
||||
}
|
||||
definition, _, err := state.GetPromptSnapshot(context.Background(), id)
|
||||
if definition != nil || !errors.Is(err, promptdef.ErrPromptDefinitionNotFound) {
|
||||
t.Errorf("snapshot lookup %q=(%+v, %v), want not found", id, definition, err)
|
||||
}
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
if _, err := state.GetPromptInspection(ctx, "selected"); !errors.Is(err, context.Canceled) {
|
||||
t.Errorf("canceled inspection selection error=%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadAcceptsEmptyInlineContentAndRejectsInvalidPromptFields(t *testing.T) {
|
||||
state, err := load(mapping("empty-content", map[string]any{"prompts": map[string]any{" exact.id ": map[string]any{
|
||||
"version": "v1",
|
||||
"inputs": []any{map[string]any{"name": " note ", "required": true}},
|
||||
"messages": []any{map[string]any{"role": " USER ", "content": "", "content_file": nil,
|
||||
"cache_control": map[string]any{"type": " ephemeral ", "ttl": "1h "}}},
|
||||
"output": map[string]any{"format": "text", "validation_mode": "basic"},
|
||||
}}}), nil)
|
||||
if err != nil {
|
||||
var loadError *configkit.LoadError
|
||||
if errors.As(err, &loadError) {
|
||||
t.Fatalf("%v: %+v", err, loadError.Diagnostics())
|
||||
}
|
||||
t.Fatal(err)
|
||||
}
|
||||
definition, _, err := state.prompt(" exact.id ")
|
||||
if err != nil || definition.ID != " exact.id " || definition.Templates[0].Content != "" || definition.Templates[0].Role != "user" ||
|
||||
definition.Inputs[0].Name != "note" || definition.Templates[0].CacheControl.TTL != "1h" {
|
||||
t.Fatalf("explicit empty content or literal ID was changed: %+v, %v", definition, err)
|
||||
}
|
||||
if _, _, err := state.prompt("exact.id"); !errors.Is(err, errPromptNotFound) {
|
||||
t.Fatalf("trimmed prompt identity unexpectedly matched: %v", err)
|
||||
}
|
||||
_, err = load(mapping("obsolete-field", map[string]any{"prompts": map[string]any{"example": map[string]any{
|
||||
"version": "v1",
|
||||
"id": "example", "messages": []any{map[string]any{"role": "user", "content": "ok"}},
|
||||
"output": map[string]any{"format": "text", "validation_mode": "basic"},
|
||||
}}}), nil)
|
||||
var loadError *configkit.LoadError
|
||||
if !errors.As(err, &loadError) {
|
||||
t.Fatalf("obsolete identity field should fail strict decoding: %v", err)
|
||||
}
|
||||
_, err = load(mapping("null-prompt", map[string]any{"prompts": map[string]any{"example": nil}}), nil)
|
||||
if err == nil {
|
||||
t.Fatal("null prompt collection was accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRequirementSnapshotCapturesMetadataPresenceAndNormalizedValues(t *testing.T) {
|
||||
state, err := load(mapping("requirement-metadata", map[string]any{
|
||||
"prompts": map[string]any{"example": map[string]any{
|
||||
"version": "v1",
|
||||
"session_id": "{{.session}}",
|
||||
"inputs": []any{
|
||||
map[string]any{"name": "absent"},
|
||||
map[string]any{"name": "null", "content_type": nil, "description": nil},
|
||||
map[string]any{"name": "empty", "content_type": "", "description": ""},
|
||||
map[string]any{"name": "normalized", "content_type": " text/plain ", "description": " final text "},
|
||||
},
|
||||
"messages": []any{map[string]any{"role": "user", "content": "{{input \"absent\"}}"}},
|
||||
"output": map[string]any{"format": "text", "validation_mode": "basic"},
|
||||
}},
|
||||
}), nil)
|
||||
if err != nil {
|
||||
var loadError *configkit.LoadError
|
||||
if errors.As(err, &loadError) {
|
||||
t.Fatalf("%v: %+v", err, loadError.Diagnostics())
|
||||
}
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
snapshot, ok := state.GetRequirementSnapshot("example")
|
||||
if !ok || len(snapshot.Inputs) != 4 || len(snapshot.Templates.Messages) != 1 || snapshot.Templates.Session == nil {
|
||||
t.Fatalf("requirement snapshot was not captured: %+v, %t", snapshot, ok)
|
||||
}
|
||||
for _, index := range []int{0, 1} {
|
||||
if snapshot.Inputs[index].ContentType != nil || snapshot.Inputs[index].Description != nil {
|
||||
t.Errorf("input %q should not have explicit string metadata: %+v", snapshot.Inputs[index].Name, snapshot.Inputs[index])
|
||||
}
|
||||
}
|
||||
if snapshot.Inputs[2].ContentType == nil || *snapshot.Inputs[2].ContentType != "" ||
|
||||
snapshot.Inputs[2].Description == nil || *snapshot.Inputs[2].Description != "" {
|
||||
t.Errorf("explicit empty metadata presence was lost: %+v", snapshot.Inputs[2])
|
||||
}
|
||||
if snapshot.Inputs[3].ContentType == nil || *snapshot.Inputs[3].ContentType != "text/plain" ||
|
||||
snapshot.Inputs[3].Description == nil || *snapshot.Inputs[3].Description != "final text" {
|
||||
t.Errorf("snapshot did not use normalized final metadata: %+v", snapshot.Inputs[3])
|
||||
}
|
||||
|
||||
definition, inspection, err := state.prompt("example")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(definition.Inputs) != 4 || len(inspection.Inputs) != 4 || inspection.Inputs[2].ContentType != "" || inspection.Inputs[2].Description != "" ||
|
||||
inspection.Inputs[3].ContentType != "text/plain" || inspection.Inputs[3].Description != "final text" {
|
||||
t.Fatalf("existing prompt inspection shape or normalized values changed: %+v", inspection)
|
||||
}
|
||||
|
||||
*snapshot.Inputs[2].ContentType = "caller mutation"
|
||||
snapshot.Inputs[3].Name = "caller mutation"
|
||||
snapshot.Templates.Messages[0] = nil
|
||||
again, ok := state.GetRequirementSnapshot("example")
|
||||
if !ok || *again.Inputs[2].ContentType != "" || again.Inputs[3].Name != "normalized" || again.Templates.Messages[0] == nil {
|
||||
t.Fatalf("snapshot mutation escaped into captured state: %+v, %t", again, ok)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInputMetadataPresenceDoesNotChangePromptHashOrInspection(t *testing.T) {
|
||||
variants := []struct {
|
||||
name string
|
||||
fields map[string]any
|
||||
}{
|
||||
{name: "absent"},
|
||||
{name: "null", fields: map[string]any{"content_type": nil, "description": nil}},
|
||||
{name: "empty", fields: map[string]any{"content_type": "", "description": ""}},
|
||||
}
|
||||
var expectedHash string
|
||||
for _, variant := range variants {
|
||||
t.Run(variant.name, func(t *testing.T) {
|
||||
input := map[string]any{"name": "document"}
|
||||
for name, value := range variant.fields {
|
||||
input[name] = value
|
||||
}
|
||||
state, err := load(mapping("hash-"+variant.name, map[string]any{
|
||||
"prompts": map[string]any{"same": map[string]any{
|
||||
"version": "v1",
|
||||
"inputs": []any{input},
|
||||
"messages": []any{map[string]any{"role": "user", "content": "hello"}},
|
||||
"output": map[string]any{"format": "text", "validation_mode": "basic"},
|
||||
}},
|
||||
}), nil)
|
||||
if err != nil {
|
||||
var loadError *configkit.LoadError
|
||||
if errors.As(err, &loadError) {
|
||||
t.Fatalf("%v: %+v", err, loadError.Diagnostics())
|
||||
}
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, inspection, err := state.prompt("same")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(inspection.Inputs) != 1 || inspection.Inputs[0].ContentType != "" || inspection.Inputs[0].Description != "" {
|
||||
t.Fatalf("old inspection metadata changed: %+v", inspection.Inputs)
|
||||
}
|
||||
if expectedHash == "" {
|
||||
expectedHash = inspection.PromptHash
|
||||
} else if inspection.PromptHash != expectedHash {
|
||||
t.Fatalf("metadata presence changed the definition hash: got %q, want %q", inspection.PromptHash, expectedHash)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func promptWithFile(name string) map[string]any {
|
||||
return map[string]any{"prompts": map[string]any{"example": map[string]any{
|
||||
"messages": []any{map[string]any{"role": "user", "content_file": name}},
|
||||
"output": map[string]any{"format": "text", "validation_mode": "basic"},
|
||||
}}}
|
||||
}
|
||||
|
||||
type countingFS struct {
|
||||
fs.FS
|
||||
contentOpened int
|
||||
contentClosed int
|
||||
}
|
||||
|
||||
func (filesystem *countingFS) Open(name string) (fs.File, error) {
|
||||
file, err := filesystem.FS.Open(name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if name == "assets/body.md" {
|
||||
filesystem.contentOpened++
|
||||
return &countingFile{File: file, owner: filesystem}, nil
|
||||
}
|
||||
return file, nil
|
||||
}
|
||||
|
||||
type countingFile struct {
|
||||
fs.File
|
||||
owner *countingFS
|
||||
}
|
||||
|
||||
func (file *countingFile) Close() error {
|
||||
file.owner.contentClosed++
|
||||
return file.File.Close()
|
||||
}
|
||||
215
internal/configuration/targets.go
Normal file
215
internal/configuration/targets.go
Normal file
@@ -0,0 +1,215 @@
|
||||
package configuration
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/backend"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/defaults"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/jsonvalue"
|
||||
profilepkg "gitea.maximumdirect.net/eric/promptkit/internal/profile"
|
||||
)
|
||||
|
||||
func (state *loaded) buildExecutionProfiles() error {
|
||||
backendConfigs := indexedBackends(&state.result.Config)
|
||||
profileConfigs := indexedProfiles(&state.result.Config)
|
||||
definitions := make(map[string]domain.Backend, len(backendConfigs))
|
||||
for _, id := range sortedKeys(backendConfigs) {
|
||||
config := backendConfigs[id]
|
||||
definition := domain.Backend{ID: id, ExtraParams: config.ExtraParams}
|
||||
if config.SmokeTests != nil {
|
||||
smokeTests := &domain.BackendSmokeTestConfig{DefaultProfile: config.SmokeTests.DefaultProfile}
|
||||
if config.SmokeTests.Reachability != nil {
|
||||
smokeTests.Reachability = &domain.ReachabilityProbeConfig{Endpoint: config.SmokeTests.Reachability.Endpoint}
|
||||
}
|
||||
if config.SmokeTests.Authentication != nil {
|
||||
smokeTests.Authentication = &domain.AuthenticationProbeConfig{
|
||||
Kind: config.SmokeTests.Authentication.Kind,
|
||||
Endpoint: config.SmokeTests.Authentication.Endpoint,
|
||||
}
|
||||
}
|
||||
definition.SmokeTests = smokeTests
|
||||
}
|
||||
if config.Endpoint != nil {
|
||||
definition.Endpoint = *config.Endpoint
|
||||
}
|
||||
if config.APIKeyEnv != nil {
|
||||
definition.APIKeyEnv = *config.APIKeyEnv
|
||||
}
|
||||
if config.ConcurrencyLimit != nil {
|
||||
definition.ConcurrencyLimit = *config.ConcurrencyLimit
|
||||
}
|
||||
if config.QueueCapacity != nil {
|
||||
definition.QueueCapacity = *config.QueueCapacity
|
||||
definition.QueueCapacitySet = definition.ConcurrencyLimit > 0
|
||||
}
|
||||
definitions[id] = definition
|
||||
}
|
||||
registry, err := backend.NewEffectiveRegistry(definitions)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
state.registry = registry
|
||||
state.indexedProfiles = make(map[string]domain.ExecutionProfile, len(profileConfigs))
|
||||
for _, id := range sortedKeys(profileConfigs) {
|
||||
config := profileConfigs[id]
|
||||
fields := state.profiles[id]
|
||||
target := defaults.ExecutionTargetDefault()
|
||||
if config.Backend != nil && *config.Backend != "" {
|
||||
selected, err := registry.GetBackend(*config.Backend)
|
||||
if err != nil {
|
||||
return fmt.Errorf("profile %q backend: %w", id, err)
|
||||
}
|
||||
target.BackendID = selected.ID
|
||||
target.Endpoint = selected.Endpoint
|
||||
target.APIKeyEnv = selected.APIKeyEnv
|
||||
target.ExtraParams = selected.ExtraParams
|
||||
}
|
||||
presence := domain.ExecutionTargetPresence{}
|
||||
if fields["endpoint"] != fieldAbsent {
|
||||
target.Endpoint = valueOrZero(config.Endpoint)
|
||||
}
|
||||
if fields["model"] != fieldAbsent {
|
||||
target.Model = valueOrZero(config.Model)
|
||||
}
|
||||
if fields["temperature"] != fieldAbsent {
|
||||
target.Temperature = valueOrZero(config.Temperature)
|
||||
presence.Temperature = fields["temperature"] == fieldValue
|
||||
}
|
||||
if fields["top_p"] != fieldAbsent {
|
||||
target.TopP = valueOrZero(config.TopP)
|
||||
presence.TopP = fields["top_p"] == fieldValue
|
||||
}
|
||||
if fields["max_tokens"] != fieldAbsent {
|
||||
target.MaxTokens = valueOrZero(config.MaxTokens)
|
||||
presence.MaxTokens = fields["max_tokens"] == fieldValue
|
||||
}
|
||||
if fields["timeout_seconds"] == fieldValue {
|
||||
target.TimeoutSeconds = valueOrZero(config.TimeoutSeconds)
|
||||
presence.TimeoutSeconds = true
|
||||
}
|
||||
if fields["service_tier"] != fieldAbsent {
|
||||
target.ServiceTier = valueOrZero(config.ServiceTier)
|
||||
}
|
||||
if fields["reasoning_effort"] != fieldAbsent {
|
||||
target.ReasoningEffort = valueOrZero(config.ReasoningEffort)
|
||||
}
|
||||
if fields["api_key_env"] != fieldAbsent {
|
||||
target.APIKeyEnv = valueOrZero(config.APIKeyEnv)
|
||||
}
|
||||
if fields["api_key_required"] != fieldAbsent {
|
||||
target.APIKeyRequired = valueOrZero(config.APIKeyRequired)
|
||||
}
|
||||
if target.APIKeyRequired && fields["api_key_env"] == fieldAbsent {
|
||||
target.APIKeyEnv = ""
|
||||
}
|
||||
switch fields["extra_params"] {
|
||||
case fieldNull:
|
||||
target.ExtraParams = nil
|
||||
case fieldValue:
|
||||
merged, err := overlayProviderMap(target.ExtraParams, config.ExtraParams)
|
||||
if err != nil {
|
||||
return fmt.Errorf("profile %q extra parameters: %w", id, err)
|
||||
}
|
||||
target.ExtraParams = merged
|
||||
}
|
||||
profile := domain.ExecutionProfile{ID: id, ResolvedTarget: &target, TargetPresence: presence}
|
||||
state.indexedProfiles[id] = profile
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func valueOrZero[T any](value *T) T {
|
||||
if value == nil {
|
||||
var zero T
|
||||
return zero
|
||||
}
|
||||
return *value
|
||||
}
|
||||
|
||||
// overlayProviderMap applies only the backend-to-profile JSON-object default
|
||||
// merge. ConfigKit has already resolved source layering and inheritance.
|
||||
func overlayProviderMap(lower, upper map[string]any) (map[string]any, error) {
|
||||
merged, err := jsonvalue.CopyMap(lower)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if merged == nil {
|
||||
merged = make(map[string]any)
|
||||
}
|
||||
for key, value := range upper {
|
||||
if child, ok := value.(map[string]any); ok {
|
||||
if existing, ok := merged[key].(map[string]any); ok {
|
||||
combined, err := overlayProviderMap(existing, child)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
merged[key] = combined
|
||||
continue
|
||||
}
|
||||
}
|
||||
copied, err := jsonvalue.Copy(value)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
merged[key] = copied
|
||||
}
|
||||
return merged, nil
|
||||
}
|
||||
|
||||
func (state *loaded) lookupProfile(ctx context.Context, id string) (domain.ExecutionProfile, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return domain.ExecutionProfile{}, err
|
||||
}
|
||||
if state == nil {
|
||||
return domain.ExecutionProfile{}, fmt.Errorf("%w: %q", profilepkg.ErrProfileNotFound, id)
|
||||
}
|
||||
profile, found := state.indexedProfiles[id]
|
||||
if !found {
|
||||
return domain.ExecutionProfile{}, fmt.Errorf("%w: %q", profilepkg.ErrProfileNotFound, id)
|
||||
}
|
||||
return profile, nil
|
||||
}
|
||||
|
||||
// GetProfile returns an independently owned indexed profile for the internal
|
||||
// runner's lookup boundary.
|
||||
func (state *loaded) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) {
|
||||
profile, err := state.lookupProfile(ctx, id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
target := *profile.ResolvedTarget
|
||||
params, err := jsonvalue.CopyMap(target.ExtraParams)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
target.ExtraParams = params
|
||||
profile.ResolvedTarget = &target
|
||||
return &profile, nil
|
||||
}
|
||||
|
||||
// GetProfileIdentity returns the exact lookup ID for a captured profile
|
||||
// without copying its execution target.
|
||||
func (state *loaded) GetProfileIdentity(ctx context.Context, id string) (string, error) {
|
||||
if _, err := state.lookupProfile(ctx, id); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return id, nil
|
||||
}
|
||||
|
||||
// GetBackend returns an independently owned effective backend definition.
|
||||
func (state *loaded) GetBackend(id string) (domain.Backend, error) {
|
||||
if state == nil {
|
||||
return domain.Backend{}, fmt.Errorf("backend registry is not configured")
|
||||
}
|
||||
return state.registry.GetBackend(id)
|
||||
}
|
||||
|
||||
// CapacityPolicies returns the selected backend admission policies.
|
||||
func (state *loaded) CapacityPolicies() map[string]domain.BackendCapacityPolicy {
|
||||
if state == nil {
|
||||
return nil
|
||||
}
|
||||
return state.registry.CapacityPolicies()
|
||||
}
|
||||
166
internal/configuration/targets_test.go
Normal file
166
internal/configuration/targets_test.go
Normal file
@@ -0,0 +1,166 @@
|
||||
package configuration
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/defaults"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/llm"
|
||||
profilepkg "gitea.maximumdirect.net/eric/promptkit/internal/profile"
|
||||
)
|
||||
|
||||
func TestLoadBuildsEffectiveTargetsAfterInheritance(t *testing.T) {
|
||||
state, err := load(mapping("effective-targets", map[string]any{
|
||||
"backends": map[string]any{
|
||||
"first": map[string]any{"endpoint": "https://first.example/v1", "api_key_env": "FIRST_KEY",
|
||||
"extra_params": map[string]any{"source": "first", "nested": map[string]any{"old": true}}},
|
||||
"second": map[string]any{"endpoint": "https://second.example/v1", "api_key_env": "SECOND_KEY",
|
||||
"concurrency_limit": 2, "queue_capacity": 0,
|
||||
"extra_params": map[string]any{"source": "second", "nested": map[string]any{"backend": true}}},
|
||||
},
|
||||
"profiles": map[string]any{
|
||||
"base": map[string]any{"backend": "first", "model": "model", "temperature": 0.8,
|
||||
"top_p": 0.9, "max_tokens": 100, "timeout_seconds": 30, "api_key_required": true,
|
||||
"extra_params": map[string]any{"nested": map[string]any{"profile": true}}},
|
||||
"child": map[string]any{"extends": "base", "backend": "second", "temperature": 0.0,
|
||||
"top_p": nil, "max_tokens": 0, "timeout_seconds": nil, "api_key_env": nil,
|
||||
"api_key_required": false,
|
||||
"extra_params": map[string]any{"nested": map[string]any{"child": nil}}},
|
||||
"direct-key": map[string]any{"backend": "second", "api_key_required": true},
|
||||
"named-key": map[string]any{"backend": "second", "api_key_required": true, "api_key_env": "CUSTOM_KEY"},
|
||||
"no-provider-map": map[string]any{"backend": "second", "extra_params": nil},
|
||||
"no-timeout": map[string]any{"backend": "second", "timeout_seconds": 0},
|
||||
"partial": map[string]any{},
|
||||
"endpoint-only": map[string]any{"endpoint": "https://standalone.example/v1"},
|
||||
"cleared-endpoint": map[string]any{"extends": "base", "endpoint": nil},
|
||||
},
|
||||
}), nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
selected, err := state.registry.GetBackend("second")
|
||||
if err != nil || selected.QueueCapacity != 0 || !selected.QueueCapacitySet {
|
||||
t.Fatalf("backend capacity was not preserved: %+v, %v", selected, err)
|
||||
}
|
||||
child, err := state.GetProfile(context.Background(), "child")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := state.GetProfile(context.Background(), " child "); err == nil {
|
||||
t.Fatal("profile lookup silently changed a literal ID")
|
||||
}
|
||||
target := child.ResolvedTarget
|
||||
if target.BackendID != "second" || target.Endpoint != "https://second.example/v1" || target.Model != "model" ||
|
||||
target.Temperature != 0 || target.TopP != 0 || target.MaxTokens != 0 ||
|
||||
target.TimeoutSeconds != defaults.ExecutionDefaultTimeoutSeconds || target.APIKeyEnv != "" || target.APIKeyRequired {
|
||||
t.Fatalf("effective child target is wrong: %+v", target)
|
||||
}
|
||||
if !child.TargetPresence.Temperature || !child.TargetPresence.MaxTokens || child.TargetPresence.TopP || child.TargetPresence.TimeoutSeconds {
|
||||
t.Fatalf("numeric presence was not retained: %+v", child.TargetPresence)
|
||||
}
|
||||
wantParams := map[string]any{"source": "second", "nested": map[string]any{
|
||||
"backend": true, "profile": true, "child": nil,
|
||||
}}
|
||||
if !reflect.DeepEqual(target.ExtraParams, wantParams) {
|
||||
t.Fatalf("backend/profile provider map overlay failed: %+v", target.ExtraParams)
|
||||
}
|
||||
if _, exists := target.ExtraParams["nested"].(map[string]any)["old"]; exists {
|
||||
t.Fatal("previous backend derived parameter leaked into child")
|
||||
}
|
||||
for id, wantEnv := range map[string]string{"direct-key": "", "named-key": "CUSTOM_KEY"} {
|
||||
profile, err := state.GetProfile(context.Background(), id)
|
||||
if err != nil || profile.ResolvedTarget.APIKeyEnv != wantEnv {
|
||||
t.Fatalf("credential source for %s: %+v, %v", id, profile, err)
|
||||
}
|
||||
}
|
||||
noMap, _ := state.GetProfile(context.Background(), "no-provider-map")
|
||||
if noMap.ResolvedTarget.ExtraParams != nil {
|
||||
t.Fatalf("explicit null provider map did not suppress backend defaults: %+v", noMap.ResolvedTarget.ExtraParams)
|
||||
}
|
||||
noTimeout, _ := state.GetProfile(context.Background(), "no-timeout")
|
||||
if noTimeout.ResolvedTarget.TimeoutSeconds != 0 || !noTimeout.TargetPresence.TimeoutSeconds {
|
||||
t.Fatalf("explicit zero timeout was lost: %+v", noTimeout)
|
||||
}
|
||||
partial, _ := state.GetProfile(context.Background(), "partial")
|
||||
if partial.ResolvedTarget.Endpoint != "" || partial.ResolvedTarget.Model != "" {
|
||||
t.Fatalf("partial profile gained a runnable target: %+v", partial.ResolvedTarget)
|
||||
}
|
||||
identity, err := state.GetProfileIdentity(context.Background(), "partial")
|
||||
if err != nil || identity != "partial" {
|
||||
t.Fatalf("partial profile identity=(%q, %v)", identity, err)
|
||||
}
|
||||
if _, err := state.GetProfileIdentity(context.Background(), "missing"); !errors.Is(err, profilepkg.ErrProfileNotFound) {
|
||||
t.Fatalf("missing profile identity error=%v", err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
if _, err := state.GetProfileIdentity(ctx, "partial"); !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("canceled profile identity lookup error=%v", err)
|
||||
}
|
||||
endpointOnly, _ := state.GetProfile(context.Background(), "endpoint-only")
|
||||
if endpointOnly.ResolvedTarget.Endpoint != "https://standalone.example/v1" ||
|
||||
endpointOnly.ResolvedTarget.BackendID != "" || endpointOnly.ResolvedTarget.Model != "" {
|
||||
t.Fatalf("endpoint-only profile was not retained: %+v", endpointOnly.ResolvedTarget)
|
||||
}
|
||||
clearedEndpoint, _ := state.GetProfile(context.Background(), "cleared-endpoint")
|
||||
if clearedEndpoint.ResolvedTarget.Endpoint != "" || clearedEndpoint.ResolvedTarget.BackendID != "first" {
|
||||
t.Fatalf("explicit null recovered backend endpoint: %+v", clearedEndpoint.ResolvedTarget)
|
||||
}
|
||||
child.ResolvedTarget.ExtraParams["source"] = "mutated"
|
||||
again, _ := state.GetProfile(context.Background(), "child")
|
||||
if again.ResolvedTarget.ExtraParams["source"] != "second" {
|
||||
t.Fatal("profile lookup exposed snapshot ownership")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadedZeroControlsReachBuiltInRequest(t *testing.T) {
|
||||
payloads := make(chan map[string]any, 1)
|
||||
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
||||
var payload map[string]any
|
||||
if err := json.NewDecoder(request.Body).Decode(&payload); err != nil {
|
||||
writer.WriteHeader(http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
payloads <- payload
|
||||
writer.Header().Set("Content-Type", "application/json")
|
||||
_, _ = writer.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
state, err := load(mapping("wire-controls", map[string]any{
|
||||
"backends": map[string]any{"local": map[string]any{"endpoint": server.URL + "/v1"}},
|
||||
"profiles": map[string]any{"direct": map[string]any{
|
||||
"backend": "local", "model": "example", "temperature": 0.0, "max_tokens": 0, "top_p": nil,
|
||||
}},
|
||||
}), nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
profile, err := state.GetProfile(context.Background(), "direct")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
client, err := llm.NewOpenAICompatibleClient(llm.OpenAICompatibleConfig{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = client.Generate(context.Background(), domain.GenerateRequest{
|
||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hello"}}},
|
||||
Target: *profile.ResolvedTarget, TargetPresence: profile.TargetPresence,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
payload := <-payloads
|
||||
if payload["temperature"] != float64(0) || payload["max_tokens"] != float64(0) {
|
||||
t.Fatalf("explicit zeros were omitted from outbound request: %+v", payload)
|
||||
}
|
||||
if _, exists := payload["top_p"]; exists {
|
||||
t.Fatalf("explicit null top_p was sent: %+v", payload)
|
||||
}
|
||||
}
|
||||
@@ -61,7 +61,6 @@ type CacheControl struct {
|
||||
// RunRequest represents a request to generate a single artifact.
|
||||
type RunRequest struct {
|
||||
PromptID string
|
||||
PromptVersion string
|
||||
ProfileID string
|
||||
SessionID string
|
||||
APIKey string `json:"-" yaml:"-"`
|
||||
@@ -81,7 +80,7 @@ type RunResult struct {
|
||||
RawOutput string
|
||||
Validation ValidationResult
|
||||
PromptID string
|
||||
PromptVersion string
|
||||
PromptVersion *string
|
||||
PromptHash string
|
||||
SessionID string
|
||||
RenderedPromptHash string
|
||||
@@ -101,7 +100,7 @@ type RunResult struct {
|
||||
// It must never include resolved API key values, model output, or validation data.
|
||||
type PreparedRun struct {
|
||||
PromptID string
|
||||
PromptVersion string
|
||||
PromptVersion *string
|
||||
PromptHash string
|
||||
SelectedProfileID string
|
||||
SelectedBackendID string
|
||||
@@ -135,10 +134,16 @@ type Artifact struct {
|
||||
Hash string
|
||||
}
|
||||
|
||||
// PromptIdentity identifies one configured prompt and its descriptive version label.
|
||||
type PromptIdentity struct {
|
||||
ID string
|
||||
Version *string
|
||||
}
|
||||
|
||||
// PromptDefinition represents a configured prompt execution definition.
|
||||
type PromptDefinition struct {
|
||||
ID string `yaml:"id"`
|
||||
Version string `yaml:"version"`
|
||||
Version *string `yaml:"version"`
|
||||
DefaultProfile string `yaml:"default_profile"`
|
||||
Description string `yaml:"description"`
|
||||
SessionID string `yaml:"session_id" json:"session_id,omitempty"`
|
||||
@@ -151,7 +156,7 @@ type PromptDefinition struct {
|
||||
// PromptInspection is the resolved result of exact prompt inspection.
|
||||
type PromptInspection struct {
|
||||
PromptID string
|
||||
PromptVersion string
|
||||
PromptVersion *string
|
||||
PromptHash string
|
||||
DefaultProfileID string
|
||||
Inputs []PromptInput
|
||||
@@ -174,12 +179,14 @@ type PromptMessageTemplate struct {
|
||||
CacheControl *CacheControl `yaml:"cache_control,omitempty" json:"cache_control,omitempty"`
|
||||
}
|
||||
|
||||
// Backend describes reusable OpenAI-compatible connection defaults.
|
||||
// Backend describes reusable OpenAI-compatible connection defaults and optional
|
||||
// diagnostic metadata.
|
||||
type Backend struct {
|
||||
ID string
|
||||
Endpoint string
|
||||
APIKeyEnv string
|
||||
ExtraParams map[string]any
|
||||
SmokeTests *BackendSmokeTestConfig
|
||||
ConcurrencyLimit int
|
||||
QueueCapacity int
|
||||
QueueCapacitySet bool
|
||||
@@ -192,22 +199,11 @@ type BackendCapacityPolicy struct {
|
||||
QueueCapacity int
|
||||
}
|
||||
|
||||
// ExecutionProfile describes how and where to execute a model.
|
||||
// ExecutionProfile holds a captured configuration target for execution.
|
||||
type ExecutionProfile struct {
|
||||
ID string `yaml:"id"`
|
||||
BaseProfileID string `yaml:"base_profile"`
|
||||
BackendID string `yaml:"backend"`
|
||||
Endpoint string `yaml:"endpoint"`
|
||||
Model string `yaml:"model"`
|
||||
Temperature float64 `yaml:"temperature"`
|
||||
MaxTokens int `yaml:"max_tokens"`
|
||||
TopP float64 `yaml:"top_p"`
|
||||
TimeoutSeconds int `yaml:"timeout_seconds"`
|
||||
ServiceTier string `yaml:"service_tier"`
|
||||
ReasoningEffort string `yaml:"reasoning_effort"`
|
||||
APIKeyEnv string `yaml:"api_key_env"`
|
||||
APIKeyRequired bool `yaml:"-" json:"-"`
|
||||
ExtraParams map[string]any `yaml:"extra_params"`
|
||||
ID string
|
||||
ResolvedTarget *ExecutionTarget
|
||||
TargetPresence ExecutionTargetPresence
|
||||
}
|
||||
|
||||
// ExecutionTargetOverride represents per-request runtime setting overrides.
|
||||
@@ -224,8 +220,9 @@ type ExecutionTargetOverride struct {
|
||||
ExtraParams map[string]any `json:"extra_params,omitempty"`
|
||||
}
|
||||
|
||||
// ExecutionTargetPresence tracks which effective runtime fields came from an
|
||||
// explicit request override even when the resolved value is a zero value.
|
||||
// ExecutionTargetPresence tracks effective numeric values that must be sent
|
||||
// even when they are zero. Values may come from configuration or a request
|
||||
// override; explicit configuration null suppresses the corresponding bit.
|
||||
type ExecutionTargetPresence struct {
|
||||
Temperature bool
|
||||
MaxTokens bool
|
||||
|
||||
@@ -68,7 +68,7 @@ type ExecutionEvent struct {
|
||||
RunID string
|
||||
Time time.Time
|
||||
PromptID string
|
||||
PromptVersion string
|
||||
PromptVersion *string
|
||||
ProfileID string
|
||||
BackendID string
|
||||
Model string
|
||||
|
||||
58
internal/domain/requirements.go
Normal file
58
internal/domain/requirements.go
Normal file
@@ -0,0 +1,58 @@
|
||||
package domain
|
||||
|
||||
// RequirementKind classifies how a prompt may depend on an input or variable.
|
||||
type RequirementKind string
|
||||
|
||||
const (
|
||||
RequirementRequired RequirementKind = "required"
|
||||
RequirementConditional RequirementKind = "conditional"
|
||||
RequirementOptional RequirementKind = "optional"
|
||||
)
|
||||
|
||||
// InputRequirement describes one declared or template-referenced input.
|
||||
type InputRequirement struct {
|
||||
Name string
|
||||
Kind RequirementKind
|
||||
Declared bool
|
||||
ContentType *string
|
||||
Description *string
|
||||
Explanation string
|
||||
}
|
||||
|
||||
// VariableRequirement describes one caller variable referenced by a prompt.
|
||||
type VariableRequirement struct {
|
||||
Name string
|
||||
Kind RequirementKind
|
||||
Supplied bool
|
||||
Explanation string
|
||||
}
|
||||
|
||||
// RequirementIssueCode identifies a limit that may affect requirement
|
||||
// discovery.
|
||||
type RequirementIssueCode string
|
||||
|
||||
const (
|
||||
RequirementIssueMissingVariable RequirementIssueCode = "missing_variable"
|
||||
RequirementIssueInputContent RequirementIssueCode = "input_content"
|
||||
RequirementIssueDynamicInputName RequirementIssueCode = "dynamic_input_name"
|
||||
RequirementIssueUnsupportedConstruct RequirementIssueCode = "unsupported_construct"
|
||||
)
|
||||
|
||||
// RequirementIssue describes one limitation on requirement discovery.
|
||||
type RequirementIssue struct {
|
||||
Code RequirementIssueCode
|
||||
InputName string
|
||||
VariableName string
|
||||
Description string
|
||||
}
|
||||
|
||||
// RequirementInspection contains the inventories discovered for one prompt.
|
||||
type RequirementInspection struct {
|
||||
PromptID string
|
||||
PromptVersion *string
|
||||
ProfileID string
|
||||
Inputs []InputRequirement
|
||||
Variables []VariableRequirement
|
||||
Complete bool
|
||||
Issues []RequirementIssue
|
||||
}
|
||||
70
internal/domain/resource_identity.go
Normal file
70
internal/domain/resource_identity.go
Normal file
@@ -0,0 +1,70 @@
|
||||
package domain
|
||||
|
||||
import "strings"
|
||||
|
||||
func validResourceID(id string) bool {
|
||||
if !strings.Contains(id, "/") {
|
||||
return strings.TrimSpace(id) != ""
|
||||
}
|
||||
if strings.Count(id, "/") != 1 {
|
||||
return false
|
||||
}
|
||||
namespace, localID, _ := strings.Cut(id, "/")
|
||||
return namespace != "" && localID != ""
|
||||
}
|
||||
|
||||
func validResourceNamespace(namespace string) bool {
|
||||
return strings.TrimSpace(namespace) != "" && !strings.Contains(namespace, "/")
|
||||
}
|
||||
|
||||
func validLocalResourceKey(key string) bool {
|
||||
return strings.TrimSpace(key) != "" && !strings.Contains(key, "/")
|
||||
}
|
||||
|
||||
func qualifyResourceID(id, namespace string) (string, bool) {
|
||||
if !validResourceID(id) || (namespace != "" && !validResourceNamespace(namespace)) {
|
||||
return "", false
|
||||
}
|
||||
if strings.Contains(id, "/") || namespace == "" {
|
||||
return id, true
|
||||
}
|
||||
return namespace + "/" + id, true
|
||||
}
|
||||
|
||||
func resourceNamespaceFromID(id string) (string, bool) {
|
||||
if !validResourceID(id) || !strings.Contains(id, "/") {
|
||||
return "", false
|
||||
}
|
||||
namespace, _, _ := strings.Cut(id, "/")
|
||||
return namespace, true
|
||||
}
|
||||
|
||||
// IsValidResourceID reports whether id is an unqualified resource ID or one
|
||||
// valid namespace/local-ID pair.
|
||||
func IsValidResourceID(id string) bool { return validResourceID(id) }
|
||||
|
||||
// QualifyResourceID validates id and applies namespace to an unqualified ID.
|
||||
// An already-qualified ID is returned unchanged.
|
||||
func QualifyResourceID(id, namespace string) (string, bool) {
|
||||
return qualifyResourceID(id, namespace)
|
||||
}
|
||||
|
||||
// IsValidResourceNamespace reports whether namespace is nonblank and contains
|
||||
// no namespace separator.
|
||||
func IsValidResourceNamespace(namespace string) bool { return validResourceNamespace(namespace) }
|
||||
|
||||
// IsValidLocalResourceKey reports whether key is a nonblank key for a
|
||||
// namespace-local resource mapping.
|
||||
func IsValidLocalResourceKey(key string) bool { return validLocalResourceKey(key) }
|
||||
|
||||
// QualifyLocalResourceID combines a validated namespace and local mapping key.
|
||||
// An empty namespace leaves a valid unqualified key unchanged.
|
||||
func QualifyLocalResourceID(namespace, key string) (string, bool) {
|
||||
if !validLocalResourceKey(key) || (namespace != "" && !validResourceNamespace(namespace)) {
|
||||
return "", false
|
||||
}
|
||||
if namespace == "" {
|
||||
return key, true
|
||||
}
|
||||
return namespace + "/" + key, true
|
||||
}
|
||||
110
internal/domain/resource_identity_test.go
Normal file
110
internal/domain/resource_identity_test.go
Normal file
@@ -0,0 +1,110 @@
|
||||
package domain
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestValidResourceID(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
id string
|
||||
want bool
|
||||
}{
|
||||
{name: "ordinary", id: "draft", want: true},
|
||||
{name: "dotted", id: "group.prompt.v2", want: true},
|
||||
{name: "preserved surrounding whitespace", id: " draft ", want: true},
|
||||
{name: "qualified", id: "group/draft", want: true},
|
||||
{name: "qualified components preserve spelling", id: " Group / draft ", want: true},
|
||||
{name: "empty", id: "", want: false},
|
||||
{name: "blank", id: " \t", want: false},
|
||||
{name: "missing namespace", id: "/draft", want: false},
|
||||
{name: "missing local id", id: "group/", want: false},
|
||||
{name: "multiple separators", id: "group/subgroup/draft", want: false},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
if got := validResourceID(test.id); got != test.want {
|
||||
t.Fatalf("validResourceID(%q) = %t, want %t", test.id, got, test.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidResourceNamespace(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
namespace string
|
||||
want bool
|
||||
}{
|
||||
{name: "ordinary", namespace: "group", want: true},
|
||||
{name: "dotted", namespace: "group.v2", want: true},
|
||||
{name: "preserved whitespace", namespace: " group ", want: true},
|
||||
{name: "empty", namespace: "", want: false},
|
||||
{name: "blank", namespace: " \t", want: false},
|
||||
{name: "slash", namespace: "group/subgroup", want: false},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
if got := validResourceNamespace(test.namespace); got != test.want {
|
||||
t.Fatalf("validResourceNamespace(%q) = %t, want %t", test.namespace, got, test.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidLocalResourceKey(t *testing.T) {
|
||||
if !validLocalResourceKey("draft.v2") || !validLocalResourceKey(" draft ") {
|
||||
t.Fatal("expected nonblank slash-free keys to be valid")
|
||||
}
|
||||
for _, key := range []string{"", " \t", "group/draft"} {
|
||||
if validLocalResourceKey(key) {
|
||||
t.Errorf("validLocalResourceKey(%q) = true, want false", key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestQualifyResourceID(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
id string
|
||||
namespace string
|
||||
want string
|
||||
wantOK bool
|
||||
}{
|
||||
{name: "without scope", id: "draft", want: "draft", wantOK: true},
|
||||
{name: "with scope", id: "draft", namespace: "group", want: "group/draft", wantOK: true},
|
||||
{name: "qualified bypass", id: "other/draft", namespace: "group", want: "other/draft", wantOK: true},
|
||||
{name: "preserve spelling", id: " draft ", namespace: " Group ", want: " Group / draft ", wantOK: true},
|
||||
{name: "omission remains caller policy", id: "", wantOK: false},
|
||||
{name: "malformed ID", id: "group/draft/extra", namespace: "group", wantOK: false},
|
||||
{name: "malformed scope", id: "draft", namespace: "group/subgroup", wantOK: false},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
got, ok := qualifyResourceID(test.id, test.namespace)
|
||||
if got != test.want || ok != test.wantOK {
|
||||
t.Fatalf("qualifyResourceID(%q, %q) = (%q, %t), want (%q, %t)", test.id, test.namespace, got, ok, test.want, test.wantOK)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestResourceNamespaceFromID(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
id string
|
||||
namespace string
|
||||
wantOK bool
|
||||
}{
|
||||
{name: "qualified", id: "team/item", namespace: "team", wantOK: true},
|
||||
{name: "same-prefix namespace stays distinct", id: "team-extra/item", namespace: "team-extra", wantOK: true},
|
||||
{name: "unqualified", id: "item", wantOK: false},
|
||||
{name: "malformed", id: "team/sub/item", wantOK: false},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
got, ok := resourceNamespaceFromID(test.id)
|
||||
if got != test.namespace || ok != test.wantOK {
|
||||
t.Fatalf("resourceNamespaceFromID(%q) = (%q, %t), want (%q, %t)", test.id, got, ok, test.namespace, test.wantOK)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
263
internal/domain/smoke_diagnostics.go
Normal file
263
internal/domain/smoke_diagnostics.go
Normal file
@@ -0,0 +1,263 @@
|
||||
package domain
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/jsonvalue"
|
||||
)
|
||||
|
||||
// BackendSmokeTestConfig describes optional backend-owned diagnostic defaults.
|
||||
type BackendSmokeTestConfig struct {
|
||||
DefaultProfile *string
|
||||
Reachability *ReachabilityProbeConfig
|
||||
Authentication *AuthenticationProbeConfig
|
||||
}
|
||||
|
||||
// ReachabilityProbeConfig describes an optional endpoint for reachability tests.
|
||||
type ReachabilityProbeConfig struct {
|
||||
Endpoint *string
|
||||
}
|
||||
|
||||
// AuthenticationProbeConfig describes an optional authentication diagnostic.
|
||||
type AuthenticationProbeConfig struct {
|
||||
Kind *string
|
||||
Endpoint *string
|
||||
}
|
||||
|
||||
// SmokeTestKind identifies one built-in diagnostic operation.
|
||||
type SmokeTestKind string
|
||||
|
||||
const (
|
||||
SmokeTestReachability SmokeTestKind = "reachability"
|
||||
SmokeTestAuthentication SmokeTestKind = "authentication"
|
||||
SmokeTestTextInference SmokeTestKind = "text_inference"
|
||||
SmokeTestStructuredOutput SmokeTestKind = "structured_output"
|
||||
)
|
||||
|
||||
// SmokeTestRequest selects one diagnostic and its execution target.
|
||||
type SmokeTestRequest struct {
|
||||
Kind SmokeTestKind
|
||||
BackendID string
|
||||
PromptID string
|
||||
ProfileID string
|
||||
Execution *ExecutionTargetOverride
|
||||
APIKey string `json:"-" yaml:"-"`
|
||||
TransientRetries *int
|
||||
}
|
||||
|
||||
// SmokeTestReport contains factual diagnostic results, not generated content.
|
||||
type SmokeTestReport struct {
|
||||
Kind SmokeTestKind
|
||||
Supported bool
|
||||
Applicable *bool
|
||||
Completed bool
|
||||
Reason SmokeTestReason
|
||||
Target SmokeTestTarget
|
||||
Facts SmokeTestFacts
|
||||
Timing SmokeTestTiming
|
||||
Attempts []SmokeTestAttempt
|
||||
Usage ObservedTokenUsage
|
||||
Failure ExecutionFailureKind
|
||||
}
|
||||
|
||||
// SmokeTestTarget identifies the selected diagnostic target and its settings.
|
||||
type SmokeTestTarget struct {
|
||||
BackendID string
|
||||
ProfileID string
|
||||
PromptID string
|
||||
PromptVersion *string
|
||||
DefaultSourceBackendID string
|
||||
SelectionSource string
|
||||
Endpoint string
|
||||
EffectiveSettings ExecutionTarget
|
||||
SettingsPresence ExecutionTargetPresence
|
||||
}
|
||||
|
||||
// SmokeTestFacts contains optional factual observations from a diagnostic.
|
||||
type SmokeTestFacts struct {
|
||||
CredentialsSupplied *bool
|
||||
EndpointReachable *bool
|
||||
CredentialsAccepted *bool
|
||||
ResponseReceived *bool
|
||||
GenerationCompleted *bool
|
||||
TextReturned *bool
|
||||
ExpectedTextMatched *bool
|
||||
StructuredOutputRequested *bool
|
||||
ValidJSON *bool
|
||||
SchemaMatched *bool
|
||||
}
|
||||
|
||||
// SmokeTestTiming records elapsed diagnostic and phase durations.
|
||||
type SmokeTestTiming struct {
|
||||
StartedAt time.Time
|
||||
Duration time.Duration
|
||||
CapacityWait *time.Duration
|
||||
RequestDuration *time.Duration
|
||||
RetryDelay *time.Duration
|
||||
EvaluationDuration *time.Duration
|
||||
}
|
||||
|
||||
// SmokeTestAttempt records one dispatched attempt or an interrupted acquisition.
|
||||
type SmokeTestAttempt struct {
|
||||
Dispatched bool
|
||||
InitialEndpoint string
|
||||
Endpoint string
|
||||
HTTPStatus *int
|
||||
Facts SmokeTestFacts
|
||||
Timing SmokeTestTiming
|
||||
Failure ExecutionFailureKind
|
||||
Usage ObservedTokenUsage
|
||||
}
|
||||
|
||||
// SmokeTestReason identifies additional factual context for a diagnostic.
|
||||
type SmokeTestReason string
|
||||
|
||||
const (
|
||||
SmokeTestReasonNone SmokeTestReason = ""
|
||||
SmokeTestReasonAuthenticationNotApplicable SmokeTestReason = "authentication_not_applicable"
|
||||
SmokeTestReasonProbeNotConfigured SmokeTestReason = "probe_not_configured"
|
||||
SmokeTestReasonHTTPDiagnosticsUnavailable SmokeTestReason = "http_diagnostics_unavailable"
|
||||
SmokeTestReasonRequiredCredentialMissing SmokeTestReason = "required_credential_missing"
|
||||
SmokeTestReasonAuthenticationResponseUnrecognized SmokeTestReason = "authentication_response_unrecognized"
|
||||
)
|
||||
|
||||
// CloneBackend returns a copy of a backend, including its mutable settings.
|
||||
func CloneBackend(backend Backend) (Backend, error) {
|
||||
cloned := backend
|
||||
var err error
|
||||
cloned.ExtraParams, err = jsonvalue.CopyMap(backend.ExtraParams)
|
||||
if err != nil {
|
||||
return Backend{}, err
|
||||
}
|
||||
cloned.SmokeTests = CloneBackendSmokeTestConfig(backend.SmokeTests)
|
||||
return cloned, nil
|
||||
}
|
||||
|
||||
// CloneBackendSmokeTestConfig returns an independent copy of optional metadata.
|
||||
func CloneBackendSmokeTestConfig(config *BackendSmokeTestConfig) *BackendSmokeTestConfig {
|
||||
if config == nil {
|
||||
return nil
|
||||
}
|
||||
cloned := *config
|
||||
cloned.DefaultProfile = cloneString(config.DefaultProfile)
|
||||
if config.Reachability != nil {
|
||||
reachability := *config.Reachability
|
||||
reachability.Endpoint = cloneString(config.Reachability.Endpoint)
|
||||
cloned.Reachability = &reachability
|
||||
}
|
||||
if config.Authentication != nil {
|
||||
authentication := *config.Authentication
|
||||
authentication.Kind = cloneString(config.Authentication.Kind)
|
||||
authentication.Endpoint = cloneString(config.Authentication.Endpoint)
|
||||
cloned.Authentication = &authentication
|
||||
}
|
||||
return &cloned
|
||||
}
|
||||
|
||||
// CloneSmokeTestRequest returns a copy that owns its optional settings.
|
||||
func CloneSmokeTestRequest(request SmokeTestRequest) (SmokeTestRequest, error) {
|
||||
cloned := request
|
||||
cloned.TransientRetries = cloneInt(request.TransientRetries)
|
||||
if request.Execution == nil {
|
||||
return cloned, nil
|
||||
}
|
||||
execution := *request.Execution
|
||||
execution.Temperature = cloneFloat64(request.Execution.Temperature)
|
||||
execution.MaxTokens = cloneInt(request.Execution.MaxTokens)
|
||||
execution.TopP = cloneFloat64(request.Execution.TopP)
|
||||
execution.TimeoutSeconds = cloneInt(request.Execution.TimeoutSeconds)
|
||||
execution.ReasoningEffort = cloneString(request.Execution.ReasoningEffort)
|
||||
var err error
|
||||
execution.ExtraParams, err = jsonvalue.CopyMap(request.Execution.ExtraParams)
|
||||
if err != nil {
|
||||
return SmokeTestRequest{}, err
|
||||
}
|
||||
cloned.Execution = &execution
|
||||
return cloned, nil
|
||||
}
|
||||
|
||||
// CloneSmokeTestReport returns a copy with independent optional facts and maps.
|
||||
func CloneSmokeTestReport(report SmokeTestReport) (SmokeTestReport, error) {
|
||||
cloned := report
|
||||
cloned.Applicable = cloneBool(report.Applicable)
|
||||
cloned.Target.PromptVersion = cloneString(report.Target.PromptVersion)
|
||||
cloned.Facts = cloneSmokeTestFacts(report.Facts)
|
||||
cloned.Timing = cloneSmokeTestTiming(report.Timing)
|
||||
var err error
|
||||
cloned.Target.EffectiveSettings.ExtraParams, err = jsonvalue.CopyMap(report.Target.EffectiveSettings.ExtraParams)
|
||||
if err != nil {
|
||||
return SmokeTestReport{}, err
|
||||
}
|
||||
if report.Attempts != nil {
|
||||
cloned.Attempts = make([]SmokeTestAttempt, len(report.Attempts))
|
||||
for index, attempt := range report.Attempts {
|
||||
cloned.Attempts[index] = attempt
|
||||
cloned.Attempts[index].HTTPStatus = cloneInt(attempt.HTTPStatus)
|
||||
cloned.Attempts[index].Facts = cloneSmokeTestFacts(attempt.Facts)
|
||||
cloned.Attempts[index].Timing = cloneSmokeTestTiming(attempt.Timing)
|
||||
}
|
||||
}
|
||||
return cloned, nil
|
||||
}
|
||||
|
||||
func cloneSmokeTestFacts(facts SmokeTestFacts) SmokeTestFacts {
|
||||
facts.CredentialsSupplied = cloneBool(facts.CredentialsSupplied)
|
||||
facts.EndpointReachable = cloneBool(facts.EndpointReachable)
|
||||
facts.CredentialsAccepted = cloneBool(facts.CredentialsAccepted)
|
||||
facts.ResponseReceived = cloneBool(facts.ResponseReceived)
|
||||
facts.GenerationCompleted = cloneBool(facts.GenerationCompleted)
|
||||
facts.TextReturned = cloneBool(facts.TextReturned)
|
||||
facts.ExpectedTextMatched = cloneBool(facts.ExpectedTextMatched)
|
||||
facts.StructuredOutputRequested = cloneBool(facts.StructuredOutputRequested)
|
||||
facts.ValidJSON = cloneBool(facts.ValidJSON)
|
||||
facts.SchemaMatched = cloneBool(facts.SchemaMatched)
|
||||
return facts
|
||||
}
|
||||
|
||||
func cloneSmokeTestTiming(timing SmokeTestTiming) SmokeTestTiming {
|
||||
timing.CapacityWait = cloneDuration(timing.CapacityWait)
|
||||
timing.RequestDuration = cloneDuration(timing.RequestDuration)
|
||||
timing.RetryDelay = cloneDuration(timing.RetryDelay)
|
||||
timing.EvaluationDuration = cloneDuration(timing.EvaluationDuration)
|
||||
return timing
|
||||
}
|
||||
|
||||
func cloneBool(value *bool) *bool {
|
||||
if value == nil {
|
||||
return nil
|
||||
}
|
||||
cloned := *value
|
||||
return &cloned
|
||||
}
|
||||
|
||||
func cloneDuration(value *time.Duration) *time.Duration {
|
||||
if value == nil {
|
||||
return nil
|
||||
}
|
||||
cloned := *value
|
||||
return &cloned
|
||||
}
|
||||
|
||||
func cloneFloat64(value *float64) *float64 {
|
||||
if value == nil {
|
||||
return nil
|
||||
}
|
||||
cloned := *value
|
||||
return &cloned
|
||||
}
|
||||
|
||||
func cloneInt(value *int) *int {
|
||||
if value == nil {
|
||||
return nil
|
||||
}
|
||||
cloned := *value
|
||||
return &cloned
|
||||
}
|
||||
|
||||
func cloneString(value *string) *string {
|
||||
if value == nil {
|
||||
return nil
|
||||
}
|
||||
cloned := *value
|
||||
return &cloned
|
||||
}
|
||||
115
internal/domain/smoke_diagnostics_test.go
Normal file
115
internal/domain/smoke_diagnostics_test.go
Normal file
@@ -0,0 +1,115 @@
|
||||
package domain
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestCloneBackendCopiesDiagnosticMetadataAndProviderValues(t *testing.T) {
|
||||
profile := "profile"
|
||||
reachabilityEndpoint := "https://probe.example/health"
|
||||
authKind := "openrouter_key"
|
||||
authEndpoint := "https://probe.example/key"
|
||||
backend := Backend{
|
||||
ExtraParams: map[string]any{"nested": map[string]any{"value": "original"}},
|
||||
SmokeTests: &BackendSmokeTestConfig{
|
||||
DefaultProfile: &profile,
|
||||
Reachability: &ReachabilityProbeConfig{Endpoint: &reachabilityEndpoint},
|
||||
Authentication: &AuthenticationProbeConfig{
|
||||
Kind: &authKind,
|
||||
Endpoint: &authEndpoint,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
cloned, err := CloneBackend(backend)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
*cloned.SmokeTests.DefaultProfile = "changed"
|
||||
*cloned.SmokeTests.Reachability.Endpoint = "changed"
|
||||
*cloned.SmokeTests.Authentication.Kind = "changed"
|
||||
*cloned.SmokeTests.Authentication.Endpoint = "changed"
|
||||
cloned.ExtraParams["nested"].(map[string]any)["value"] = "changed"
|
||||
|
||||
if profile != "profile" || reachabilityEndpoint != "https://probe.example/health" ||
|
||||
authKind != "openrouter_key" || authEndpoint != "https://probe.example/key" {
|
||||
t.Fatal("cloned diagnostic metadata shared source pointers")
|
||||
}
|
||||
if got := backend.ExtraParams["nested"].(map[string]any)["value"]; got != "original" {
|
||||
t.Fatalf("source provider parameter changed through clone: %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCloneSmokeTestRequestOwnsOptionalSettings(t *testing.T) {
|
||||
temperature := 0.25
|
||||
retries := 2
|
||||
reasoning := "low"
|
||||
request := SmokeTestRequest{
|
||||
APIKey: "secret",
|
||||
TransientRetries: &retries,
|
||||
Execution: &ExecutionTargetOverride{
|
||||
Temperature: &temperature,
|
||||
ReasoningEffort: &reasoning,
|
||||
ExtraParams: map[string]any{"nested": map[string]any{"value": "original"}},
|
||||
},
|
||||
}
|
||||
|
||||
cloned, err := CloneSmokeTestRequest(request)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
*cloned.TransientRetries = 3
|
||||
*cloned.Execution.Temperature = 0.5
|
||||
*cloned.Execution.ReasoningEffort = "high"
|
||||
cloned.Execution.ExtraParams["nested"].(map[string]any)["value"] = "changed"
|
||||
|
||||
if retries != 2 || temperature != 0.25 || reasoning != "low" {
|
||||
t.Fatal("cloned request shared optional scalar pointers")
|
||||
}
|
||||
if got := request.Execution.ExtraParams["nested"].(map[string]any)["value"]; got != "original" {
|
||||
t.Fatalf("source provider parameter changed through clone: %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCloneSmokeTestReportOwnsMutableValues(t *testing.T) {
|
||||
applicable := true
|
||||
reachable := true
|
||||
status := 200
|
||||
wait := 3 * time.Millisecond
|
||||
promptVersion := "v1"
|
||||
report := SmokeTestReport{
|
||||
Applicable: &applicable,
|
||||
Target: SmokeTestTarget{
|
||||
PromptVersion: &promptVersion,
|
||||
EffectiveSettings: ExecutionTarget{
|
||||
ExtraParams: map[string]any{"nested": map[string]any{"value": "original"}},
|
||||
},
|
||||
},
|
||||
Facts: SmokeTestFacts{EndpointReachable: &reachable},
|
||||
Attempts: []SmokeTestAttempt{{
|
||||
HTTPStatus: &status,
|
||||
Facts: SmokeTestFacts{EndpointReachable: &reachable},
|
||||
Timing: SmokeTestTiming{CapacityWait: &wait},
|
||||
}},
|
||||
}
|
||||
|
||||
cloned, err := CloneSmokeTestReport(report)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
*cloned.Applicable = false
|
||||
*cloned.Facts.EndpointReachable = false
|
||||
*cloned.Attempts[0].HTTPStatus = 503
|
||||
*cloned.Attempts[0].Facts.EndpointReachable = false
|
||||
*cloned.Attempts[0].Timing.CapacityWait = time.Second
|
||||
*cloned.Target.PromptVersion = "changed"
|
||||
cloned.Target.EffectiveSettings.ExtraParams["nested"].(map[string]any)["value"] = "changed"
|
||||
|
||||
if !applicable || !reachable || status != 200 || wait != 3*time.Millisecond || promptVersion != "v1" {
|
||||
t.Fatal("cloned report shared optional scalar pointers")
|
||||
}
|
||||
if got := report.Target.EffectiveSettings.ExtraParams["nested"].(map[string]any)["value"]; got != "original" {
|
||||
t.Fatalf("source provider parameter changed through clone: %v", got)
|
||||
}
|
||||
}
|
||||
@@ -11,49 +11,41 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
openrouter "gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v2"
|
||||
rakestrawhome "gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v2"
|
||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/artifact"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/backend"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/capacity"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/catalog"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/configuration"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/defaults"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/generation"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/llm"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/profile"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/prompt"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/promptdef"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/usecase"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/validate"
|
||||
)
|
||||
|
||||
// Config carries construction settings selected by the public facade.
|
||||
type Config struct {
|
||||
PromptDir string
|
||||
ProfileDir string
|
||||
Sources ConfigurationSources
|
||||
SchemaDir string
|
||||
Timeout time.Duration
|
||||
HTTPClient *http.Client
|
||||
TransientRetries *int
|
||||
}
|
||||
|
||||
// ConfigurationSources identifies caller-selected ConfigKit layers.
|
||||
type ConfigurationSources struct {
|
||||
Underlays []configkit.Underlay
|
||||
Main *configkit.Input
|
||||
}
|
||||
|
||||
// Options collects construction choices from the public facade.
|
||||
type Options struct {
|
||||
llmClient llm.Client
|
||||
artifactReader artifact.Reader
|
||||
promptDefinitions promptdef.Repository
|
||||
profiles profile.Repository
|
||||
fallbackProfiles profile.Repository
|
||||
memoryProfiles profile.Repository
|
||||
backends []domain.Backend
|
||||
validator validate.Validator
|
||||
promptSource bool
|
||||
profileSource bool
|
||||
fallbackProfileSource bool
|
||||
memorySource bool
|
||||
validatorSource bool
|
||||
artifactSource bool
|
||||
llmClient llm.Client
|
||||
artifactReader artifact.Reader
|
||||
validator validate.Validator
|
||||
validatorSource bool
|
||||
artifactSource bool
|
||||
}
|
||||
|
||||
// SetLLMClient selects the client collaborator for one engine.
|
||||
@@ -65,72 +57,6 @@ func (o *Options) SetArtifactReader(reader artifact.Reader) {
|
||||
o.artifactSource = true
|
||||
}
|
||||
|
||||
// SetPromptFS selects a prompt source rooted in fsys.
|
||||
func (o *Options) SetPromptFS(fsys fs.FS, root string) error {
|
||||
if err := validateSourceFS(fsys, root); err != nil {
|
||||
return err
|
||||
}
|
||||
o.promptDefinitions = promptdef.NewFSRepository(fsys, root)
|
||||
o.promptSource = true
|
||||
return nil
|
||||
}
|
||||
|
||||
// SetPromptFile selects a single prompt file.
|
||||
func (o *Options) SetPromptFile(path string) error {
|
||||
fsys, root, directory, err := fileSource(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
o.promptDefinitions = promptdef.NewFileRepository(fsys, root, directory)
|
||||
o.promptSource = true
|
||||
return nil
|
||||
}
|
||||
|
||||
// SetProfileFS selects an ordinary profile source rooted in fsys.
|
||||
func (o *Options) SetProfileFS(fsys fs.FS, root string) error {
|
||||
if err := validateSourceFS(fsys, root); err != nil {
|
||||
return err
|
||||
}
|
||||
o.profiles = profile.NewFSRepository(fsys, root)
|
||||
o.profileSource = true
|
||||
return nil
|
||||
}
|
||||
|
||||
// SetProfileFile selects a single ordinary profile file.
|
||||
func (o *Options) SetProfileFile(path string) error {
|
||||
fsys, root, _, err := fileSource(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
o.profiles = profile.NewFSRepository(fsys, root)
|
||||
o.profileSource = true
|
||||
return nil
|
||||
}
|
||||
|
||||
// SetFallbackProfileFS selects a fallback profile source rooted in fsys.
|
||||
func (o *Options) SetFallbackProfileFS(fsys fs.FS, root string) error {
|
||||
if err := validateSourceFS(fsys, root); err != nil {
|
||||
return err
|
||||
}
|
||||
o.fallbackProfiles = profile.NewFSRepository(fsys, root)
|
||||
o.fallbackProfileSource = true
|
||||
return nil
|
||||
}
|
||||
|
||||
// SetMemoryProfiles validates and stores application-owned profile definitions.
|
||||
func (o *Options) SetMemoryProfiles(definitions []domain.ExecutionProfile) error {
|
||||
repository, err := profile.NewMemoryRepository(definitions)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
o.memoryProfiles = repository
|
||||
o.memorySource = true
|
||||
return nil
|
||||
}
|
||||
|
||||
// AddBackend appends one backend registration.
|
||||
func (o *Options) AddBackend(backend domain.Backend) { o.backends = append(o.backends, backend) }
|
||||
|
||||
// SetSchemaFS selects a schema source rooted in fsys.
|
||||
func (o *Options) SetSchemaFS(fsys fs.FS, root string) error {
|
||||
if err := validateSourceFS(fsys, root); err != nil {
|
||||
@@ -154,7 +80,41 @@ func (o *Options) SetSchemaFile(path string) error {
|
||||
|
||||
// Engine forwards domain operations to the single assembled use-case runner.
|
||||
type Engine struct {
|
||||
runner *usecase.Runner
|
||||
runner *usecase.Runner
|
||||
smokeTester *usecase.SmokeTester
|
||||
inspection configuration.Inspection
|
||||
listing identityInventory
|
||||
namespaces resourceNamespaces
|
||||
}
|
||||
|
||||
type resourceNamespaces struct {
|
||||
prompt string
|
||||
profile string
|
||||
backend string
|
||||
}
|
||||
|
||||
type identityInventory interface {
|
||||
ListBackends(context.Context) ([]string, error)
|
||||
ListProfiles(context.Context) ([]string, error)
|
||||
ListPrompts(context.Context) ([]domain.PromptIdentity, error)
|
||||
ListNamespaces(context.Context, []string) ([]string, error)
|
||||
ListCollections(context.Context) ([]configuration.CollectionIdentity, error)
|
||||
}
|
||||
|
||||
// ListingFilterMode selects namespace membership for one identity listing.
|
||||
type ListingFilterMode uint8
|
||||
|
||||
const (
|
||||
ListingDefault ListingFilterMode = iota
|
||||
ListingNamed
|
||||
ListingUnqualified
|
||||
ListingAll
|
||||
)
|
||||
|
||||
// ListingFilter optionally overrides the namespace selected by an engine view.
|
||||
type ListingFilter struct {
|
||||
Mode ListingFilterMode
|
||||
Name string
|
||||
}
|
||||
|
||||
// New assembles an engine from facade-selected configuration and collaborators.
|
||||
@@ -163,33 +123,16 @@ func New(cfg Config, options Options) (*Engine, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
promptDefinitions := options.promptDefinitions
|
||||
if !options.promptSource {
|
||||
if strings.TrimSpace(cfg.PromptDir) == "" {
|
||||
return nil, fmt.Errorf("prompt directory is required")
|
||||
}
|
||||
promptDefinitions = promptdef.NewFilesystemRepository(cfg.PromptDir)
|
||||
underlays := cfg.Sources.Underlays
|
||||
main := configkit.Input{Source: configkit.NoSource()}
|
||||
if cfg.Sources.Main != nil {
|
||||
main = *cfg.Sources.Main
|
||||
}
|
||||
|
||||
maintained, err := catalog.Load(
|
||||
catalog.Source{Name: "OpenRouter", ExpectedBackendID: backend.OpenRouterID, FS: openrouter.FS(), Root: openrouter.Root},
|
||||
catalog.Source{Name: "Rakestrawhome", ExpectedBackendID: backend.RakestrawHomeID, FS: rakestrawhome.FS(), Root: rakestrawhome.Root},
|
||||
)
|
||||
snapshot, err := configuration.Load(main, underlays)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to load maintained catalogs: %w", err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
maintainedProfiles, err := newMaintainedProfileRepository(maintained.Profiles, maintained.Backends)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to construct Promptkit-maintained profiles: %w", err)
|
||||
}
|
||||
|
||||
profiles := newProfileRepository(cfg.ProfileDir, options, maintainedProfiles)
|
||||
registry, err := backend.NewRegistry(maintained.Backends, options.backends)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to construct backend registry: %w", err)
|
||||
}
|
||||
manager, err := capacity.NewManager(registry.CapacityPolicies())
|
||||
manager, err := capacity.NewManager(snapshot.CapacityPolicies())
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to construct backend capacity manager: %w", err)
|
||||
}
|
||||
@@ -203,11 +146,13 @@ func New(cfg Config, options Options) (*Engine, error) {
|
||||
validator = validate.NewStandardValidator(schemaDir)
|
||||
}
|
||||
var client llm.Client
|
||||
var diagnosticHTTPClient *llm.OpenAICompatibleClient
|
||||
if options.llmClient == nil {
|
||||
builtIn, newErr := llm.NewOpenAICompatibleClient(llm.OpenAICompatibleConfig{Timeout: cfg.Timeout, HTTPClient: cfg.HTTPClient})
|
||||
if newErr != nil {
|
||||
return nil, newErr
|
||||
}
|
||||
diagnosticHTTPClient = builtIn
|
||||
client = generation.NewBuiltIn(manager, builtIn)
|
||||
} else {
|
||||
client = generation.NewInjected(manager, options.llmClient)
|
||||
@@ -217,23 +162,31 @@ func New(cfg Config, options Options) (*Engine, error) {
|
||||
reader = artifact.NewCompositeReader()
|
||||
}
|
||||
|
||||
return &Engine{runner: usecase.NewRunnerWithRepairer(promptDefinitions, profiles, registry, reader, prompt.NewGoRenderer(), client, validator, usecase.NewDefaultOutputRepairer(client), manager, defaultRetries)}, nil
|
||||
runner := usecase.NewRunnerWithRepairer(snapshot, snapshot, reader, prompt.NewCompiledRenderer(snapshot), client, validator, usecase.NewDefaultOutputRepairer(client), manager, defaultRetries)
|
||||
return &Engine{
|
||||
runner: runner,
|
||||
smokeTester: usecase.NewSmokeTester(runner, diagnosticHTTPClient),
|
||||
inspection: snapshot.Inspection(),
|
||||
listing: snapshot,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func newProfileRepository(profileDir string, options Options, maintained profile.Repository) profile.Repository {
|
||||
repository := maintained
|
||||
if options.fallbackProfileSource {
|
||||
repository = profile.NewOverlayRepository(options.fallbackProfiles, repository)
|
||||
// WithNamespaces returns an immutable view sharing this engine's assembled
|
||||
// collaborators and configuration snapshot. Caller-supplied selection IDs are
|
||||
// qualified at each operation boundary; captured configuration references are
|
||||
// not rewritten.
|
||||
func (e *Engine) WithNamespaces(prompt, profile, backend string) *Engine {
|
||||
view := *e
|
||||
view.namespaces = resourceNamespaces{prompt: prompt, profile: profile, backend: backend}
|
||||
return &view
|
||||
}
|
||||
|
||||
// InspectConfiguration returns read-only operations over the captured result.
|
||||
func (e *Engine) InspectConfiguration() configuration.Inspection {
|
||||
if e == nil {
|
||||
return nil
|
||||
}
|
||||
if options.profileSource {
|
||||
repository = profile.NewOverlayRepository(options.profiles, repository)
|
||||
} else if strings.TrimSpace(profileDir) != "" {
|
||||
repository = profile.NewOverlayRepository(profile.NewFilesystemRepository(profileDir), repository)
|
||||
}
|
||||
if options.memorySource {
|
||||
repository = profile.NewOverlayRepository(options.memoryProfiles, repository)
|
||||
}
|
||||
return profile.NewResolvingRepository(repository)
|
||||
return e.inspection
|
||||
}
|
||||
|
||||
func validateSourceFS(fsys fs.FS, root string) error {
|
||||
@@ -265,21 +218,213 @@ func fileSource(name string) (fs.FS, string, string, error) {
|
||||
return os.DirFS(directory), filepath.ToSlash(base), directory, nil
|
||||
}
|
||||
|
||||
func (e *Engine) InspectPrompt(ctx context.Context, id, version string) (*domain.PromptInspection, error) {
|
||||
return e.runner.InspectPrompt(ctx, id, version)
|
||||
func (e *Engine) InspectPrompt(ctx context.Context, id string) (*domain.PromptInspection, error) {
|
||||
var err error
|
||||
id, err = e.qualifyResourceID(id, e.namespaces.prompt, "prompt")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return e.runner.InspectPrompt(ctx, id)
|
||||
}
|
||||
func (e *Engine) InspectRequirements(
|
||||
ctx context.Context,
|
||||
request usecase.RequirementInspectionRequest,
|
||||
) (*domain.RequirementInspection, error) {
|
||||
var err error
|
||||
request.PromptID, err = e.qualifyResourceID(request.PromptID, e.namespaces.prompt, "prompt")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
request.ProfileID, err = e.qualifyResourceID(request.ProfileID, e.namespaces.profile, "profile")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return e.runner.InspectRequirements(ctx, request)
|
||||
}
|
||||
func (e *Engine) InspectProfile(ctx context.Context, id string) (*domain.ProfileInspection, error) {
|
||||
var err error
|
||||
id, err = e.qualifyResourceID(id, e.namespaces.profile, "profile")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return e.runner.InspectProfile(ctx, id)
|
||||
}
|
||||
|
||||
func (e *Engine) ListBackends(ctx context.Context, filters ...ListingFilter) ([]string, error) {
|
||||
return filterIdentityList(ctx, filters, e.namespaces.backend, e.listing.ListBackends)
|
||||
}
|
||||
|
||||
func (e *Engine) ListProfiles(ctx context.Context, filters ...ListingFilter) ([]string, error) {
|
||||
return filterIdentityList(ctx, filters, e.namespaces.profile, e.listing.ListProfiles)
|
||||
}
|
||||
|
||||
func (e *Engine) ListPrompts(ctx context.Context, filters ...ListingFilter) ([]domain.PromptIdentity, error) {
|
||||
mode, namespace, err := resolveListingFilter(filters, e.namespaces.prompt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
prompts, err := e.listing.ListPrompts(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var filtered []domain.PromptIdentity
|
||||
for _, prompt := range prompts {
|
||||
if matchesListingNamespace(prompt.ID, mode, namespace) {
|
||||
filtered = append(filtered, prompt)
|
||||
}
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return filtered, nil
|
||||
}
|
||||
|
||||
func (e *Engine) ListNamespaces(ctx context.Context, kinds []string) ([]string, error) {
|
||||
for _, kind := range kinds {
|
||||
switch kind {
|
||||
case "prompt", "profile", "backend":
|
||||
default:
|
||||
return nil, fmt.Errorf("%w: unknown resource kind %q", usecase.ErrInvalidRequest, kind)
|
||||
}
|
||||
}
|
||||
return e.listing.ListNamespaces(ctx, kinds)
|
||||
}
|
||||
|
||||
func (e *Engine) ListCollections(ctx context.Context) ([]configuration.CollectionIdentity, error) {
|
||||
return e.listing.ListCollections(ctx)
|
||||
}
|
||||
|
||||
func filterIdentityList(
|
||||
ctx context.Context,
|
||||
filters []ListingFilter,
|
||||
scope string,
|
||||
list func(context.Context) ([]string, error),
|
||||
) ([]string, error) {
|
||||
mode, namespace, err := resolveListingFilter(filters, scope)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ids, err := list(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var filtered []string
|
||||
for _, id := range ids {
|
||||
if matchesListingNamespace(id, mode, namespace) {
|
||||
filtered = append(filtered, id)
|
||||
}
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return filtered, nil
|
||||
}
|
||||
|
||||
func resolveListingFilter(filters []ListingFilter, scope string) (ListingFilterMode, string, error) {
|
||||
if len(filters) > 1 {
|
||||
return 0, "", fmt.Errorf("%w: at most one listing filter is allowed", usecase.ErrInvalidRequest)
|
||||
}
|
||||
filter := ListingFilter{}
|
||||
if len(filters) == 1 {
|
||||
filter = filters[0]
|
||||
}
|
||||
switch filter.Mode {
|
||||
case ListingDefault:
|
||||
if scope == "" {
|
||||
return ListingAll, "", nil
|
||||
}
|
||||
return ListingNamed, scope, nil
|
||||
case ListingNamed:
|
||||
if !domain.IsValidResourceNamespace(filter.Name) {
|
||||
return 0, "", fmt.Errorf("%w: invalid listing namespace", usecase.ErrInvalidRequest)
|
||||
}
|
||||
return ListingNamed, filter.Name, nil
|
||||
case ListingUnqualified, ListingAll:
|
||||
return filter.Mode, "", nil
|
||||
default:
|
||||
return 0, "", fmt.Errorf("%w: unknown listing filter mode", usecase.ErrInvalidRequest)
|
||||
}
|
||||
}
|
||||
|
||||
func matchesListingNamespace(id string, mode ListingFilterMode, namespace string) bool {
|
||||
switch mode {
|
||||
case ListingNamed:
|
||||
idNamespace, _, qualified := strings.Cut(id, "/")
|
||||
return qualified && idNamespace == namespace
|
||||
case ListingUnqualified:
|
||||
return !strings.Contains(id, "/")
|
||||
case ListingAll:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
func (e *Engine) Prepare(ctx context.Context, request domain.RunRequest) (*domain.PreparedRun, error) {
|
||||
request, err := e.qualifyRunRequest(request)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return e.runner.Prepare(ctx, request)
|
||||
}
|
||||
func (e *Engine) PrepareExecution(ctx context.Context, request domain.RunRequest) (*usecase.PreparedExecution, error) {
|
||||
request, err := e.qualifyRunRequest(request)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return e.runner.PrepareExecution(ctx, request)
|
||||
}
|
||||
func (e *Engine) Run(ctx context.Context, request domain.RunRequest) (*domain.RunResult, error) {
|
||||
request, err := e.qualifyRunRequest(request)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return e.runner.Run(ctx, request)
|
||||
}
|
||||
|
||||
func (e *Engine) qualifyRunRequest(request domain.RunRequest) (domain.RunRequest, error) {
|
||||
var err error
|
||||
request.PromptID, err = e.qualifyResourceID(request.PromptID, e.namespaces.prompt, "prompt")
|
||||
if err != nil {
|
||||
return domain.RunRequest{}, err
|
||||
}
|
||||
request.ProfileID, err = e.qualifyResourceID(request.ProfileID, e.namespaces.profile, "profile")
|
||||
if err != nil {
|
||||
return domain.RunRequest{}, err
|
||||
}
|
||||
return request, nil
|
||||
}
|
||||
|
||||
func (e *Engine) qualifyResourceID(id, namespace, kind string) (string, error) {
|
||||
if id == "" {
|
||||
return "", nil
|
||||
}
|
||||
qualified, ok := domain.QualifyResourceID(id, namespace)
|
||||
if !ok {
|
||||
return "", fmt.Errorf("%w: invalid %s ID", usecase.ErrInvalidRequest, kind)
|
||||
}
|
||||
return qualified, nil
|
||||
}
|
||||
|
||||
func (e *Engine) RunPrepared(ctx context.Context, prepared *usecase.PreparedExecution) (*domain.RunResult, error) {
|
||||
return e.runner.RunPrepared(ctx, prepared)
|
||||
}
|
||||
|
||||
func (e *Engine) SmokeTest(ctx context.Context, request domain.SmokeTestRequest) (*domain.SmokeTestReport, error) {
|
||||
if e == nil || e.smokeTester == nil {
|
||||
return nil, fmt.Errorf("%w: smoke tester is not configured", usecase.ErrInvalidRequest)
|
||||
}
|
||||
var err error
|
||||
request.PromptID, err = e.qualifyResourceID(request.PromptID, e.namespaces.prompt, "prompt")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
request.ProfileID, err = e.qualifyResourceID(request.ProfileID, e.namespaces.profile, "profile")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
request.BackendID, err = e.qualifyResourceID(request.BackendID, e.namespaces.backend, "backend")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return e.smokeTester.Run(ctx, request)
|
||||
}
|
||||
|
||||
@@ -1,50 +0,0 @@
|
||||
package engine
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/backend"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/profile"
|
||||
)
|
||||
|
||||
func maintainedProfileDefinitions() []domain.ExecutionProfile {
|
||||
return []domain.ExecutionProfile{
|
||||
{
|
||||
ID: "promptkit-openrouter-standard",
|
||||
BaseProfileID: "gemini-flash-latest",
|
||||
},
|
||||
{
|
||||
ID: "promptkit-openrouter-auto",
|
||||
BackendID: backend.OpenRouterID,
|
||||
Model: "openrouter/auto",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func newMaintainedProfileRepository(catalogProfiles profile.Repository, maintainedBackends []domain.Backend) (profile.Repository, error) {
|
||||
definitions := maintainedProfileDefinitions()
|
||||
owned, err := profile.NewMemoryRepository(definitions)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create Promptkit-maintained profiles: %w", err)
|
||||
}
|
||||
|
||||
combined := profile.NewOverlayRepository(owned, catalogProfiles)
|
||||
resolved := profile.NewResolvingRepository(combined)
|
||||
registry, err := backend.NewRegistry(maintainedBackends, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("construct maintained backend registry: %w", err)
|
||||
}
|
||||
for _, definition := range definitions {
|
||||
effective, err := resolved.GetProfile(context.Background(), definition.ID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("resolve Promptkit-maintained profile %q: %w", definition.ID, err)
|
||||
}
|
||||
if _, err := registry.GetBackend(effective.BackendID); err != nil {
|
||||
return nil, fmt.Errorf("Promptkit-maintained profile %q selects unavailable backend %q: %w", definition.ID, effective.BackendID, err)
|
||||
}
|
||||
}
|
||||
|
||||
return combined, nil
|
||||
}
|
||||
@@ -1,78 +0,0 @@
|
||||
package engine
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/backend"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/profile"
|
||||
)
|
||||
|
||||
func TestNewMaintainedProfileRepository(t *testing.T) {
|
||||
newCatalogProfiles := func(t *testing.T, definitions []domain.ExecutionProfile) profile.Repository {
|
||||
t.Helper()
|
||||
repository, err := profile.NewMemoryRepository(definitions)
|
||||
if err != nil {
|
||||
t.Fatalf("create catalog profiles: %v", err)
|
||||
}
|
||||
return repository
|
||||
}
|
||||
newBackend := func(id string) domain.Backend {
|
||||
return domain.Backend{ID: id, Endpoint: "https://" + id + ".example/v1", APIKeyEnv: "TEST_API_KEY"}
|
||||
}
|
||||
|
||||
t.Run("resolves inherited and direct definitions", func(t *testing.T) {
|
||||
repository, err := newMaintainedProfileRepository(
|
||||
newCatalogProfiles(t, []domain.ExecutionProfile{{
|
||||
ID: "gemini-flash-latest",
|
||||
BackendID: backend.OpenRouterID,
|
||||
Model: "catalog-selected-model",
|
||||
}}),
|
||||
[]domain.Backend{newBackend(backend.OpenRouterID)},
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("create maintained profiles: %v", err)
|
||||
}
|
||||
|
||||
resolved := profile.NewResolvingRepository(repository)
|
||||
standard, err := resolved.GetProfile(context.Background(), "promptkit-openrouter-standard")
|
||||
if err != nil {
|
||||
t.Fatalf("resolve inherited profile: %v", err)
|
||||
}
|
||||
if standard.BackendID != backend.OpenRouterID || standard.Model != "catalog-selected-model" {
|
||||
t.Fatalf("unexpected inherited profile: %#v", standard)
|
||||
}
|
||||
|
||||
auto, err := resolved.GetProfile(context.Background(), "promptkit-openrouter-auto")
|
||||
if err != nil {
|
||||
t.Fatalf("resolve direct profile: %v", err)
|
||||
}
|
||||
definitions := maintainedProfileDefinitions()
|
||||
if auto.BackendID != definitions[1].BackendID || auto.Model != definitions[1].Model {
|
||||
t.Fatalf("unexpected direct profile: %#v", auto)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("rejects a missing inherited base", func(t *testing.T) {
|
||||
if _, err := newMaintainedProfileRepository(
|
||||
newCatalogProfiles(t, nil),
|
||||
[]domain.Backend{newBackend(backend.OpenRouterID)},
|
||||
); err == nil {
|
||||
t.Fatal("expected missing base error")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("rejects an unavailable effective backend", func(t *testing.T) {
|
||||
if _, err := newMaintainedProfileRepository(
|
||||
newCatalogProfiles(t, []domain.ExecutionProfile{{
|
||||
ID: "gemini-flash-latest",
|
||||
BackendID: "other",
|
||||
Model: "catalog-selected-model",
|
||||
}}),
|
||||
[]domain.Backend{newBackend("other")},
|
||||
); err == nil {
|
||||
t.Fatal("expected unavailable backend error")
|
||||
}
|
||||
})
|
||||
}
|
||||
124
internal/engine/smoke_test.go
Normal file
124
internal/engine/smoke_test.go
Normal file
@@ -0,0 +1,124 @@
|
||||
package engine
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
)
|
||||
|
||||
func engineSmokeBackendInput() configkit.Input {
|
||||
return configkit.Input{Source: configkit.Mapping("smoke-test backend", map[string]any{
|
||||
"backends": map[string]any{"test-backend": map[string]any{
|
||||
"endpoint": "https://test-backend.example/v1",
|
||||
"smoke_tests": map[string]any{
|
||||
"default_profile": "test-profile",
|
||||
"authentication": map[string]any{"kind": "none"},
|
||||
},
|
||||
}},
|
||||
"profiles": map[string]any{"test-profile": map[string]any{
|
||||
"backend": "test-backend",
|
||||
"model": "synthetic-model",
|
||||
}},
|
||||
})}
|
||||
}
|
||||
|
||||
func TestEngineSmokeTestUsesInjectedGenerationWithoutHTTPSupport(t *testing.T) {
|
||||
calls := 0
|
||||
options := Options{}
|
||||
options.SetLLMClient(engineSmokeClientFunc(func(_ context.Context, request domain.GenerateRequest) (*domain.GenerateResponse, error) {
|
||||
calls++
|
||||
if len(request.Prompt.Messages) != 1 || request.Prompt.Messages[0].Role != "user" {
|
||||
t.Fatalf("injected generation request = %+v", request)
|
||||
}
|
||||
return &domain.GenerateResponse{Content: "engine response"}, nil
|
||||
}))
|
||||
input := engineSmokeBackendInput()
|
||||
assembled, err := New(Config{Sources: ConfigurationSources{Main: &input}}, options)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
report, err := assembled.SmokeTest(context.Background(), domain.SmokeTestRequest{
|
||||
Kind: domain.SmokeTestTextInference, BackendID: "test-backend",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if calls != 1 || !report.Completed || !report.Supported || report.Applicable == nil || !*report.Applicable ||
|
||||
report.Facts.GenerationCompleted == nil || !*report.Facts.GenerationCompleted ||
|
||||
report.Facts.TextReturned == nil || !*report.Facts.TextReturned ||
|
||||
report.Facts.ExpectedTextMatched == nil || *report.Facts.ExpectedTextMatched ||
|
||||
report.Facts.EndpointReachable != nil || report.Target.BackendID != "test-backend" {
|
||||
t.Fatalf("injected smoke report calls=%d: %+v", calls, report)
|
||||
}
|
||||
if report.Target.Endpoint != "" || report.Target.EffectiveSettings.Endpoint == "" || len(report.Attempts) != 1 ||
|
||||
report.Attempts[0].InitialEndpoint != "" || report.Attempts[0].Endpoint != "" {
|
||||
t.Fatalf("injected report claimed HTTP destinations: %+v", report)
|
||||
}
|
||||
|
||||
authentication, err := assembled.SmokeTest(context.Background(), domain.SmokeTestRequest{
|
||||
Kind: domain.SmokeTestAuthentication, BackendID: "test-backend",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !authentication.Supported || authentication.Applicable == nil || *authentication.Applicable ||
|
||||
authentication.Reason != domain.SmokeTestReasonAuthenticationNotApplicable || len(authentication.Attempts) != 0 {
|
||||
t.Fatalf("injected authentication report = %+v", authentication)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEngineSmokeTestUsesBuiltInHTTPClientForInferenceEvidence(t *testing.T) {
|
||||
var observedURL string
|
||||
var authorization string
|
||||
options := Options{}
|
||||
input := engineSmokeBackendInput()
|
||||
assembled, err := New(Config{Sources: ConfigurationSources{Main: &input}, HTTPClient: &http.Client{Transport: engineRoundTripperFunc(func(request *http.Request) (*http.Response, error) {
|
||||
observedURL = request.URL.String()
|
||||
authorization = request.Header.Get("Authorization")
|
||||
return &http.Response{
|
||||
StatusCode: http.StatusOK,
|
||||
Header: make(http.Header),
|
||||
Body: io.NopCloser(strings.NewReader(`{"choices":[{"message":{"content":"engine response"}}],"usage":{"prompt_tokens":1,"completion_tokens":1,"total_tokens":2}}`)),
|
||||
}, nil
|
||||
})}}, options)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
report, err := assembled.SmokeTest(context.Background(), domain.SmokeTestRequest{
|
||||
Kind: domain.SmokeTestTextInference, BackendID: "test-backend",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if observedURL == "" || authorization != "" {
|
||||
t.Fatalf("built-in request URL=%q authorization=%q", observedURL, authorization)
|
||||
}
|
||||
if !report.Completed || report.Facts.EndpointReachable == nil || !*report.Facts.EndpointReachable ||
|
||||
report.Facts.ResponseReceived == nil || !*report.Facts.ResponseReceived ||
|
||||
report.Facts.GenerationCompleted == nil || !*report.Facts.GenerationCompleted ||
|
||||
report.Facts.CredentialsSupplied == nil || *report.Facts.CredentialsSupplied ||
|
||||
report.Facts.TextReturned == nil || !*report.Facts.TextReturned ||
|
||||
report.Facts.ExpectedTextMatched == nil || *report.Facts.ExpectedTextMatched || len(report.Attempts) != 1 ||
|
||||
report.Attempts[0].Endpoint != observedURL {
|
||||
t.Fatalf("built-in smoke report = %+v", report)
|
||||
}
|
||||
}
|
||||
|
||||
type engineSmokeClientFunc func(context.Context, domain.GenerateRequest) (*domain.GenerateResponse, error)
|
||||
|
||||
func (f engineSmokeClientFunc) Generate(ctx context.Context, request domain.GenerateRequest) (*domain.GenerateResponse, error) {
|
||||
return f(ctx, request)
|
||||
}
|
||||
|
||||
type engineRoundTripperFunc func(*http.Request) (*http.Response, error)
|
||||
|
||||
func (f engineRoundTripperFunc) RoundTrip(request *http.Request) (*http.Response, error) {
|
||||
return f(request)
|
||||
}
|
||||
@@ -72,14 +72,43 @@ func (e *ExhaustedError) Attempts() uint64 {
|
||||
// Generate executes either one opaque client call or a frozen built-in HTTP
|
||||
// sequence with one permit per actual dispatch.
|
||||
func (c *Coordinator) Generate(ctx context.Context, req domain.GenerateRequest) (*domain.GenerateResponse, error) {
|
||||
response, err, _ := c.generate(ctx, req, false)
|
||||
return response, err
|
||||
}
|
||||
|
||||
// GenerateWithEvidence runs the built-in retry path and returns its factual
|
||||
// attempt evidence alongside any response retained before cancellation.
|
||||
func (c *Coordinator) GenerateWithEvidence(
|
||||
ctx context.Context,
|
||||
req domain.GenerateRequest,
|
||||
) (*domain.GenerateResponse, error, GenerationEvidence) {
|
||||
return c.generate(ctx, req, true)
|
||||
}
|
||||
|
||||
func (c *Coordinator) generate(
|
||||
ctx context.Context,
|
||||
req domain.GenerateRequest,
|
||||
captureEvidence bool,
|
||||
) (*domain.GenerateResponse, error, GenerationEvidence) {
|
||||
if c.injected != nil {
|
||||
return c.generateInjected(ctx, req)
|
||||
return c.generateInjected(ctx, req, captureEvidence)
|
||||
}
|
||||
prepared, err := c.builtIn.PrepareGeneration(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, err, GenerationEvidence{}
|
||||
}
|
||||
|
||||
evidence := GenerationEvidence{}
|
||||
if captureEvidence {
|
||||
evidence.InitialEndpoint = prepared.InitialEndpoint()
|
||||
}
|
||||
var usage observation.UsageAccumulator
|
||||
finish := func(response *domain.GenerateResponse, err error) (*domain.GenerateResponse, error, GenerationEvidence) {
|
||||
if captureEvidence {
|
||||
evidence.Usage = usage.Observed()
|
||||
}
|
||||
return response, err, evidence
|
||||
}
|
||||
var attempts, retriesUsed uint64
|
||||
recorder := observation.FromContext(ctx)
|
||||
generation := req.Generation
|
||||
@@ -88,56 +117,102 @@ func (c *Coordinator) Generate(ctx context.Context, req domain.GenerateRequest)
|
||||
}
|
||||
for {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
return finish(nil, err)
|
||||
}
|
||||
ordinal := attempts + 1
|
||||
outcome, dispatched, capacityWait, generationDuration := c.execute(ctx, req.Target.BackendID, prepared)
|
||||
outcome, dispatched, capacityWait, generationDuration, startedAt := c.execute(ctx, req.Target.BackendID, prepared, captureEvidence)
|
||||
if dispatched {
|
||||
attempts++
|
||||
}
|
||||
if captureEvidence {
|
||||
attempt := AttemptEvidence{
|
||||
StartedAt: startedAt.UTC(),
|
||||
Dispatched: dispatched,
|
||||
ResponseReceived: outcome.Evidence.ResponseReceived,
|
||||
HTTP: cloneHTTPEvidence(outcome.Evidence),
|
||||
CapacityWait: capacityWait,
|
||||
RequestDuration: generationDuration,
|
||||
Failure: ClassifyFailure(outcome.Err),
|
||||
ResponseDecoded: outcome.ResponseDecoded,
|
||||
GenerationCompleted: outcome.GenerationCompleted,
|
||||
}
|
||||
if outcome.StructuredOutputRequested != nil {
|
||||
requested := *outcome.StructuredOutputRequested
|
||||
attempt.StructuredOutputRequested = &requested
|
||||
}
|
||||
if dispatched {
|
||||
usage.Add(outcome.Usage, outcome.UsagePresence)
|
||||
var attemptUsage observation.UsageAccumulator
|
||||
attemptUsage.Add(outcome.Usage, outcome.UsagePresence)
|
||||
attempt.Usage = attemptUsage.Observed()
|
||||
}
|
||||
evidence.Attempts = append(evidence.Attempts, attempt)
|
||||
}
|
||||
if recorder != nil {
|
||||
recorder.Attempt(observation.AttemptData{Generation: generation, Attempt: ordinal,
|
||||
Dispatched: dispatched, CapacityWait: capacityWait, GenerationDuration: generationDuration,
|
||||
Outcome: attemptOutcome(outcome.Err), Failure: attemptFailure(outcome.Err),
|
||||
Outcome: attemptOutcome(outcome.Err), Failure: ClassifyFailure(outcome.Err),
|
||||
StatusCode: outcome.StatusCode, Usage: outcome.Usage, UsagePresence: outcome.UsagePresence})
|
||||
}
|
||||
if outcome.Err == nil {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
if captureEvidence {
|
||||
return finish(outcome.Response, err)
|
||||
}
|
||||
return finish(nil, err)
|
||||
}
|
||||
return outcome.Response, nil
|
||||
return finish(outcome.Response, nil)
|
||||
}
|
||||
if !dispatched || !eligibleStatus(outcome) {
|
||||
return nil, outcome.Err
|
||||
if captureEvidence {
|
||||
return finish(outcome.Response, outcome.Err)
|
||||
}
|
||||
return finish(nil, outcome.Err)
|
||||
}
|
||||
if retriesUsed == req.TransientRetries {
|
||||
if retriesUsed == 0 {
|
||||
return nil, outcome.Err
|
||||
return finish(nil, outcome.Err)
|
||||
}
|
||||
return nil, &ExhaustedError{attempts: attempts, cause: outcome.Err}
|
||||
return finish(nil, &ExhaustedError{attempts: attempts, cause: outcome.Err})
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
return finish(nil, err)
|
||||
}
|
||||
retryOrdinal := retriesUsed + 1
|
||||
delay := retryDelay(c.backoff(retryOrdinal), outcome.RetryNotBefore, c.now())
|
||||
waitStart := c.now()
|
||||
waitErr := c.wait(ctx, delay)
|
||||
waitDuration := elapsed(waitStart, c.now())
|
||||
if captureEvidence {
|
||||
evidence.RetryDelays = append(evidence.RetryDelays, waitDuration)
|
||||
}
|
||||
if recorder != nil {
|
||||
recorder.Delay(observation.DelayData{Generation: generation, Attempt: attempts,
|
||||
Elapsed: elapsed(waitStart, c.now()), Failure: attemptFailure(waitErr)})
|
||||
Elapsed: waitDuration, Failure: ClassifyFailure(waitErr)})
|
||||
}
|
||||
if waitErr != nil {
|
||||
return nil, waitErr
|
||||
return finish(nil, waitErr)
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
return finish(nil, err)
|
||||
}
|
||||
retriesUsed++
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Coordinator) generateInjected(ctx context.Context, req domain.GenerateRequest) (*domain.GenerateResponse, error) {
|
||||
func (c *Coordinator) generateInjected(
|
||||
ctx context.Context,
|
||||
req domain.GenerateRequest,
|
||||
captureEvidence bool,
|
||||
) (*domain.GenerateResponse, error, GenerationEvidence) {
|
||||
evidence := GenerationEvidence{}
|
||||
var usage observation.UsageAccumulator
|
||||
finish := func(response *domain.GenerateResponse, err error) (*domain.GenerateResponse, error, GenerationEvidence) {
|
||||
if captureEvidence {
|
||||
evidence.Usage = usage.Observed()
|
||||
}
|
||||
return response, err, evidence
|
||||
}
|
||||
recorder := observation.FromContext(ctx)
|
||||
generation := req.Generation
|
||||
if generation == 0 {
|
||||
@@ -147,18 +222,32 @@ func (c *Coordinator) generateInjected(ctx context.Context, req domain.GenerateR
|
||||
release, err := c.manager.Acquire(ctx, req.Target.BackendID)
|
||||
capacityWait := c.capacityWait(req.Target.BackendID, start)
|
||||
if err != nil {
|
||||
if recorder != nil {
|
||||
recorder.Attempt(observation.AttemptData{Generation: generation, Attempt: 1, CapacityWait: capacityWait, Outcome: domain.ExecutionOutcomeError, Failure: attemptFailure(err)})
|
||||
if captureEvidence {
|
||||
evidence.Attempts = append(evidence.Attempts, AttemptEvidence{
|
||||
StartedAt: start.UTC(),
|
||||
CapacityWait: capacityWait,
|
||||
Failure: ClassifyFailure(err),
|
||||
})
|
||||
}
|
||||
return nil, err
|
||||
if recorder != nil {
|
||||
recorder.Attempt(observation.AttemptData{Generation: generation, Attempt: 1, CapacityWait: capacityWait, Outcome: domain.ExecutionOutcomeError, Failure: ClassifyFailure(err)})
|
||||
}
|
||||
return finish(nil, err)
|
||||
}
|
||||
defer release()
|
||||
if err := ctx.Err(); err != nil {
|
||||
release()
|
||||
if recorder != nil {
|
||||
recorder.Attempt(observation.AttemptData{Generation: generation, Attempt: 1, CapacityWait: capacityWait, Outcome: domain.ExecutionOutcomeError, Failure: attemptFailure(err)})
|
||||
if captureEvidence {
|
||||
evidence.Attempts = append(evidence.Attempts, AttemptEvidence{
|
||||
StartedAt: start.UTC(),
|
||||
CapacityWait: capacityWait,
|
||||
Failure: ClassifyFailure(err),
|
||||
})
|
||||
}
|
||||
return nil, err
|
||||
if recorder != nil {
|
||||
recorder.Attempt(observation.AttemptData{Generation: generation, Attempt: 1, CapacityWait: capacityWait, Outcome: domain.ExecutionOutcomeError, Failure: ClassifyFailure(err)})
|
||||
}
|
||||
return finish(nil, err)
|
||||
}
|
||||
dispatchStart := c.now()
|
||||
response, err := c.injected.Generate(ctx, req)
|
||||
@@ -166,10 +255,33 @@ func (c *Coordinator) generateInjected(ctx context.Context, req domain.GenerateR
|
||||
if err == nil && response == nil {
|
||||
err = llm.ErrMalformedResponse
|
||||
}
|
||||
if captureEvidence {
|
||||
attempt := AttemptEvidence{
|
||||
StartedAt: start.UTC(),
|
||||
Dispatched: true,
|
||||
ResponseReceived: response != nil,
|
||||
RequestDuration: duration,
|
||||
Failure: ClassifyFailure(err),
|
||||
GenerationCompleted: err == nil && response != nil,
|
||||
}
|
||||
requested := req.StructuredOutput != nil
|
||||
attempt.StructuredOutputRequested = &requested
|
||||
var reportedUsage domain.TokenUsage
|
||||
var reportedPresence *domain.TokenUsagePresence
|
||||
if err == nil && response != nil {
|
||||
reportedUsage = response.Usage
|
||||
reportedPresence = response.UsagePresence
|
||||
}
|
||||
usage.Add(reportedUsage, reportedPresence)
|
||||
var attemptUsage observation.UsageAccumulator
|
||||
attemptUsage.Add(reportedUsage, reportedPresence)
|
||||
attempt.Usage = attemptUsage.Observed()
|
||||
evidence.Attempts = append(evidence.Attempts, attempt)
|
||||
}
|
||||
release()
|
||||
if recorder != nil {
|
||||
data := observation.AttemptData{Generation: generation, Attempt: 1, Dispatched: true, CapacityWait: capacityWait, GenerationDuration: duration,
|
||||
Outcome: attemptOutcome(err), Failure: attemptFailure(err)}
|
||||
Outcome: attemptOutcome(err), Failure: ClassifyFailure(err)}
|
||||
if err == nil {
|
||||
data.Usage = response.Usage
|
||||
data.UsagePresence = response.UsagePresence
|
||||
@@ -177,28 +289,38 @@ func (c *Coordinator) generateInjected(ctx context.Context, req domain.GenerateR
|
||||
recorder.Attempt(data)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return finish(nil, err)
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
if captureEvidence {
|
||||
return finish(response, err)
|
||||
}
|
||||
return finish(nil, err)
|
||||
}
|
||||
return response, nil
|
||||
return finish(response, nil)
|
||||
}
|
||||
|
||||
func (c *Coordinator) execute(ctx context.Context, backendID string, prepared *llm.PreparedGeneration) (outcome llm.AttemptOutcome, dispatched bool, capacityWait, generationDuration time.Duration) {
|
||||
func (c *Coordinator) execute(ctx context.Context, backendID string, prepared *llm.PreparedGeneration, captureEvidence bool) (outcome llm.AttemptOutcome, dispatched bool, capacityWait, generationDuration time.Duration, startedAt time.Time) {
|
||||
start := c.now()
|
||||
startedAt = start
|
||||
release, err := c.manager.Acquire(ctx, backendID)
|
||||
capacityWait = c.capacityWait(backendID, start)
|
||||
if err != nil {
|
||||
return llm.AttemptOutcome{Err: err}, false, capacityWait, 0
|
||||
return llm.AttemptOutcome{Err: err}, false, capacityWait, 0, startedAt
|
||||
}
|
||||
defer release()
|
||||
if err := ctx.Err(); err != nil {
|
||||
return llm.AttemptOutcome{Err: err}, false, capacityWait, 0
|
||||
return llm.AttemptOutcome{Err: err}, false, capacityWait, 0, startedAt
|
||||
}
|
||||
dispatchStart := c.now()
|
||||
outcome = prepared.Execute(ctx)
|
||||
return outcome, true, capacityWait, elapsed(dispatchStart, c.now())
|
||||
if captureEvidence {
|
||||
outcome = prepared.ExecuteWithEvidence(ctx)
|
||||
dispatched = outcome.Evidence.DispatchInvoked
|
||||
} else {
|
||||
outcome = prepared.Execute(ctx)
|
||||
dispatched = true
|
||||
}
|
||||
return outcome, dispatched, capacityWait, elapsed(dispatchStart, c.now()), startedAt
|
||||
}
|
||||
|
||||
func (c *Coordinator) capacityWait(backendID string, start time.Time) time.Duration {
|
||||
@@ -222,7 +344,8 @@ func attemptOutcome(err error) domain.ExecutionOutcome {
|
||||
return domain.ExecutionOutcomeError
|
||||
}
|
||||
|
||||
func attemptFailure(err error) domain.ExecutionFailureKind {
|
||||
// ClassifyFailure maps an execution error to the shared content-free category.
|
||||
func ClassifyFailure(err error) domain.ExecutionFailureKind {
|
||||
if err == nil {
|
||||
return domain.ExecutionFailureNone
|
||||
}
|
||||
|
||||
43
internal/generation/evidence.go
Normal file
43
internal/generation/evidence.go
Normal file
@@ -0,0 +1,43 @@
|
||||
package generation
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/llm"
|
||||
)
|
||||
|
||||
// GenerationEvidence contains invocation-local request evidence without raw
|
||||
// provider bodies, headers, credentials, or generated content.
|
||||
type GenerationEvidence struct {
|
||||
// InitialEndpoint is the frozen built-in request URL, if prepared.
|
||||
// Injected generation leaves it unknown.
|
||||
InitialEndpoint string
|
||||
Attempts []AttemptEvidence
|
||||
RetryDelays []time.Duration
|
||||
Usage domain.ObservedTokenUsage
|
||||
}
|
||||
|
||||
// AttemptEvidence describes one capacity acquisition and any dispatch it
|
||||
// reached. Generated content remains on the separate response return value.
|
||||
type AttemptEvidence struct {
|
||||
StartedAt time.Time
|
||||
Dispatched bool
|
||||
ResponseReceived bool
|
||||
HTTP llm.HTTPExecutionEvidence
|
||||
CapacityWait time.Duration
|
||||
RequestDuration time.Duration
|
||||
Failure domain.ExecutionFailureKind
|
||||
Usage domain.ObservedTokenUsage
|
||||
ResponseDecoded bool
|
||||
GenerationCompleted bool
|
||||
StructuredOutputRequested *bool
|
||||
}
|
||||
|
||||
func cloneHTTPEvidence(evidence llm.HTTPExecutionEvidence) llm.HTTPExecutionEvidence {
|
||||
if evidence.HTTPStatus != nil {
|
||||
status := *evidence.HTTPStatus
|
||||
evidence.HTTPStatus = &status
|
||||
}
|
||||
return evidence
|
||||
}
|
||||
321
internal/generation/evidence_test.go
Normal file
321
internal/generation/evidence_test.go
Normal file
@@ -0,0 +1,321 @@
|
||||
package generation
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/capacity"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/llm"
|
||||
)
|
||||
|
||||
func TestGenerateWithEvidenceRetainsAttemptsAndUsage(t *testing.T) {
|
||||
var now = time.Date(2026, 10, 13, 12, 0, 0, 0, time.UTC)
|
||||
calls := 0
|
||||
coordinator := testCoordinator(t, func(*http.Request) (*http.Response, error) {
|
||||
calls++
|
||||
if calls == 1 {
|
||||
now = now.Add(4 * time.Millisecond)
|
||||
return response(http.StatusServiceUnavailable, `{"error":{}}`), nil
|
||||
}
|
||||
now = now.Add(7 * time.Millisecond)
|
||||
return response(http.StatusOK, `{"choices":[{"message":{"content":"ok"}}],"usage":{"prompt_tokens":3}}`), nil
|
||||
})
|
||||
coordinator.now = func() time.Time { return now }
|
||||
coordinator.wait = func(context.Context, time.Duration) error {
|
||||
now = now.Add(19 * time.Millisecond)
|
||||
return nil
|
||||
}
|
||||
|
||||
result, err, evidence := coordinator.GenerateWithEvidence(context.Background(), testRequest(1))
|
||||
if err != nil || result == nil || result.Content != "ok" || calls != 2 {
|
||||
t.Fatalf("result=%+v error=%v calls=%d", result, err, calls)
|
||||
}
|
||||
if len(evidence.Attempts) != 2 || len(evidence.RetryDelays) != 1 || evidence.RetryDelays[0] != 19*time.Millisecond {
|
||||
t.Fatalf("attempts/delays = %#v / %#v", evidence.Attempts, evidence.RetryDelays)
|
||||
}
|
||||
first, second := evidence.Attempts[0], evidence.Attempts[1]
|
||||
if !first.Dispatched || !first.HTTP.RequestObserved || !first.HTTP.ResponseReceived || first.HTTP.HTTPStatus == nil || *first.HTTP.HTTPStatus != http.StatusServiceUnavailable {
|
||||
t.Fatalf("first attempt evidence = %+v", first)
|
||||
}
|
||||
if first.Failure != domain.ExecutionFailureProviderStatus || first.RequestDuration != 4*time.Millisecond {
|
||||
t.Fatalf("first attempt failure/timing = %v / %v", first.Failure, first.RequestDuration)
|
||||
}
|
||||
if !second.Dispatched || !second.GenerationCompleted || !second.ResponseDecoded || second.HTTP.HTTPStatus == nil || *second.HTTP.HTTPStatus != http.StatusOK {
|
||||
t.Fatalf("second attempt evidence = %+v", second)
|
||||
}
|
||||
if second.Failure != domain.ExecutionFailureNone || second.RequestDuration != 7*time.Millisecond {
|
||||
t.Fatalf("second attempt failure/timing = %v / %v", second.Failure, second.RequestDuration)
|
||||
}
|
||||
if !second.Usage.PromptTokens.Known || !second.Usage.PromptTokens.Complete || second.Usage.PromptTokens.Value != 3 {
|
||||
t.Fatalf("per-attempt usage = %+v", second.Usage.PromptTokens)
|
||||
}
|
||||
if !evidence.Usage.PromptTokens.Known || evidence.Usage.PromptTokens.Complete || evidence.Usage.PromptTokens.Value != 3 {
|
||||
t.Fatalf("aggregate usage = %+v", evidence.Usage.PromptTokens)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateWithEvidenceRetainsExhaustedAttempts(t *testing.T) {
|
||||
calls := 0
|
||||
coordinator := testCoordinator(t, func(*http.Request) (*http.Response, error) {
|
||||
calls++
|
||||
return response(http.StatusServiceUnavailable, `{"error":{}}`), nil
|
||||
})
|
||||
coordinator.wait = func(context.Context, time.Duration) error { return nil }
|
||||
|
||||
result, err, evidence := coordinator.GenerateWithEvidence(context.Background(), testRequest(1))
|
||||
var exhausted *ExhaustedError
|
||||
if result != nil || !errors.As(err, &exhausted) || calls != 2 || exhausted.Attempts() != 2 {
|
||||
t.Fatalf("result=%+v error=%v calls=%d", result, err, calls)
|
||||
}
|
||||
if len(evidence.Attempts) != 2 || len(evidence.RetryDelays) != 1 {
|
||||
t.Fatalf("attempt/delay evidence = %#v / %#v", evidence.Attempts, evidence.RetryDelays)
|
||||
}
|
||||
for _, attempt := range evidence.Attempts {
|
||||
if !attempt.Dispatched || attempt.Failure != domain.ExecutionFailureProviderStatus || attempt.HTTP.HTTPStatus == nil || *attempt.HTTP.HTTPStatus != http.StatusServiceUnavailable {
|
||||
t.Fatalf("exhausted attempt evidence = %+v", attempt)
|
||||
}
|
||||
}
|
||||
if evidence.Usage.PromptTokens.Known || evidence.Usage.PromptTokens.Complete {
|
||||
t.Fatalf("missing provider usage was claimed: %+v", evidence.Usage.PromptTokens)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateWithEvidenceRetainsInterruptedBackoffWithoutAttempt(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
var now = time.Date(2026, 10, 13, 12, 0, 0, 0, time.UTC)
|
||||
calls := 0
|
||||
coordinator := testCoordinator(t, func(*http.Request) (*http.Response, error) {
|
||||
calls++
|
||||
now = now.Add(3 * time.Millisecond)
|
||||
return response(http.StatusServiceUnavailable, `{"error":{}}`), nil
|
||||
})
|
||||
coordinator.now = func() time.Time { return now }
|
||||
coordinator.wait = func(ctx context.Context, _ time.Duration) error {
|
||||
now = now.Add(11 * time.Millisecond)
|
||||
cancel()
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
result, err, evidence := coordinator.GenerateWithEvidence(ctx, testRequest(2))
|
||||
if result != nil || !errors.Is(err, context.Canceled) || calls != 1 {
|
||||
t.Fatalf("result=%+v error=%v calls=%d", result, err, calls)
|
||||
}
|
||||
if len(evidence.Attempts) != 1 || !evidence.Attempts[0].Dispatched || len(evidence.RetryDelays) != 1 || evidence.RetryDelays[0] != 11*time.Millisecond {
|
||||
t.Fatalf("interrupted evidence = %#v", evidence)
|
||||
}
|
||||
if evidence.Attempts[0].HTTP.HTTPStatus == nil || *evidence.Attempts[0].HTTP.HTTPStatus != http.StatusServiceUnavailable {
|
||||
t.Fatalf("last dispatched response was lost: %+v", evidence.Attempts[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateWithEvidenceRecordsCanceledReacquisition(t *testing.T) {
|
||||
var now = time.Date(2026, 10, 13, 12, 0, 0, 0, time.UTC)
|
||||
calls := 0
|
||||
coordinator := testCoordinator(t, func(*http.Request) (*http.Response, error) {
|
||||
calls++
|
||||
now = now.Add(2 * time.Millisecond)
|
||||
return response(http.StatusServiceUnavailable, `{"error":{}}`), nil
|
||||
})
|
||||
coordinator.now = func() time.Time { return now }
|
||||
baseContext, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
contextWithCancelOnWait := &cancelOnDoneContext{Context: baseContext, cancel: cancel}
|
||||
var releaseHeld func()
|
||||
coordinator.wait = func(context.Context, time.Duration) error {
|
||||
release, err := coordinator.manager.Acquire(context.Background(), "limited")
|
||||
if err != nil {
|
||||
t.Fatalf("hold permit: %v", err)
|
||||
}
|
||||
releaseHeld = release
|
||||
contextWithCancelOnWait.armed.Store(true)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Signal cancellation only when the second acquisition has queued behind
|
||||
// the held permit, avoiding scheduler- or sleep-based synchronization.
|
||||
result, err, evidence := coordinator.GenerateWithEvidence(contextWithCancelOnWait, testRequest(2))
|
||||
if releaseHeld != nil {
|
||||
releaseHeld()
|
||||
}
|
||||
if result != nil || !errors.Is(err, context.Canceled) || calls != 1 {
|
||||
t.Fatalf("result=%+v error=%v calls=%d", result, err, calls)
|
||||
}
|
||||
if len(evidence.Attempts) != 2 || !evidence.Attempts[0].Dispatched || evidence.Attempts[1].Dispatched {
|
||||
t.Fatalf("acquisition evidence = %#v", evidence.Attempts)
|
||||
}
|
||||
if evidence.Attempts[1].Failure != domain.ExecutionFailureCanceled {
|
||||
t.Fatalf("canceled acquisition failure = %v", evidence.Attempts[1].Failure)
|
||||
}
|
||||
if evidence.Attempts[0].HTTP.HTTPStatus == nil || *evidence.Attempts[0].HTTP.HTTPStatus != http.StatusServiceUnavailable {
|
||||
t.Fatalf("last dispatched evidence was lost: %+v", evidence.Attempts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateWithEvidenceNoDispatchHasCompleteZeroUsage(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
coordinator := testCoordinator(t, func(*http.Request) (*http.Response, error) {
|
||||
t.Fatal("canceled generation dispatched")
|
||||
return nil, nil
|
||||
})
|
||||
|
||||
result, err, evidence := coordinator.GenerateWithEvidence(ctx, testRequest(0))
|
||||
if result != nil || !errors.Is(err, context.Canceled) || len(evidence.Attempts) != 0 {
|
||||
t.Fatalf("result=%+v error=%v evidence=%+v", result, err, evidence)
|
||||
}
|
||||
if !evidence.Usage.PromptTokens.Known || !evidence.Usage.PromptTokens.Complete || evidence.Usage.PromptTokens.Value != 0 {
|
||||
t.Fatalf("no-dispatch usage = %+v", evidence.Usage.PromptTokens)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateWithEvidenceForInjectedClient(t *testing.T) {
|
||||
opaqueFailure := errors.New("injected client failed")
|
||||
presence := &domain.TokenUsagePresence{PromptTokens: true}
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
response *domain.GenerateResponse
|
||||
err error
|
||||
wantResponse bool
|
||||
wantGeneration bool
|
||||
wantFailure domain.ExecutionFailureKind
|
||||
wantUsage bool
|
||||
wantStructured bool
|
||||
}{
|
||||
{
|
||||
name: "success",
|
||||
response: &domain.GenerateResponse{Content: "private output", Usage: domain.TokenUsage{PromptTokens: 4}, UsagePresence: presence},
|
||||
wantResponse: true,
|
||||
wantGeneration: true,
|
||||
wantUsage: true,
|
||||
wantStructured: true,
|
||||
},
|
||||
{
|
||||
name: "nil response",
|
||||
wantFailure: domain.ExecutionFailureMalformedResponse,
|
||||
},
|
||||
{
|
||||
name: "opaque failure",
|
||||
err: opaqueFailure,
|
||||
wantFailure: domain.ExecutionFailureGeneration,
|
||||
},
|
||||
{
|
||||
name: "response with error",
|
||||
response: &domain.GenerateResponse{Content: "must not be evaluated", Usage: domain.TokenUsage{PromptTokens: 9}, UsagePresence: presence},
|
||||
err: opaqueFailure,
|
||||
wantResponse: true,
|
||||
wantFailure: domain.ExecutionFailureGeneration,
|
||||
wantStructured: true,
|
||||
},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
client := &evidenceInjectedClient{response: tc.response, err: tc.err}
|
||||
coordinator := newEvidenceInjectedCoordinator(t, client)
|
||||
req := testRequest(7)
|
||||
if tc.wantStructured {
|
||||
req.StructuredOutput = &domain.StructuredOutputSpec{}
|
||||
}
|
||||
|
||||
result, err, evidence := coordinator.GenerateWithEvidence(context.Background(), req)
|
||||
if client.calls != 1 || len(evidence.Attempts) != 1 || len(evidence.RetryDelays) != 0 {
|
||||
t.Fatalf("calls=%d attempts=%+v retry delays=%v", client.calls, evidence.Attempts, evidence.RetryDelays)
|
||||
}
|
||||
attempt := evidence.Attempts[0]
|
||||
if !attempt.Dispatched || attempt.ResponseReceived != tc.wantResponse || attempt.GenerationCompleted != tc.wantGeneration || attempt.Failure != tc.wantFailure {
|
||||
t.Fatalf("attempt evidence = %+v", attempt)
|
||||
}
|
||||
if attempt.HTTP != (llm.HTTPExecutionEvidence{}) {
|
||||
t.Fatalf("injected call claimed HTTP evidence: %+v", attempt.HTTP)
|
||||
}
|
||||
if attempt.StructuredOutputRequested == nil || *attempt.StructuredOutputRequested != tc.wantStructured {
|
||||
t.Fatalf("structured-output request fact = %v", attempt.StructuredOutputRequested)
|
||||
}
|
||||
if tc.wantFailure == domain.ExecutionFailureNone {
|
||||
if err != nil || result == nil || result.Content != "private output" {
|
||||
t.Fatalf("result=%+v error=%v", result, err)
|
||||
}
|
||||
} else if result != nil || err == nil {
|
||||
t.Fatalf("result=%+v error=%v; expected failed call", result, err)
|
||||
}
|
||||
if attempt.Usage.PromptTokens.Known != tc.wantUsage || evidence.Usage.PromptTokens.Known != tc.wantUsage {
|
||||
t.Fatalf("attempt/aggregate usage = %+v / %+v", attempt.Usage, evidence.Usage)
|
||||
}
|
||||
if tc.wantUsage && (attempt.Usage.PromptTokens.Value != 4 || evidence.Usage.PromptTokens.Value != 4) {
|
||||
t.Fatalf("reported usage was not retained: %+v / %+v", attempt.Usage, evidence.Usage)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateWithEvidenceRetainsInjectedSuccessAfterCancellation(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
client := &evidenceInjectedClient{
|
||||
response: &domain.GenerateResponse{Content: "private output"},
|
||||
afterGenerate: func() { cancel() },
|
||||
}
|
||||
coordinator := newEvidenceInjectedCoordinator(t, client)
|
||||
req := testRequest(5)
|
||||
req.StructuredOutput = &domain.StructuredOutputSpec{}
|
||||
|
||||
result, err, evidence := coordinator.GenerateWithEvidence(ctx, req)
|
||||
if !errors.Is(err, context.Canceled) || result == nil || result.Content != "private output" {
|
||||
t.Fatalf("result=%+v error=%v", result, err)
|
||||
}
|
||||
if client.calls != 1 || len(evidence.Attempts) != 1 {
|
||||
t.Fatalf("calls=%d attempts=%+v", client.calls, evidence.Attempts)
|
||||
}
|
||||
attempt := evidence.Attempts[0]
|
||||
if !attempt.ResponseReceived || !attempt.GenerationCompleted || attempt.Failure != domain.ExecutionFailureNone {
|
||||
t.Fatalf("successful generation evidence was lost: %+v", attempt)
|
||||
}
|
||||
if attempt.HTTP != (llm.HTTPExecutionEvidence{}) {
|
||||
t.Fatalf("injected call claimed HTTP evidence: %+v", attempt.HTTP)
|
||||
}
|
||||
}
|
||||
|
||||
type evidenceInjectedClient struct {
|
||||
calls int
|
||||
response *domain.GenerateResponse
|
||||
err error
|
||||
afterGenerate func()
|
||||
}
|
||||
|
||||
func (c *evidenceInjectedClient) Generate(context.Context, domain.GenerateRequest) (*domain.GenerateResponse, error) {
|
||||
c.calls++
|
||||
if c.afterGenerate != nil {
|
||||
c.afterGenerate()
|
||||
}
|
||||
return c.response, c.err
|
||||
}
|
||||
|
||||
func newEvidenceInjectedCoordinator(t *testing.T, client llm.Client) *Coordinator {
|
||||
t.Helper()
|
||||
manager, err := capacity.NewManager(map[string]domain.BackendCapacityPolicy{
|
||||
"limited": {ConcurrencyLimit: 1},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return NewInjected(manager, client)
|
||||
}
|
||||
|
||||
type cancelOnDoneContext struct {
|
||||
context.Context
|
||||
cancel context.CancelFunc
|
||||
once sync.Once
|
||||
armed atomic.Bool
|
||||
}
|
||||
|
||||
func (c *cancelOnDoneContext) Done() <-chan struct{} {
|
||||
if c.armed.Load() {
|
||||
c.once.Do(c.cancel)
|
||||
}
|
||||
return c.Context.Done()
|
||||
}
|
||||
191
internal/llm/http_evidence.go
Normal file
191
internal/llm/http_evidence.go
Normal file
@@ -0,0 +1,191 @@
|
||||
package llm
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
)
|
||||
|
||||
// HTTPExecutionEvidence contains only facts observable at the configured
|
||||
// RoundTripper boundary. It intentionally excludes request headers and bodies.
|
||||
type HTTPExecutionEvidence struct {
|
||||
DispatchInvoked bool
|
||||
RequestObserved bool
|
||||
Endpoint string
|
||||
CredentialsSupplied bool
|
||||
ResponseReceived bool
|
||||
HTTPStatus *int
|
||||
}
|
||||
|
||||
type httpEvidenceCollector struct {
|
||||
mu sync.Mutex
|
||||
evidence HTTPExecutionEvidence
|
||||
}
|
||||
|
||||
func (c *httpEvidenceCollector) observeRequest(request *http.Request) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
|
||||
c.evidence.RequestObserved = true
|
||||
c.evidence.Endpoint = endpointWithoutUserInfo(request)
|
||||
c.evidence.CredentialsSupplied = requestHasCredentials(request)
|
||||
c.evidence.ResponseReceived = false
|
||||
c.evidence.HTTPStatus = nil
|
||||
}
|
||||
|
||||
func (c *httpEvidenceCollector) observeResponse(response *http.Response) {
|
||||
if response == nil {
|
||||
return
|
||||
}
|
||||
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
status := response.StatusCode
|
||||
c.evidence.ResponseReceived = true
|
||||
c.evidence.HTTPStatus = &status
|
||||
}
|
||||
|
||||
func (c *httpEvidenceCollector) snapshot() HTTPExecutionEvidence {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
|
||||
evidence := c.evidence
|
||||
if c.evidence.HTTPStatus != nil {
|
||||
status := *c.evidence.HTTPStatus
|
||||
evidence.HTTPStatus = &status
|
||||
}
|
||||
return evidence
|
||||
}
|
||||
|
||||
type observingRoundTripper struct {
|
||||
next http.RoundTripper
|
||||
evidence *httpEvidenceCollector
|
||||
}
|
||||
|
||||
func (t observingRoundTripper) RoundTrip(request *http.Request) (*http.Response, error) {
|
||||
t.evidence.observeRequest(request)
|
||||
response, err := t.next.RoundTrip(request)
|
||||
t.evidence.observeResponse(response)
|
||||
return response, err
|
||||
}
|
||||
|
||||
// doWithEvidence executes one request with a shallow copy of client. The copy
|
||||
// preserves timeout, redirect, and cookie-jar behavior without changing client.
|
||||
func doWithEvidence(client *http.Client, request *http.Request) (*http.Response, error, HTTPExecutionEvidence) {
|
||||
if client == nil {
|
||||
client = &http.Client{}
|
||||
}
|
||||
|
||||
collector := &httpEvidenceCollector{}
|
||||
clientCopy := *client
|
||||
transport := clientCopy.Transport
|
||||
if transport == nil {
|
||||
transport = http.DefaultTransport
|
||||
}
|
||||
clientCopy.Transport = observingRoundTripper{next: transport, evidence: collector}
|
||||
|
||||
response, err := clientCopy.Do(request)
|
||||
observed := collector.snapshot()
|
||||
observed.DispatchInvoked = true
|
||||
return response, err, observed
|
||||
}
|
||||
|
||||
// GetWithEvidence performs a plain GET with the configured built-in HTTP
|
||||
// client. A positive timeoutSeconds applies to the request through response
|
||||
// body closure, matching the whole-request behavior of http.Client.Timeout.
|
||||
func (c *OpenAICompatibleClient) GetWithEvidence(
|
||||
ctx context.Context,
|
||||
endpoint string,
|
||||
timeoutSeconds int,
|
||||
) (*http.Response, HTTPExecutionEvidence, error) {
|
||||
return c.getWithEvidence(ctx, endpoint, timeoutSeconds, "")
|
||||
}
|
||||
|
||||
// GetWithAuthenticationEvidence performs a plain GET with credentials resolved
|
||||
// from the selected target. Missing credentials are allowed for diagnostic
|
||||
// authentication probes even when generation requires a key.
|
||||
func (c *OpenAICompatibleClient) GetWithAuthenticationEvidence(
|
||||
ctx context.Context,
|
||||
endpoint string,
|
||||
timeoutSeconds int,
|
||||
target domain.ExecutionTarget,
|
||||
) (*http.Response, HTTPExecutionEvidence, error) {
|
||||
return c.getWithEvidence(ctx, endpoint, timeoutSeconds, resolveAPIKey(target))
|
||||
}
|
||||
|
||||
func (c *OpenAICompatibleClient) getWithEvidence(
|
||||
ctx context.Context,
|
||||
endpoint string,
|
||||
timeoutSeconds int,
|
||||
apiKey string,
|
||||
) (*http.Response, HTTPExecutionEvidence, error) {
|
||||
requestContext := ctx
|
||||
var cancel context.CancelFunc
|
||||
if timeoutSeconds > 0 {
|
||||
requestContext, cancel = context.WithTimeout(ctx, secondsDuration(timeoutSeconds))
|
||||
}
|
||||
|
||||
request, err := http.NewRequestWithContext(requestContext, http.MethodGet, endpoint, nil)
|
||||
if err != nil {
|
||||
if cancel != nil {
|
||||
cancel()
|
||||
}
|
||||
return nil, HTTPExecutionEvidence{}, err
|
||||
}
|
||||
if apiKey != "" {
|
||||
request.Header.Set("Authorization", "Bearer "+apiKey)
|
||||
}
|
||||
|
||||
response, err, evidence := doWithEvidence(c.httpClient, request)
|
||||
if response == nil || response.Body == nil {
|
||||
if cancel != nil {
|
||||
cancel()
|
||||
}
|
||||
return response, evidence, err
|
||||
}
|
||||
if cancel != nil {
|
||||
response.Body = &cancelOnCloseReadCloser{ReadCloser: response.Body, cancel: cancel}
|
||||
}
|
||||
return response, evidence, err
|
||||
}
|
||||
|
||||
func secondsDuration(seconds int) time.Duration {
|
||||
return time.Duration(seconds) * time.Second
|
||||
}
|
||||
|
||||
type cancelOnCloseReadCloser struct {
|
||||
io.ReadCloser
|
||||
cancel context.CancelFunc
|
||||
once sync.Once
|
||||
}
|
||||
|
||||
func (r *cancelOnCloseReadCloser) Close() error {
|
||||
r.once.Do(r.cancel)
|
||||
return r.ReadCloser.Close()
|
||||
}
|
||||
|
||||
func endpointWithoutUserInfo(request *http.Request) string {
|
||||
if request == nil || request.URL == nil {
|
||||
return ""
|
||||
}
|
||||
endpoint := *request.URL
|
||||
endpoint.User = nil
|
||||
return endpoint.String()
|
||||
}
|
||||
|
||||
func requestHasCredentials(request *http.Request) bool {
|
||||
if request == nil {
|
||||
return false
|
||||
}
|
||||
if request.Header.Get("Authorization") != "" {
|
||||
return true
|
||||
}
|
||||
return request.URL != nil && request.URL.User != nil
|
||||
}
|
||||
|
||||
var _ http.RoundTripper = observingRoundTripper{}
|
||||
var _ io.ReadCloser = (*cancelOnCloseReadCloser)(nil)
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user