Compare commits
119 Commits
v0.6.0
...
62a4a20529
| Author | SHA1 | Date | |
|---|---|---|---|
| 62a4a20529 | |||
| 8c68b28421 | |||
| 66c55b7397 | |||
| 48343300e2 | |||
| f17a38f60b | |||
| aba5e960aa | |||
| d818e0d162 | |||
| ba9c972faa | |||
| 17b6c7bca5 | |||
| 2969a83a71 | |||
| 7f58261498 | |||
| defb240b6e | |||
| b158a8ec2d | |||
| 2b847ac2ae | |||
| 5f7b19403f | |||
| 80131fa160 | |||
| 521291d80f | |||
| 0f43460f84 | |||
| 08409dd472 | |||
| 34f5980973 | |||
| 7b012033a6 | |||
| 37f8dba6c3 | |||
| c7bdaa076b | |||
| 12d6d80024 | |||
| d8147cac49 | |||
| a16c2a38d6 | |||
| 843d07b636 | |||
| dc6fd2752d | |||
| 15e4067954 | |||
| 6e34fb9de5 | |||
| 5843ceef82 | |||
| 1cd0c611f2 | |||
| 6ed582f4b5 | |||
| db05b56166 | |||
| 0c08038b4f | |||
| 48378a3269 | |||
| cef8524b31 | |||
| 1cc754c29e | |||
| 93e53d1199 | |||
| 63bedf2a97 | |||
| 0e1ad784d1 | |||
| be07f6d873 | |||
| 836919ec4b | |||
| 07a131bd33 | |||
| ae4540822e | |||
| 65d5894648 | |||
| d31a563ca1 | |||
| ea94d66fc2 | |||
| 056bfd0289 | |||
| bd975f17f2 | |||
| 4431f34bc9 | |||
| 1580b25ea6 | |||
| 6dfd6bd719 | |||
| 9f69da716b | |||
| 5dd5b2f3cc | |||
| d79f1e9562 | |||
| c8e4abcdbd | |||
| 6ce54c1d79 | |||
| 2356f7c73d | |||
| ba1e85a805 | |||
| 4c3828857c | |||
| 02fd09955d | |||
| 7389410966 | |||
| 3bc624ec3f | |||
| e5870453f9 | |||
| 06251e385e | |||
| 722b5ddf59 | |||
| 47bc9a6655 | |||
| bd2b7b1862 | |||
| c892eacb65 | |||
| ff73104f7e | |||
| 7a858c2365 | |||
| e4540aa619 | |||
| a4264bf4b4 | |||
| 81f41564a2 | |||
| 6b5a2497cc | |||
| 422cc6c978 | |||
| f48e042565 | |||
| d9442850ef | |||
| 2e828157b6 | |||
| 4189146536 | |||
| 59be507d2f | |||
| b45cea4084 | |||
| efe885c6b2 | |||
| 404a4c331d | |||
| 174516cb39 | |||
| b0999112cc | |||
| ed0f7527d5 | |||
| 5064cf833d | |||
| 8745d256bd | |||
| 2e76003fd5 | |||
| 36ce5a5099 | |||
| 465dc1389d | |||
| ae6f1a9865 | |||
| ee99dc9478 | |||
| 00ee5893e9 | |||
| 64d1cffd89 | |||
| f9e8afa2c3 | |||
| e827631d8c | |||
| 115fe8ba58 | |||
| c53250f023 | |||
| 3d99483219 | |||
| 67f788b1e2 | |||
| 764103a2e2 | |||
| a08dd83d1f | |||
| e8922d8ec5 | |||
| 2d44305a8a | |||
| a11c80291e | |||
| 3239567297 | |||
| c239304c2a | |||
| 159b02116f | |||
| e5b7adfb49 | |||
| fa2384e696 | |||
| af0bd3f31a | |||
| 5fff8cd623 | |||
| b2a6c47778 | |||
| a1805fe550 | |||
| 93af155254 | |||
| d783b687a5 |
@@ -1,2 +1,6 @@
|
||||
Please review `docs/development.md` for initial orientation in this repository
|
||||
and follow its task-specific reading guide.
|
||||
Start with `docs/development.md` for orientation and use its policy reading guide
|
||||
to select the policies applicable to the task. Do not read the entire suite by
|
||||
default.
|
||||
|
||||
Read and follow `docs/policy/agents.md` before recommending, planning, or
|
||||
implementing repository changes.
|
||||
|
||||
23
COPYRIGHT
Normal file
23
COPYRIGHT
Normal file
@@ -0,0 +1,23 @@
|
||||
PromptKit
|
||||
|
||||
Copyright (C) 2026 Eric Rakestraw
|
||||
|
||||
SPDX-License-Identifier: LGPL-3.0-or-later
|
||||
|
||||
This library is free software: you can redistribute it and/or modify it
|
||||
under the terms of the GNU Lesser General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
The GNU Lesser General Public License version 3 is provided in LICENSE.
|
||||
|
||||
This library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Lesser General Public License for more details.
|
||||
|
||||
Policy documents copied or adapted from Standard Repository Policies remain licensed
|
||||
under CC-BY-SA-4.0. See docs/oss/attribution.md for the affected material,
|
||||
upstream credit, and license links.
|
||||
|
||||
Third-party code and assets are subject to their respective license terms.
|
||||
44
README.md
44
README.md
@@ -8,7 +8,8 @@ gitea.maximumdirect.net/eric/promptkit
|
||||
```
|
||||
|
||||
The root `promptkit` package provides the supported public engine. Consumers
|
||||
can configure filesystem or in-memory prompt, profile, and schema sources,
|
||||
can configure file, embedded, or programmatic backend, profile, and prompt
|
||||
definitions alongside filesystem or in-memory schemas,
|
||||
prepare requests without generation, run requests with the built-in
|
||||
OpenAI-compatible client, or inject their own model client and artifact reader.
|
||||
|
||||
@@ -20,9 +21,12 @@ Run the maintained offline preparation example from the repository root:
|
||||
go run ./examples/go-library/prepare
|
||||
```
|
||||
|
||||
It loads a repository-local prompt, supplies an in-memory profile and inline
|
||||
input, and prints deterministic preparation metadata without contacting a
|
||||
provider or requiring credentials. Read the
|
||||
It loads a repository-local prompt, supplies a programmatic profile and inline
|
||||
input, inspects the captured configuration, and prints deterministic
|
||||
preparation metadata without contacting a provider or requiring credentials.
|
||||
The [consumer guide](docs/consumers/pkg-promptkit.md) also shows a local backend,
|
||||
ordered file/embedded sources, programmatic mappings, and how to replace an
|
||||
engine when source files change. Read the
|
||||
[example source](examples/go-library/prepare/main.go), the
|
||||
[Go package consumer guide](docs/consumers/pkg-promptkit.md), and the
|
||||
[framework format reference](docs/formats.md) to build a consumer workflow.
|
||||
@@ -33,10 +37,32 @@ boundary and constraints that framework work must preserve.
|
||||
|
||||
## Release Guidance
|
||||
|
||||
Consumers upgrading from `v0.5.0` to `v0.6.0` should read the
|
||||
[v0.6.0 changelog and migration guide](docs/releases/v0.6.0.md).
|
||||
The [v0.11.2 release notes](docs/releases/v0.11.2.md) cover policy alignment
|
||||
and copied-policy attribution updates.
|
||||
|
||||
The [v0.11.1 release notes](docs/releases/v0.11.1.md) cover the MIT-licensed
|
||||
catalog updates and licensing documentation improvements.
|
||||
|
||||
Consumers upgrading to `v0.11.0` should read the
|
||||
[v0.11.0 changelog and migration guide](docs/releases/v0.11.0.md), particularly
|
||||
the default retry behavior and license change.
|
||||
|
||||
Consumers upgrading to the `v0.10.0` profile-policy transition
|
||||
should read the [v0.10.0 transition guide](docs/releases/v0.10.0.md).
|
||||
|
||||
Consumers upgrading from `v0.8.0` to `v0.9.0` should read the
|
||||
[v0.9.0 changelog and migration guide](docs/releases/v0.9.0.md).
|
||||
|
||||
Consumers upgrading from `v0.7.0` to `v0.8.0` can consult the
|
||||
[v0.8.0 changelog and migration guide](docs/releases/v0.8.0.md).
|
||||
|
||||
Earlier adopters can consult the
|
||||
[v0.7.0 changelog and migration guide](docs/releases/v0.7.0.md).
|
||||
|
||||
Consumers upgrading from `v0.5.0` to `v0.6.0` can consult the
|
||||
[v0.6.0 changelog and migration guide](docs/releases/v0.6.0.md).
|
||||
|
||||
Consumers upgrading from `v0.4.0` to `v0.5.0` can consult the
|
||||
[v0.5.0 changelog and migration guide](docs/releases/v0.5.0.md).
|
||||
|
||||
Consumers upgrading from `v0.3.0` to `v0.4.0` should read the
|
||||
@@ -53,4 +79,8 @@ Consumers moving from `v0.1.0` to `v0.2.0` should read the
|
||||
[Scriptorium](https://gitea.maximumdirect.net/eric/scriptorium) is the CLI and
|
||||
HTTP application built on Promptkit.
|
||||
|
||||
Promptkit is licensed under the [GNU General Public License version 3](LICENSE).
|
||||
## License
|
||||
|
||||
Promptkit is licensed under the [LGPL v3.0 or later](COPYRIGHT).
|
||||
|
||||
[Open Source attribution notices](docs/oss/attribution.md) are available.
|
||||
|
||||
129
appended_messages_contract_test.go
Normal file
129
appended_messages_contract_test.go
Normal file
@@ -0,0 +1,129 @@
|
||||
package promptkit_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"reflect"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
func TestAppendedMessageRoleConstantsAreStrings(t *testing.T) {
|
||||
var (
|
||||
developer string = promptkit.RoleDeveloper
|
||||
system string = promptkit.RoleSystem
|
||||
user string = promptkit.RoleUser
|
||||
assistant string = promptkit.RoleAssistant
|
||||
)
|
||||
if developer != "developer" || system != "system" || user != "user" || assistant != "assistant" {
|
||||
t.Fatalf("unexpected role constants: %q %q %q %q", developer, system, user, assistant)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAppendedMessagesComposeFrozenEffectivePrompt(t *testing.T) {
|
||||
cacheControl := &promptkit.CacheControl{Type: promptkit.CacheControlEphemeral, TTL: "1h"}
|
||||
appended := []promptkit.RenderedMessage{
|
||||
{Role: " \tAsSiStAnT\n", Content: "previous response"},
|
||||
{Role: promptkit.RoleUser, Content: "corrective request", CacheControl: cacheControl},
|
||||
}
|
||||
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "output"}}
|
||||
engine := newSimpleEngine(t, "configured prompt", promptkit.WithLLMClient(client))
|
||||
request := promptkit.RunRequest{PromptID: "prompt", AppendedMessages: appended}
|
||||
|
||||
plain, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
|
||||
if err != nil {
|
||||
t.Fatalf("prepare plain request: %v", err)
|
||||
}
|
||||
prepared, err := engine.Prepare(context.Background(), request)
|
||||
if err != nil {
|
||||
t.Fatalf("prepare appended request: %v", err)
|
||||
}
|
||||
expected := []promptkit.RenderedMessage{
|
||||
{Role: promptkit.RoleUser, Content: "configured prompt"},
|
||||
{Role: promptkit.RoleAssistant, Content: "previous response"},
|
||||
{Role: promptkit.RoleUser, Content: "corrective request", CacheControl: &promptkit.CacheControl{Type: promptkit.CacheControlEphemeral, TTL: "1h"}},
|
||||
}
|
||||
if !reflect.DeepEqual(prepared.Messages, expected) || prepared.PromptHash != plain.PromptHash || prepared.RenderedPromptHash == plain.RenderedPromptHash {
|
||||
t.Fatalf("prepared effective prompt = %#v, plain = %#v", prepared, plain)
|
||||
}
|
||||
|
||||
equivalent, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt", AppendedMessages: []promptkit.RenderedMessage{
|
||||
{Role: promptkit.RoleAssistant, Content: "previous response"},
|
||||
{Role: promptkit.RoleUser, Content: "corrective request", CacheControl: &promptkit.CacheControl{Type: promptkit.CacheControlEphemeral, TTL: "1h"}},
|
||||
}})
|
||||
if err != nil || equivalent.RenderedPromptHash != prepared.RenderedPromptHash {
|
||||
t.Fatalf("normalized-equivalent request = (%#v, %v), want matching rendered hash %q", equivalent, err, prepared.RenderedPromptHash)
|
||||
}
|
||||
|
||||
for _, variant := range []promptkit.RunRequest{
|
||||
{PromptID: "prompt", AppendedMessages: []promptkit.RenderedMessage{{Role: promptkit.RoleAssistant, Content: "changed"}, expected[2]}},
|
||||
{PromptID: "prompt", AppendedMessages: []promptkit.RenderedMessage{expected[2], expected[1]}},
|
||||
{PromptID: "prompt", AppendedMessages: []promptkit.RenderedMessage{{Role: promptkit.RoleAssistant, Content: "previous response"}, {Role: promptkit.RoleUser, Content: "corrective request"}}},
|
||||
} {
|
||||
variantPrepared, prepareErr := engine.Prepare(context.Background(), variant)
|
||||
if prepareErr != nil || variantPrepared.RenderedPromptHash == prepared.RenderedPromptHash || variantPrepared.PromptHash != prepared.PromptHash {
|
||||
t.Fatalf("variant preparation = (%#v, %v)", variantPrepared, prepareErr)
|
||||
}
|
||||
}
|
||||
|
||||
result, err := engine.Run(context.Background(), request)
|
||||
if err != nil || result.RenderedPromptHash != prepared.RenderedPromptHash || !reflect.DeepEqual(client.requests[0].Prompt.Messages, expected) {
|
||||
t.Fatalf("run result = (%#v, %v), request = %#v", result, err, client.requests)
|
||||
}
|
||||
|
||||
execution, err := engine.PrepareExecution(context.Background(), request)
|
||||
if err != nil {
|
||||
t.Fatalf("prepare execution: %v", err)
|
||||
}
|
||||
appended[0].Content = "changed caller content"
|
||||
cacheControl.TTL = ""
|
||||
firstDetails := execution.Details()
|
||||
firstDetails.Messages[1].Content = "changed details content"
|
||||
firstDetails.Messages[2].CacheControl.Type = "changed details cache"
|
||||
secondDetails := execution.Details()
|
||||
if !reflect.DeepEqual(secondDetails.Messages, expected) || secondDetails.RenderedPromptHash != prepared.RenderedPromptHash {
|
||||
t.Fatalf("prepared execution details = %#v, want frozen %#v", secondDetails, expected)
|
||||
}
|
||||
|
||||
result, err = engine.RunPrepared(context.Background(), execution)
|
||||
if err != nil || result.RenderedPromptHash != prepared.RenderedPromptHash || !reflect.DeepEqual(client.requests[1].Prompt.Messages, expected) {
|
||||
t.Fatalf("prepared result = (%#v, %v), requests = %#v", result, err, client.requests)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInvalidAppendedMessagesFailBeforeModelWork(t *testing.T) {
|
||||
var modelCalls atomic.Int64
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{}, promptkit.WithLLMClient(countingLLMClient{calls: &modelCalls}))
|
||||
if err != nil {
|
||||
t.Fatalf("construct engine: %v", err)
|
||||
}
|
||||
request := promptkit.RunRequest{
|
||||
PromptID: "unreached",
|
||||
AppendedMessages: []promptkit.RenderedMessage{{
|
||||
Role: "unsupported-role",
|
||||
Content: "sensitive appended content",
|
||||
}},
|
||||
}
|
||||
|
||||
operations := []struct {
|
||||
name string
|
||||
run func() error
|
||||
}{
|
||||
{name: "Prepare", run: func() error { _, err := engine.Prepare(context.Background(), request); return err }},
|
||||
{name: "PrepareExecution", run: func() error { _, err := engine.PrepareExecution(context.Background(), request); return err }},
|
||||
{name: "Run", run: func() error { _, err := engine.Run(context.Background(), request); return err }},
|
||||
}
|
||||
for _, operation := range operations {
|
||||
t.Run(operation.name, func(t *testing.T) {
|
||||
err := operation.run()
|
||||
if !errors.Is(err, promptkit.ErrInvalidRequest) {
|
||||
t.Fatalf("error = %v, want ErrInvalidRequest", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
if modelCalls.Load() != 0 {
|
||||
t.Fatalf("invalid appended message invoked the model %d times", modelCalls.Load())
|
||||
}
|
||||
}
|
||||
135
backends.go
135
backends.go
@@ -1,100 +1,69 @@
|
||||
package promptkit
|
||||
|
||||
import (
|
||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/backend"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
)
|
||||
|
||||
// BackendOpenRouter is the reserved ID of Promptkit's built-in OpenRouter
|
||||
// backend.
|
||||
// BackendOpenRouter identifies the maintained OpenRouter backend.
|
||||
const BackendOpenRouter = backend.OpenRouterID
|
||||
|
||||
// BackendLocal is the case-sensitive conventional ID used by [LocalBackend].
|
||||
// It is not a built-in or reserved backend and must be registered with
|
||||
// [WithBackend].
|
||||
// BackendRakestrawHome identifies the maintained Rakestrawhome backend.
|
||||
const BackendRakestrawHome = backend.RakestrawHomeID
|
||||
|
||||
// BackendLocal is a conventional ID for a caller-configured local backend.
|
||||
const BackendLocal = "local"
|
||||
|
||||
// Backend configures one engine-scoped OpenAI-compatible backend.
|
||||
//
|
||||
// Backend has no stable JSON representation. Use keyed literals so additions
|
||||
// to this configuration value do not break source compatibility.
|
||||
type Backend struct {
|
||||
// ID is the stable, case-sensitive registry key. NewEngine trims it and
|
||||
// requires a non-blank value. BackendOpenRouter is reserved.
|
||||
ID string
|
||||
// Endpoint is the OpenAI-compatible base endpoint. NewEngine trims it and
|
||||
// requires an absolute HTTP or HTTPS URL with a host and without user
|
||||
// information, a query string, or a fragment. Paths are allowed.
|
||||
Endpoint string
|
||||
// APIKeyEnv optionally names the environment variable containing the API
|
||||
// key. NewEngine trims it and requires the portable form
|
||||
// [A-Za-z_][A-Za-z0-9_]*. Store only the name, never a credential value.
|
||||
APIKeyEnv string
|
||||
// ExtraParams contains backend-wide request defaults. Values must be
|
||||
// JSON-compatible, finite, acyclic, and keyed by non-empty strings. Keys
|
||||
// must not be model, session_id, messages, temperature, max_tokens, top_p,
|
||||
// service_tier, reasoning_effort, or response_format. An empty map supplies
|
||||
// no defaults. NewEngine deeply copies the map and rejects excessively deep
|
||||
// or large values for safety.
|
||||
ExtraParams map[string]any
|
||||
// ConcurrencyLimit is the maximum number of simultaneous model-generation
|
||||
// calls allowed for this backend within one Engine. Zero leaves the backend
|
||||
// unlimited. A negative value makes NewEngine fail with ErrInvalidConfig.
|
||||
ConcurrencyLimit int
|
||||
// QueueCapacity controls how many additional Run or RunPrepared calls may
|
||||
// be admitted beyond ConcurrencyLimit. Nil uses 1024 when ConcurrencyLimit
|
||||
// is positive; a pointer uses its exact value, including zero. The pointed-to
|
||||
// value must be non-negative, and QueueCapacity must be nil when
|
||||
// ConcurrencyLimit is zero. Their sum must fit in an int. WithBackend copies
|
||||
// the value and does not retain the pointer.
|
||||
// BackendConfig is a presence-aware convenience value for a keyed backend
|
||||
// definition. Nil pointer fields are omitted, while non-nil pointers preserve
|
||||
// explicit zero or blank values. A nil ExtraParams map is omitted; a non-nil
|
||||
// empty map supplies an empty mapping. Use raw ConfigKit Mapping inputs for
|
||||
// explicit null. BackendConfig has no stable JSON representation.
|
||||
type BackendConfig struct {
|
||||
// Endpoint is the required absolute HTTP(S) base URL after all layers merge.
|
||||
Endpoint *string
|
||||
// APIKeyEnv optionally names an environment variable, not a credential.
|
||||
APIKeyEnv *string
|
||||
// ConcurrencyLimit is zero for unlimited generation; negative is invalid.
|
||||
ConcurrencyLimit *int
|
||||
// QueueCapacity defaults to 1024 for positive concurrency when omitted.
|
||||
// An explicit value, including zero, requires positive concurrency.
|
||||
QueueCapacity *int
|
||||
// ExtraParams supplies JSON-compatible provider request defaults.
|
||||
ExtraParams map[string]any
|
||||
}
|
||||
|
||||
// LocalBackend returns a caller-owned Backend for a conventional local
|
||||
// OpenAI-compatible endpoint. It sets ID to BackendLocal and copies endpoint
|
||||
// and concurrencyLimit into Endpoint and ConcurrencyLimit without
|
||||
// normalization or validation. APIKeyEnv, ExtraParams, and QueueCapacity keep
|
||||
// their zero values.
|
||||
//
|
||||
// LocalBackend does not read environment variables, register the value, or
|
||||
// mutate engine or package state. Supply the returned value through
|
||||
// [WithBackend]; [NewEngine] then applies the ordinary backend validation and
|
||||
// concurrency semantics, including default queue capacity for a positive
|
||||
// limit, unlimited behavior for zero, and ErrInvalidConfig for a negative
|
||||
// limit.
|
||||
func LocalBackend(endpoint string, concurrencyLimit int) Backend {
|
||||
return Backend{
|
||||
ID: BackendLocal,
|
||||
Endpoint: endpoint,
|
||||
ConcurrencyLimit: concurrencyLimit,
|
||||
// BackendSource creates a labeled ConfigKit mapping from keyed backend values.
|
||||
// It encodes fields without validating them; NewEngine validates the final
|
||||
// effective configuration after all selected source layers are combined.
|
||||
// Mapping captures a copy of supported input values at creation; unsupported
|
||||
// values fail when the source is loaded.
|
||||
func BackendSource(label string, backends map[string]BackendConfig) configkit.Source {
|
||||
definitions := make(map[string]any, len(backends))
|
||||
for id, config := range backends {
|
||||
fields := make(map[string]any)
|
||||
if config.Endpoint != nil {
|
||||
fields["endpoint"] = *config.Endpoint
|
||||
}
|
||||
if config.APIKeyEnv != nil {
|
||||
fields["api_key_env"] = *config.APIKeyEnv
|
||||
}
|
||||
if config.ConcurrencyLimit != nil {
|
||||
fields["concurrency_limit"] = *config.ConcurrencyLimit
|
||||
}
|
||||
if config.QueueCapacity != nil {
|
||||
fields["queue_capacity"] = *config.QueueCapacity
|
||||
}
|
||||
if config.ExtraParams != nil {
|
||||
fields["extra_params"] = providerMapForMapping(config.ExtraParams)
|
||||
}
|
||||
definitions[id] = fields
|
||||
}
|
||||
return configkit.Mapping(label, map[string]any{"backends": definitions})
|
||||
}
|
||||
|
||||
// WithBackend adds one Backend registration to the constructed Engine.
|
||||
//
|
||||
// Registrations accumulate in option order. Every normalized ID must be unique
|
||||
// across consumer registrations and built-ins; a duplicate or invalid
|
||||
// definition makes NewEngine fail with ErrInvalidConfig. In particular,
|
||||
// BackendOpenRouter cannot be replaced. The immutable registration is scoped
|
||||
// to the resulting Engine and cannot be enumerated, replaced, removed, or
|
||||
// mutated after construction. WithBackend does not install package-global
|
||||
// state.
|
||||
func WithBackend(backend Backend) Option {
|
||||
queueCapacity := 0
|
||||
queueCapacitySet := backend.QueueCapacity != nil
|
||||
if queueCapacitySet {
|
||||
queueCapacity = *backend.QueueCapacity
|
||||
}
|
||||
return optionFunc(func(options *engineOptions) error {
|
||||
options.backends = append(options.backends, domain.Backend{
|
||||
ID: backend.ID,
|
||||
Endpoint: backend.Endpoint,
|
||||
APIKeyEnv: backend.APIKeyEnv,
|
||||
ExtraParams: backend.ExtraParams,
|
||||
ConcurrencyLimit: backend.ConcurrencyLimit,
|
||||
QueueCapacity: queueCapacity,
|
||||
QueueCapacitySet: queueCapacitySet,
|
||||
})
|
||||
return nil
|
||||
})
|
||||
// LocalBackend supplies endpoint and concurrency pointers for a caller-chosen
|
||||
// backend ID; it does not select an ID or validate either value.
|
||||
func LocalBackend(endpoint string, concurrencyLimit int) BackendConfig {
|
||||
return BackendConfig{Endpoint: &endpoint, ConcurrencyLimit: &concurrencyLimit}
|
||||
}
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
@@ -150,17 +151,7 @@ func TestUnlimitedBackendsRetainInjectedClientConcurrency(t *testing.T) {
|
||||
{
|
||||
name: "endpoint-only profile",
|
||||
configure: func(t *testing.T, client promptkit.LLMClient) *promptkit.Engine {
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{},
|
||||
promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."),
|
||||
promptkit.WithProfiles(promptkit.Profile{
|
||||
ID: "profile", Endpoint: "http://endpoint.example/v1", Model: "model",
|
||||
}),
|
||||
promptkit.WithLLMClient(client),
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("construct endpoint-only engine: %v", err)
|
||||
}
|
||||
return engine
|
||||
return newSimpleEngine(t, "message", promptkit.WithLLMClient(client))
|
||||
},
|
||||
},
|
||||
}
|
||||
@@ -250,27 +241,35 @@ func newBackendCapacityEngine(
|
||||
reader promptkit.ArtifactReader,
|
||||
) *promptkit.Engine {
|
||||
t.Helper()
|
||||
promptFS := contractPromptFS("prompt", "profile", "message")
|
||||
promptID, content := "prompt", "message"
|
||||
var inputs []any
|
||||
if reader != nil {
|
||||
promptFS = contractInputPromptFS()
|
||||
promptID, content = "input-prompt", `{{input "input"}}`
|
||||
inputs = []any{map[string]any{"name": "input", "required": true}}
|
||||
}
|
||||
options := []promptkit.Option{
|
||||
promptkit.WithPromptFS(promptFS, "."),
|
||||
promptkit.WithBackend(promptkit.Backend{
|
||||
ID: "limited",
|
||||
Endpoint: "http://backend.example/v1",
|
||||
ConcurrencyLimit: limit,
|
||||
QueueCapacity: queueCapacity,
|
||||
}),
|
||||
promptkit.WithProfiles(promptkit.Profile{
|
||||
ID: "profile", BackendID: "limited", Model: "model",
|
||||
}),
|
||||
promptkit.WithLLMClient(client),
|
||||
backend := map[string]any{"endpoint": "http://backend.example/v1", "concurrency_limit": limit}
|
||||
if queueCapacity != nil {
|
||||
backend["queue_capacity"] = *queueCapacity
|
||||
}
|
||||
definition := map[string]any{
|
||||
"default_profile": "profile",
|
||||
"messages": []any{map[string]any{"role": "user", "content": content}},
|
||||
"output": map[string]any{"format": "text", "validation_mode": "none"},
|
||||
}
|
||||
if inputs != nil {
|
||||
definition["inputs"] = inputs
|
||||
}
|
||||
options := []promptkit.Option{promptkit.WithLLMClient(client)}
|
||||
if reader != nil {
|
||||
options = append(options, promptkit.WithArtifactReader(reader))
|
||||
}
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{}, options...)
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
|
||||
Source: configkit.Mapping("capacity fixture", map[string]any{
|
||||
"backends": map[string]any{"limited": backend},
|
||||
"profiles": map[string]any{"profile": map[string]any{"backend": "limited", "model": "model"}},
|
||||
"prompts": map[string]any{promptID: map[string]any{"1": definition}},
|
||||
}),
|
||||
}}}, options...)
|
||||
if err != nil {
|
||||
t.Fatalf("construct capacity engine: %v", err)
|
||||
}
|
||||
|
||||
82
configuration_inspection.go
Normal file
82
configuration_inspection.go
Normal file
@@ -0,0 +1,82 @@
|
||||
package promptkit
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/configuration"
|
||||
)
|
||||
|
||||
// ConfigurationInspection is a read-only view of one Engine's captured
|
||||
// effective configuration and source history. It does not expose typed
|
||||
// configuration values, filesystem contexts, or runtime credentials.
|
||||
// Report content may include application prompt and provider data, so callers
|
||||
// choose where and to whom to disclose it.
|
||||
type ConfigurationInspection struct {
|
||||
source configuration.Inspection
|
||||
}
|
||||
|
||||
// InspectConfiguration returns the ConfigKit inspection view captured by this
|
||||
// Engine. It does not reload sources, resolve credentials, or contact a model.
|
||||
// A nil Engine returns ErrInvalidConfig.
|
||||
func (e *Engine) InspectConfiguration() (ConfigurationInspection, error) {
|
||||
if e == nil || e.internal == nil {
|
||||
return ConfigurationInspection{}, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
|
||||
}
|
||||
return ConfigurationInspection{source: e.internal.InspectConfiguration()}, nil
|
||||
}
|
||||
|
||||
// Metadata returns detached effective presence and history records.
|
||||
func (v ConfigurationInspection) Metadata() configkit.Metadata {
|
||||
if v.source == nil {
|
||||
return configkit.Metadata{}
|
||||
}
|
||||
return v.source.Metadata()
|
||||
}
|
||||
|
||||
// Sources returns a detached list of loaded source descriptions.
|
||||
func (v ConfigurationInspection) Sources() []configkit.SourceInfo {
|
||||
if v.source == nil {
|
||||
return nil
|
||||
}
|
||||
return v.source.Sources()
|
||||
}
|
||||
|
||||
// Diagnostics returns detached nonfatal load diagnostics.
|
||||
func (v ConfigurationInspection) Diagnostics() []configkit.Diagnostic {
|
||||
if v.source == nil {
|
||||
return nil
|
||||
}
|
||||
return v.source.Diagnostics()
|
||||
}
|
||||
|
||||
// Explain returns detached effective and historical values for a known path.
|
||||
// Dynamic credential-shaped provider keys are redacted by ConfigKit; ordinary
|
||||
// prompt and provider content is not automatically safe to disclose.
|
||||
func (v ConfigurationInspection) Explain(path configkit.Path) (configkit.Explanation, bool) {
|
||||
if v.source == nil {
|
||||
return configkit.Explanation{}, false
|
||||
}
|
||||
return v.source.Explain(path)
|
||||
}
|
||||
|
||||
// WriteEffective writes ConfigKit's redacted effective YAML. Writer errors
|
||||
// remain available through errors.Is and errors.As. The zero view returns
|
||||
// ErrInvalidConfig.
|
||||
func (v ConfigurationInspection) WriteEffective(w io.Writer) error {
|
||||
if v.source == nil {
|
||||
return ErrInvalidConfig
|
||||
}
|
||||
return v.source.WriteEffective(w)
|
||||
}
|
||||
|
||||
// WriteExplanation writes ConfigKit's redacted history for path. Unknown paths
|
||||
// produce no output; writer errors remain inspectable. The zero view returns
|
||||
// ErrInvalidConfig.
|
||||
func (v ConfigurationInspection) WriteExplanation(w io.Writer, path configkit.Path) error {
|
||||
if v.source == nil {
|
||||
return ErrInvalidConfig
|
||||
}
|
||||
return v.source.WriteExplanation(w, path)
|
||||
}
|
||||
200
configuration_inspection_contract_test.go
Normal file
200
configuration_inspection_contract_test.go
Normal file
@@ -0,0 +1,200 @@
|
||||
package promptkit_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"testing/fstest"
|
||||
|
||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
func TestConfigurationInspectionReportsLiteralKeysAndRedactedHistory(t *testing.T) {
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||
Underlays: []configkit.Underlay{
|
||||
{Inputs: []configkit.Input{{Source: configkit.Mapping("defaults", map[string]any{
|
||||
"backends": map[string]any{"local": map[string]any{
|
||||
"endpoint": "https://example.invalid/v1", "api_key_env": "OPTIONAL_KEY",
|
||||
"extra_params": map[string]any{"token": "private-sentinel", "visible": "public-value"},
|
||||
}},
|
||||
"profiles": map[string]any{
|
||||
"base.v1": map[string]any{"backend": "local", "model": "default-model"},
|
||||
"literal/id": map[string]any{"extends": "base.v1"},
|
||||
},
|
||||
})}}},
|
||||
{Inputs: []configkit.Input{{Source: configkit.Mapping("operator", map[string]any{
|
||||
"profiles": map[string]any{"base.v1": map[string]any{"model": "operator-model"}},
|
||||
})}}},
|
||||
},
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatalf("construct layered engine: %v", err)
|
||||
}
|
||||
view, err := engine.InspectConfiguration()
|
||||
if err != nil {
|
||||
t.Fatalf("inspect configuration: %v", err)
|
||||
}
|
||||
modelPath := configkit.Keys("profiles", "base.v1", "model")
|
||||
model, found := view.Explain(modelPath)
|
||||
if !found || !strings.Contains(model.Effective.YAML, "operator-model") || len(model.History) < 2 {
|
||||
t.Fatalf("layered model explanation = (%+v, %t)", model, found)
|
||||
}
|
||||
inheritedPath := configkit.Keys("profiles", "literal/id", "model")
|
||||
inherited, found := view.Explain(inheritedPath)
|
||||
if !found || !strings.Contains(inherited.Effective.YAML, "operator-model") || len(inherited.History) == 0 || !view.Metadata().IsExplicit(inheritedPath) {
|
||||
t.Fatalf("literal-key inheritance explanation = (%+v, %t)", inherited, found)
|
||||
}
|
||||
if inheritedPath.String() != "/profiles/literal~1id/model" {
|
||||
t.Fatalf("literal-key path = %q", inheritedPath)
|
||||
}
|
||||
secretPath := configkit.Keys("backends", "local", "extra_params", "token")
|
||||
secret, found := view.Explain(secretPath)
|
||||
if !found || !secret.Effective.Redacted || strings.Contains(secret.Effective.YAML, "private-sentinel") {
|
||||
t.Fatalf("credential-shaped provider value was not redacted: (%+v, %t)", secret, found)
|
||||
}
|
||||
var effective bytes.Buffer
|
||||
if err := view.WriteEffective(&effective); err != nil {
|
||||
t.Fatalf("write effective configuration: %v", err)
|
||||
}
|
||||
if strings.Contains(effective.String(), "private-sentinel") || !strings.Contains(effective.String(), "OPTIONAL_KEY") || !strings.Contains(effective.String(), "public-value") {
|
||||
t.Fatalf("unexpected effective report: %s", effective.String())
|
||||
}
|
||||
var explanation bytes.Buffer
|
||||
if err := view.WriteExplanation(&explanation, secretPath); err != nil || strings.Contains(explanation.String(), "private-sentinel") {
|
||||
t.Fatalf("secret explanation = (%q, %v)", explanation.String(), err)
|
||||
}
|
||||
|
||||
sources := view.Sources()
|
||||
if len(sources) == 0 {
|
||||
t.Fatal("inspection returned no sources")
|
||||
}
|
||||
originalLabel := sources[0].Label
|
||||
sources[0].Label = "changed"
|
||||
if view.Sources()[0].Label != originalLabel {
|
||||
t.Fatal("source mutation reached retained inspection")
|
||||
}
|
||||
model.History[0].After.YAML = "changed"
|
||||
if again, ok := view.Explain(modelPath); !ok || again.History[0].After.YAML == "changed" {
|
||||
t.Fatal("explanation mutation reached retained inspection")
|
||||
}
|
||||
history := view.Metadata().History(modelPath)
|
||||
if len(history) == 0 {
|
||||
t.Fatal("metadata history is empty")
|
||||
}
|
||||
history[0].Reasons = []string{"changed"}
|
||||
if again := view.Metadata().History(modelPath); len(again) == 0 || (len(again[0].Reasons) > 0 && again[0].Reasons[0] == "changed") {
|
||||
t.Fatal("metadata mutation reached retained inspection")
|
||||
}
|
||||
}
|
||||
|
||||
type failingInspectionWriter struct{ err error }
|
||||
|
||||
func (writer failingInspectionWriter) Write([]byte) (int, error) { return 0, writer.err }
|
||||
|
||||
func TestConfigurationInspectionZeroAndWriterErrors(t *testing.T) {
|
||||
var engine *promptkit.Engine
|
||||
if _, err := engine.InspectConfiguration(); !errors.Is(err, promptkit.ErrInvalidConfig) {
|
||||
t.Fatalf("nil engine inspection = %v", err)
|
||||
}
|
||||
var empty promptkit.ConfigurationInspection
|
||||
path := configkit.Keys("profiles", "missing")
|
||||
if _, found := empty.Metadata().Lookup(path); found || len(empty.Sources()) != 0 || len(empty.Diagnostics()) != 0 {
|
||||
t.Fatal("zero inspection reported configuration")
|
||||
}
|
||||
if _, found := empty.Explain(path); found {
|
||||
t.Fatal("zero inspection explained an unknown path")
|
||||
}
|
||||
if err := empty.WriteEffective(io.Discard); !errors.Is(err, promptkit.ErrInvalidConfig) {
|
||||
t.Fatalf("zero effective writer = %v", err)
|
||||
}
|
||||
if err := empty.WriteExplanation(io.Discard, path); !errors.Is(err, promptkit.ErrInvalidConfig) {
|
||||
t.Fatalf("zero explanation writer = %v", err)
|
||||
}
|
||||
|
||||
valid, err := promptkit.NewEngine(promptkit.Config{})
|
||||
if err != nil {
|
||||
t.Fatalf("construct engine: %v", err)
|
||||
}
|
||||
view, err := valid.InspectConfiguration()
|
||||
if err != nil {
|
||||
t.Fatalf("inspect configuration: %v", err)
|
||||
}
|
||||
writeErr := errors.New("report writer failed")
|
||||
writer := failingInspectionWriter{err: writeErr}
|
||||
if err := view.WriteEffective(writer); !errors.Is(err, writeErr) {
|
||||
t.Fatalf("effective writer error = %v", err)
|
||||
}
|
||||
if err := view.WriteExplanation(writer, configkit.Keys("profiles", "mistral-small-3", "model")); !errors.Is(err, writeErr) {
|
||||
t.Fatalf("explanation writer error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigurationInspectionUsesCapturedSourcesWithoutProviderCalls(t *testing.T) {
|
||||
source := &unreadableAfterConstructionFS{FS: fstest.MapFS{
|
||||
"config/prompt.yaml": &fstest.MapFile{Data: []byte(capturedPromptDocument)},
|
||||
"config/body.tmpl": &fstest.MapFile{Data: []byte("hello")},
|
||||
}}
|
||||
inputs, err := promptkit.FilesystemInputs("inspection source", source, "config")
|
||||
if err != nil {
|
||||
t.Fatalf("select inputs: %v", err)
|
||||
}
|
||||
var modelCalls atomic.Int64
|
||||
client := countingLLMClient{calls: &modelCalls}
|
||||
endpoint, model := "https://example.invalid/v1", "offline-model"
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||
Underlays: []configkit.Underlay{{Inputs: inputs}, {Inputs: []configkit.Input{{Source: promptkit.ProfileSource("offline profile", map[string]promptkit.ProfileConfig{
|
||||
"offline": {Endpoint: &endpoint, Model: &model},
|
||||
})}}}},
|
||||
}}, promptkit.WithLLMClient(client))
|
||||
if err != nil {
|
||||
t.Fatalf("construct engine: %v", err)
|
||||
}
|
||||
source.closed.Store(true)
|
||||
view, err := engine.InspectConfiguration()
|
||||
if err != nil {
|
||||
t.Fatalf("inspect configuration: %v", err)
|
||||
}
|
||||
if _, found := view.Explain(configkit.Keys("prompts", "captured", "1", "messages").Index(0).Key("content_file")); !found {
|
||||
t.Fatal("captured prompt content path was not inspectable")
|
||||
}
|
||||
var report bytes.Buffer
|
||||
if err := view.WriteEffective(&report); err != nil || !strings.Contains(report.String(), "captured") {
|
||||
t.Fatalf("write captured report = (%q, %v)", report.String(), err)
|
||||
}
|
||||
if source.reads.Load() != 0 || modelCalls.Load() != 0 {
|
||||
t.Fatalf("inspection touched source or provider: reads=%d requests=%d", source.reads.Load(), modelCalls.Load())
|
||||
}
|
||||
|
||||
const calls = 16
|
||||
var wait sync.WaitGroup
|
||||
errs := make(chan error, calls)
|
||||
for index := 0; index < calls; index++ {
|
||||
wait.Add(1)
|
||||
go func(index int) {
|
||||
defer wait.Done()
|
||||
if index%2 == 0 {
|
||||
_, err := engine.InspectConfiguration()
|
||||
errs <- err
|
||||
return
|
||||
}
|
||||
_, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "captured", SessionID: "direct"})
|
||||
errs <- err
|
||||
}(index)
|
||||
}
|
||||
wait.Wait()
|
||||
close(errs)
|
||||
for err := range errs {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if source.reads.Load() != 0 || modelCalls.Load() != calls/2 {
|
||||
t.Fatalf("concurrent inspection/execution = source reads %d, model calls %d", source.reads.Load(), modelCalls.Load())
|
||||
}
|
||||
}
|
||||
83
configuration_sources.go
Normal file
83
configuration_sources.go
Normal file
@@ -0,0 +1,83 @@
|
||||
package promptkit
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/filecatalog"
|
||||
)
|
||||
|
||||
// DirectoryInputs selects the YAML files beneath an explicitly chosen host
|
||||
// directory as one strict-composition group. The returned order is deterministic;
|
||||
// it does not assign precedence among those files. Discovery is recursive over
|
||||
// .yaml and .yml files. An unavailable root fails; an empty directory yields
|
||||
// no inputs. Imports are disabled unless callers enable them on the inputs.
|
||||
func DirectoryInputs(directory string) ([]configkit.Input, error) {
|
||||
root, err := filepath.Abs(directory)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("resolve configuration directory: %w", err)
|
||||
}
|
||||
info, err := os.Stat(root)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("inspect configuration directory: %w", err)
|
||||
}
|
||||
if !info.IsDir() {
|
||||
return nil, fmt.Errorf("configuration root is not a directory: %s", root)
|
||||
}
|
||||
root, err = filepath.EvalSymlinks(root)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("resolve configuration directory: %w", err)
|
||||
}
|
||||
files, err := filecatalog.FindYAMLFiles(context.Background(), root)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("discover configuration files: %w", err)
|
||||
}
|
||||
inputs := make([]configkit.Input, 0, len(files))
|
||||
for _, file := range files {
|
||||
inputs = append(inputs, configkit.Input{
|
||||
Source: configkit.File(file),
|
||||
Filesystem: &configkit.FilesystemContext{Root: root},
|
||||
})
|
||||
}
|
||||
return inputs, nil
|
||||
}
|
||||
|
||||
// FilesystemInputs selects the YAML files beneath root in a caller-owned
|
||||
// filesystem as one strict-composition group. Discovery is recursive over
|
||||
// .yaml and .yml files, with deterministic ordering and no precedence among
|
||||
// files. The caller retains ownership of the filesystem; an unavailable root
|
||||
// fails and an empty directory yields no inputs. Imports remain disabled unless
|
||||
// callers enable them on inputs.
|
||||
func FilesystemInputs(label string, filesystem fs.FS, root string) ([]configkit.Input, error) {
|
||||
if filesystem == nil {
|
||||
return nil, fmt.Errorf("configuration filesystem is nil")
|
||||
}
|
||||
root = filecatalog.CleanFSRoot(root)
|
||||
if !fs.ValidPath(root) {
|
||||
return nil, fmt.Errorf("configuration root is not a valid filesystem path: %s", root)
|
||||
}
|
||||
info, err := fs.Stat(filesystem, root)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("inspect configuration root: %w", err)
|
||||
}
|
||||
if !info.IsDir() {
|
||||
return nil, fmt.Errorf("configuration root is not a directory: %s", root)
|
||||
}
|
||||
files, err := filecatalog.FindFSYAMLFiles(context.Background(), filesystem, root)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("discover configuration files: %w", err)
|
||||
}
|
||||
namespace := configkit.NewFilesystem(label, filesystem)
|
||||
inputs := make([]configkit.Input, 0, len(files))
|
||||
for _, file := range files {
|
||||
inputs = append(inputs, configkit.Input{
|
||||
Source: namespace.File(file),
|
||||
Filesystem: &configkit.FilesystemContext{FS: &namespace, Root: root},
|
||||
})
|
||||
}
|
||||
return inputs, nil
|
||||
}
|
||||
83
configuration_sources_test.go
Normal file
83
configuration_sources_test.go
Normal file
@@ -0,0 +1,83 @@
|
||||
package promptkit_test
|
||||
|
||||
import (
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"testing/fstest"
|
||||
|
||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
type sourceDocument struct {
|
||||
First string `yaml:"first"`
|
||||
Second string `yaml:"second"`
|
||||
}
|
||||
|
||||
func TestDirectoryInputsSelectsSortedFilesWithRootContext(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
if err := os.Mkdir(filepath.Join(root, "nested"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for name, data := range map[string]string{
|
||||
"a.yaml": "first: one\n", "nested/b.yml": "second: two\n", "nested/body.md": "message",
|
||||
} {
|
||||
if err := os.WriteFile(filepath.Join(root, name), []byte(data), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
inputs, err := promptkit.DirectoryInputs(root)
|
||||
if err != nil || len(inputs) != 2 || inputs[0].Imports || inputs[1].Imports {
|
||||
t.Fatalf("directory inputs: %+v, %v", inputs, err)
|
||||
}
|
||||
assertSourceInputs(t, inputs, "message")
|
||||
if _, err := promptkit.DirectoryInputs(filepath.Join(root, "missing")); err == nil {
|
||||
t.Fatal("missing selected directory succeeded")
|
||||
}
|
||||
if empty, err := promptkit.DirectoryInputs(t.TempDir()); err != nil || len(empty) != 0 {
|
||||
t.Fatalf("empty directory: %+v, %v", empty, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFilesystemInputsSelectsEmbeddedFilesWithRootContext(t *testing.T) {
|
||||
files := fstest.MapFS{
|
||||
"settings/a.yaml": {Data: []byte("first: one\n")},
|
||||
"settings/nested/b.yml": {Data: []byte("second: two\n")},
|
||||
"settings/nested/body.md": {Data: []byte("message")},
|
||||
}
|
||||
inputs, err := promptkit.FilesystemInputs("embedded", files, "settings")
|
||||
if err != nil || len(inputs) != 2 || inputs[0].Imports || inputs[1].Imports {
|
||||
t.Fatalf("filesystem inputs: %+v, %v", inputs, err)
|
||||
}
|
||||
assertSourceInputs(t, inputs, "message")
|
||||
if _, err := promptkit.FilesystemInputs("embedded", files, "missing"); err == nil {
|
||||
t.Fatal("missing selected root succeeded")
|
||||
}
|
||||
}
|
||||
|
||||
func assertSourceInputs(t *testing.T, inputs []configkit.Input, expected string) {
|
||||
t.Helper()
|
||||
result, err := configkit.LoadLayered(configkit.Input{Source: configkit.NoSource()}, configkit.LayeredOptions[sourceDocument]{
|
||||
Underlays: []configkit.Underlay{{Inputs: inputs}},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result.Config.First != "one" || result.Config.Second != "two" {
|
||||
t.Fatalf("configuration files did not compose: %+v", result.Config)
|
||||
}
|
||||
field, found := result.Metadata().Lookup(configkit.Keys("second"))
|
||||
if !found || len(field.Origin.Locations) != 1 {
|
||||
t.Fatalf("missing second-field origin: %+v", field)
|
||||
}
|
||||
context, found := result.SourceContext(field.Origin.Locations[0].SourceID)
|
||||
if !found || context.BaseDir != "nested" {
|
||||
t.Fatalf("unexpected rooted source context: %+v", context)
|
||||
}
|
||||
data, err := fs.ReadFile(context.FS, "nested/body.md")
|
||||
if err != nil || string(data) != expected {
|
||||
t.Fatalf("rooted resource read: %q, %v", data, err)
|
||||
}
|
||||
}
|
||||
231
construction_contract_test.go
Normal file
231
construction_contract_test.go
Normal file
@@ -0,0 +1,231 @@
|
||||
package promptkit_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"testing/fstest"
|
||||
|
||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
func TestProfileEnvironmentLookupAcceptsNonShellNames(t *testing.T) {
|
||||
const name = "promptkit.test-api-key"
|
||||
for _, test := range []struct {
|
||||
name, value, direct, want string
|
||||
unset, clear bool
|
||||
}{
|
||||
{name: "set", value: " synthetic-key ", want: "Bearer synthetic-key"},
|
||||
{name: "empty"},
|
||||
{name: "whitespace", value: " "},
|
||||
{name: "unset", unset: true},
|
||||
{name: "direct precedence", value: "synthetic-key", direct: "direct-key", want: "Bearer direct-key"},
|
||||
{name: "explicit clear", value: "synthetic-key", clear: true},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
t.Setenv(name, test.value)
|
||||
if test.unset {
|
||||
if err := os.Unsetenv(name); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
cfg := simpleConfig("hello")
|
||||
cfg.Sources.Underlays = []configkit.Underlay{{Inputs: []configkit.Input{*cfg.Sources.Main}}}
|
||||
lowerName := name
|
||||
cfg.Sources.Underlays = append(cfg.Sources.Underlays, configkit.Underlay{Inputs: []configkit.Input{{
|
||||
Source: promptkit.ProfileSource("default credentials", map[string]promptkit.ProfileConfig{
|
||||
"profile": {APIKeyEnv: &lowerName},
|
||||
}),
|
||||
}}})
|
||||
env := " " + name + " "
|
||||
if test.clear {
|
||||
env = ""
|
||||
}
|
||||
cfg.Sources.Main = &configkit.Input{Source: promptkit.ProfileSource("credentials", map[string]promptkit.ProfileConfig{
|
||||
"profile": {APIKeyEnv: &env},
|
||||
})}
|
||||
var calls int
|
||||
var auth string
|
||||
cfg.HTTPClient = &http.Client{Transport: roundTripFunc(func(request *http.Request) (*http.Response, error) {
|
||||
calls++
|
||||
auth = request.Header.Get("Authorization")
|
||||
return &http.Response{StatusCode: http.StatusOK, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{"choices":[{"message":{"content":"ok"}}]}`))}, nil
|
||||
})}
|
||||
engine, err := promptkit.NewEngine(cfg)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt", APIKey: test.direct})
|
||||
if err != nil || calls != 1 || auth != test.want {
|
||||
t.Fatalf("run: err=%v calls=%d authorization=%q, want %q", err, calls, auth, test.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPublicConstructionWithLocalBackendAndProgrammaticProfile(t *testing.T) {
|
||||
backendID, model := "workstation", "offline-model"
|
||||
prompt := configkit.Input{Source: configkit.Mapping("prompt", map[string]any{
|
||||
"prompts": map[string]any{"hello": map[string]any{"1": map[string]any{
|
||||
"default_profile": "offline",
|
||||
"messages": []any{map[string]any{"role": "user", "content": "Hello"}},
|
||||
"output": map[string]any{"format": "text", "validation_mode": "none"},
|
||||
}}},
|
||||
})}
|
||||
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "reply"}}
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{
|
||||
{Source: promptkit.BackendSource("local backend", map[string]promptkit.BackendConfig{
|
||||
backendID: promptkit.LocalBackend("http://localhost:8000/v1", 2),
|
||||
})},
|
||||
{Source: promptkit.ProfileSource("profile", map[string]promptkit.ProfileConfig{
|
||||
"offline": {Backend: &backendID, Model: &model},
|
||||
})},
|
||||
}}},
|
||||
Main: &prompt,
|
||||
}}, promptkit.WithLLMClient(client))
|
||||
if err != nil {
|
||||
t.Fatalf("construct engine: %v", err)
|
||||
}
|
||||
result, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "hello"})
|
||||
if err != nil {
|
||||
t.Fatalf("run prompt: %v", err)
|
||||
}
|
||||
if result.RawOutput != "reply" || result.SelectedBackendID != backendID || result.EffectiveModelParams.Model != model {
|
||||
t.Fatalf("unexpected result: %+v", result)
|
||||
}
|
||||
if len(client.requests) != 1 || client.requests[0].Target.Endpoint != "http://localhost:8000/v1" {
|
||||
t.Fatalf("unexpected model request: %+v", client.requests)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPublicConstructionCombinesEmbeddedOperatorAndProgrammaticInputs(t *testing.T) {
|
||||
embedded, err := promptkit.FilesystemInputs("embedded defaults", fstest.MapFS{
|
||||
"config/default.yaml": &fstest.MapFile{Data: []byte(`
|
||||
profiles:
|
||||
application:
|
||||
endpoint: https://example.invalid/v1
|
||||
model: default-model
|
||||
prompts:
|
||||
hello:
|
||||
"1":
|
||||
default_profile: application
|
||||
messages:
|
||||
- role: user
|
||||
content: Hello
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
`)},
|
||||
}, "config")
|
||||
if err != nil {
|
||||
t.Fatalf("select embedded defaults: %v", err)
|
||||
}
|
||||
operatorDirectory := t.TempDir()
|
||||
if err := os.WriteFile(filepath.Join(operatorDirectory, "override.yaml"), []byte("profiles:\n application:\n model: operator-model\n"), 0o644); err != nil {
|
||||
t.Fatalf("write operator input: %v", err)
|
||||
}
|
||||
operator, err := promptkit.DirectoryInputs(operatorDirectory)
|
||||
if err != nil {
|
||||
t.Fatalf("select operator inputs: %v", err)
|
||||
}
|
||||
programmaticModel := "programmatic-model"
|
||||
programmatic := configkit.Input{Source: promptkit.ProfileSource("programmatic settings", map[string]promptkit.ProfileConfig{
|
||||
"application": {Model: &programmaticModel},
|
||||
})}
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
apply bool
|
||||
want string
|
||||
}{
|
||||
{name: "embedded", want: "default-model"},
|
||||
{name: "operator", apply: true, want: "operator-model"},
|
||||
{name: "programmatic", apply: true, want: "programmatic-model"},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
underlays := []configkit.Underlay{{Inputs: embedded}}
|
||||
if test.apply {
|
||||
underlays = append(underlays, configkit.Underlay{Inputs: operator})
|
||||
}
|
||||
if test.name == "programmatic" {
|
||||
underlays = append(underlays, configkit.Underlay{Inputs: []configkit.Input{programmatic}})
|
||||
}
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: underlays}})
|
||||
if err != nil {
|
||||
t.Fatalf("construct engine: %v", err)
|
||||
}
|
||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "hello"})
|
||||
if err != nil {
|
||||
t.Fatalf("prepare prompt: %v", err)
|
||||
}
|
||||
if prepared.EffectiveModelParams.Model != test.want {
|
||||
t.Fatalf("model = %q, want %q", prepared.EffectiveModelParams.Model, test.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPublicConstructionPreservesConfigKitLoadError(t *testing.T) {
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
|
||||
Source: configkit.Mapping("invalid", map[string]any{"profiles": map[string]any{"broken": map[string]any{"unknown": true}}}),
|
||||
}}})
|
||||
var loadErr *configkit.LoadError
|
||||
if engine != nil || !errors.Is(err, promptkit.ErrInvalidConfig) || !errors.As(err, &loadErr) {
|
||||
t.Fatalf("construction = (%#v, %v), want invalid config preserving LoadError", engine, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPublicConstructionRejectsSuppliedInvalidMainInput(t *testing.T) {
|
||||
main := configkit.Input{}
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &main}})
|
||||
if engine != nil || !errors.Is(err, promptkit.ErrInvalidConfig) {
|
||||
t.Fatalf("construction = (%#v, %v), want invalid supplied main input", engine, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPublicConstructionCanOverlayMaintainedBackend(t *testing.T) {
|
||||
endpoint := "http://local-openrouter.example/v1"
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: promptkit.BackendSource("operator backend", map[string]promptkit.BackendConfig{
|
||||
promptkit.BackendOpenRouter: {Endpoint: &endpoint},
|
||||
})}}}},
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatalf("construct engine with backend overlay: %v", err)
|
||||
}
|
||||
inspection, err := engine.InspectProfile(context.Background(), "mistral-small-3")
|
||||
if err != nil {
|
||||
t.Fatalf("inspect catalog profile: %v", err)
|
||||
}
|
||||
if inspection.EffectiveModelParams.Endpoint != endpoint {
|
||||
t.Fatalf("effective endpoint = %q, want %q", inspection.EffectiveModelParams.Endpoint, endpoint)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreparationRejectsAmbiguousPromptVersion(t *testing.T) {
|
||||
definition := func(content string) map[string]any {
|
||||
return map[string]any{
|
||||
"messages": []any{map[string]any{"role": "user", "content": content}},
|
||||
"output": map[string]any{"format": "text", "validation_mode": "none"},
|
||||
}
|
||||
}
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||
Main: &configkit.Input{Source: configkit.Mapping("versions", map[string]any{
|
||||
"prompts": map[string]any{"hello": map[string]any{
|
||||
"1": definition("first"), "2": definition("second"),
|
||||
}},
|
||||
})},
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatalf("construct engine: %v", err)
|
||||
}
|
||||
if result, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "hello"}); result != nil || !errors.Is(err, promptkit.ErrPromptLoad) {
|
||||
t.Fatalf("ambiguous preparation = (%#v, %v), want ErrPromptLoad", result, err)
|
||||
}
|
||||
}
|
||||
73
consumer_test_helpers_test.go
Normal file
73
consumer_test_helpers_test.go
Normal file
@@ -0,0 +1,73 @@
|
||||
package promptkit_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
type fakeLLMClient struct {
|
||||
response *promptkit.GenerateResponse
|
||||
responses []*promptkit.GenerateResponse
|
||||
err error
|
||||
requests []promptkit.GenerateRequest
|
||||
}
|
||||
|
||||
func (f *fakeLLMClient) Generate(_ context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
f.requests = append(f.requests, request)
|
||||
if f.err != nil {
|
||||
return nil, f.err
|
||||
}
|
||||
if len(f.responses) > 0 {
|
||||
index := len(f.requests) - 1
|
||||
if index >= len(f.responses) {
|
||||
return nil, fmt.Errorf("no response configured for generation %d", index+1)
|
||||
}
|
||||
return f.responses[index], nil
|
||||
}
|
||||
return f.response, nil
|
||||
}
|
||||
|
||||
type countingLLMClient struct{ calls *atomic.Int64 }
|
||||
|
||||
func (client countingLLMClient) Generate(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
if client.calls != nil {
|
||||
client.calls.Add(1)
|
||||
}
|
||||
return &promptkit.GenerateResponse{Content: "ok"}, nil
|
||||
}
|
||||
|
||||
type roundTripFunc func(*http.Request) (*http.Response, error)
|
||||
|
||||
func (f roundTripFunc) RoundTrip(request *http.Request) (*http.Response, error) { return f(request) }
|
||||
|
||||
// newSimpleEngine exercises the public constructor with a complete keyed
|
||||
// document. Tests that need layering or resources select their own inputs.
|
||||
func newSimpleEngine(t *testing.T, content string, options ...promptkit.Option) *promptkit.Engine {
|
||||
t.Helper()
|
||||
engine, err := promptkit.NewEngine(simpleConfig(content), options...)
|
||||
if err != nil {
|
||||
t.Fatalf("construct consumer test engine: %v", err)
|
||||
}
|
||||
return engine
|
||||
}
|
||||
|
||||
func simpleConfig(content string) promptkit.Config {
|
||||
return promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||
Main: &configkit.Input{Source: configkit.Mapping("consumer test", map[string]any{
|
||||
"profiles": map[string]any{"profile": map[string]any{
|
||||
"endpoint": "http://example.test/v1", "model": "model",
|
||||
}},
|
||||
"prompts": map[string]any{"prompt": map[string]any{"1": map[string]any{
|
||||
"default_profile": "profile",
|
||||
"messages": []any{map[string]any{"role": "user", "content": content}},
|
||||
"output": map[string]any{"format": "text", "validation_mode": "none"},
|
||||
}}},
|
||||
})},
|
||||
}}
|
||||
}
|
||||
88
convert.go
88
convert.go
@@ -1,7 +1,9 @@
|
||||
package promptkit
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"reflect"
|
||||
"unicode/utf8"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/jsonvalue"
|
||||
@@ -12,19 +14,59 @@ func toDomainRunRequest(req RunRequest) (domain.RunRequest, error) {
|
||||
if err != nil {
|
||||
return domain.RunRequest{}, err
|
||||
}
|
||||
appendedMessages, err := toDomainAppendedMessages(req.AppendedMessages)
|
||||
if err != nil {
|
||||
return domain.RunRequest{}, err
|
||||
}
|
||||
var observer domain.ExecutionObserver
|
||||
if req.Observer != nil {
|
||||
callback := req.Observer
|
||||
observer = func(event domain.ExecutionEvent) { callback(fromDomainExecutionEvent(event)) }
|
||||
}
|
||||
return domain.RunRequest{
|
||||
PromptID: req.PromptID,
|
||||
PromptVersion: req.PromptVersion,
|
||||
ProfileID: req.ProfileID,
|
||||
SessionID: req.SessionID,
|
||||
APIKey: req.APIKey,
|
||||
Inputs: toDomainArtifactRefMap(req.Inputs),
|
||||
Vars: copyStringMap(req.Vars),
|
||||
Execution: execution,
|
||||
Validation: toDomainOutputContractPtr(req.Validation),
|
||||
PromptID: req.PromptID,
|
||||
PromptVersion: req.PromptVersion,
|
||||
ProfileID: req.ProfileID,
|
||||
SessionID: req.SessionID,
|
||||
APIKey: req.APIKey,
|
||||
Inputs: toDomainArtifactRefMap(req.Inputs),
|
||||
Vars: copyStringMap(req.Vars),
|
||||
Execution: execution,
|
||||
Validation: toDomainOutputContractPtr(req.Validation),
|
||||
AppendedMessages: appendedMessages,
|
||||
TransientRetries: copyIntPtr(req.TransientRetries),
|
||||
Observer: observer,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func toDomainAppendedMessages(messages []RenderedMessage) ([]domain.RenderedMessage, error) {
|
||||
if messages == nil {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
converted := make([]domain.RenderedMessage, len(messages))
|
||||
for index, message := range messages {
|
||||
if !utf8.ValidString(message.Content) {
|
||||
return nil, fmt.Errorf("appended message %d content must be valid UTF-8", index)
|
||||
}
|
||||
role, err := domain.NormalizeMessageRole(message.Role)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("appended message %d role: %w", index, err)
|
||||
}
|
||||
|
||||
cacheControl, err := domain.NormalizeCacheControl(toDomainCacheControl(message.CacheControl))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("appended message %d cache_control: %w", index, err)
|
||||
}
|
||||
converted[index] = domain.RenderedMessage{
|
||||
Role: role,
|
||||
Content: message.Content,
|
||||
CacheControl: cacheControl,
|
||||
}
|
||||
}
|
||||
return converted, nil
|
||||
}
|
||||
|
||||
func fromDomainPreparedRun(prepared *domain.PreparedRun) *PreparedRun {
|
||||
if prepared == nil {
|
||||
return nil
|
||||
@@ -90,8 +132,22 @@ func toDomainGenerateResponse(resp *GenerateResponse) *domain.GenerateResponse {
|
||||
return nil
|
||||
}
|
||||
return &domain.GenerateResponse{
|
||||
Content: resp.Content,
|
||||
Usage: toDomainTokenUsage(resp.Usage),
|
||||
Content: resp.Content,
|
||||
Usage: toDomainTokenUsage(resp.Usage),
|
||||
UsagePresence: toDomainTokenUsagePresence(resp.UsagePresence),
|
||||
}
|
||||
}
|
||||
|
||||
func toDomainTokenUsagePresence(presence *TokenUsagePresence) *domain.TokenUsagePresence {
|
||||
if presence == nil {
|
||||
return nil
|
||||
}
|
||||
return &domain.TokenUsagePresence{
|
||||
PromptTokens: presence.PromptTokens,
|
||||
CompletionTokens: presence.CompletionTokens,
|
||||
TotalTokens: presence.TotalTokens,
|
||||
CachedTokens: presence.CachedTokens,
|
||||
CacheWriteTokens: presence.CacheWriteTokens,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -285,6 +341,16 @@ func fromDomainRenderedMessages(messages []domain.RenderedMessage) []RenderedMes
|
||||
return out
|
||||
}
|
||||
|
||||
func toDomainCacheControl(cacheControl *CacheControl) *domain.CacheControl {
|
||||
if cacheControl == nil {
|
||||
return nil
|
||||
}
|
||||
return &domain.CacheControl{
|
||||
Type: domain.CacheControlType(cacheControl.Type),
|
||||
TTL: cacheControl.TTL,
|
||||
}
|
||||
}
|
||||
|
||||
func fromDomainCacheControl(cacheControl *domain.CacheControl) *CacheControl {
|
||||
if cacheControl == nil {
|
||||
return nil
|
||||
|
||||
108
convert_appended_messages_test.go
Normal file
108
convert_appended_messages_test.go
Normal file
@@ -0,0 +1,108 @@
|
||||
package promptkit
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
)
|
||||
|
||||
func TestToDomainAppendedMessages(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
messages []RenderedMessage
|
||||
want []domain.RenderedMessage
|
||||
wantErr string
|
||||
privateRole string
|
||||
privateText string
|
||||
}{
|
||||
{
|
||||
name: "normalizes supported roles",
|
||||
messages: []RenderedMessage{
|
||||
{Role: " DeVeLoPeR ", Content: "developer"},
|
||||
{Role: "SyStEm", Content: "system"},
|
||||
{Role: "\tUsEr\n", Content: "user"},
|
||||
{Role: "assistant", Content: "assistant"},
|
||||
},
|
||||
want: []domain.RenderedMessage{
|
||||
{Role: domain.RoleDeveloper, Content: "developer"},
|
||||
{Role: domain.RoleSystem, Content: "system"},
|
||||
{Role: domain.RoleUser, Content: "user"},
|
||||
{Role: domain.RoleAssistant, Content: "assistant"},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "invalid role UTF-8",
|
||||
messages: []RenderedMessage{{Role: string([]byte{0xff}), Content: "private-content"}},
|
||||
wantErr: "appended message 0 role",
|
||||
},
|
||||
{
|
||||
name: "invalid content UTF-8",
|
||||
messages: []RenderedMessage{{Role: "private-role", Content: string([]byte{0xff})}},
|
||||
wantErr: "appended message 0 content",
|
||||
},
|
||||
{
|
||||
name: "unsupported role",
|
||||
messages: []RenderedMessage{{Role: "private-role", Content: "private-content"}},
|
||||
wantErr: "appended message 0 role",
|
||||
privateRole: "private-role",
|
||||
privateText: "private-content",
|
||||
},
|
||||
{
|
||||
name: "invalid cache control",
|
||||
messages: []RenderedMessage{{Role: RoleUser, Content: "private-content", CacheControl: &CacheControl{Type: "private-cache"}}},
|
||||
wantErr: "appended message 0 cache_control",
|
||||
privateText: "private-content",
|
||||
},
|
||||
{
|
||||
name: "empty and whitespace content",
|
||||
messages: []RenderedMessage{
|
||||
{Role: RoleUser, Content: ""},
|
||||
{Role: RoleAssistant, Content: " \t\n "},
|
||||
},
|
||||
want: []domain.RenderedMessage{
|
||||
{Role: domain.RoleUser, Content: ""},
|
||||
{Role: domain.RoleAssistant, Content: " \t\n "},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
got, err := toDomainAppendedMessages(test.messages)
|
||||
if test.wantErr != "" {
|
||||
if err == nil || !strings.Contains(err.Error(), test.wantErr) {
|
||||
t.Fatalf("error = %v, want %q", err, test.wantErr)
|
||||
}
|
||||
for _, privateValue := range []string{test.privateRole, test.privateText} {
|
||||
if privateValue != "" && strings.Contains(err.Error(), privateValue) {
|
||||
t.Fatalf("error exposed appended message data %q: %v", privateValue, err)
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("toDomainAppendedMessages() error = %v", err)
|
||||
}
|
||||
if !reflect.DeepEqual(got, test.want) {
|
||||
t.Fatalf("toDomainAppendedMessages() = %#v, want %#v", got, test.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestToDomainAppendedMessagesCopiesCacheControl(t *testing.T) {
|
||||
cacheControl := &CacheControl{Type: CacheControlEphemeral, TTL: "1h"}
|
||||
messages := []RenderedMessage{{Role: RoleUser, Content: "content", CacheControl: cacheControl}}
|
||||
request, err := toDomainRunRequest(RunRequest{AppendedMessages: messages})
|
||||
if err != nil {
|
||||
t.Fatalf("toDomainRunRequest() error = %v", err)
|
||||
}
|
||||
|
||||
messages[0].Content = "changed"
|
||||
cacheControl.TTL = ""
|
||||
if request.AppendedMessages[0].Content != "content" || request.AppendedMessages[0].CacheControl.TTL != "1h" {
|
||||
t.Fatalf("domain request did not retain an independent appended-message copy: %#v", request.AppendedMessages)
|
||||
}
|
||||
}
|
||||
35
doc.go
35
doc.go
@@ -1,30 +1,34 @@
|
||||
// Package promptkit provides an embeddable engine for preparing and executing
|
||||
// prompt-defined LLM workflows.
|
||||
//
|
||||
// Applications construct an [Engine] with [NewEngine], select filesystem or
|
||||
// in-memory sources and optional engine-scoped [Backend] registrations, and
|
||||
// call [Engine.InspectPrompt], [Engine.InspectProfile], [Engine.Prepare],
|
||||
// Applications construct an [Engine] with [NewEngine], select ConfigKit
|
||||
// sources for prompts, profiles, and backends, and
|
||||
// call [Engine.InspectConfiguration], [Engine.InspectPrompt],
|
||||
// [Engine.InspectProfile], [Engine.Prepare],
|
||||
// [Engine.PrepareExecution], [Engine.Run], or [Engine.RunPrepared]. Concrete
|
||||
// registries, repositories, validators, and the built-in OpenAI-compatible
|
||||
// client remain internal implementation details.
|
||||
//
|
||||
// # Concurrency and ownership
|
||||
//
|
||||
// An Engine supports concurrent InspectPrompt, InspectProfile, Prepare,
|
||||
// An Engine supports concurrent InspectConfiguration, InspectPrompt,
|
||||
// InspectProfile, Prepare,
|
||||
// PrepareExecution, Run, and RunPrepared calls. Engine-local backend policies
|
||||
// bound admitted Run and RunPrepared calls and model generations where
|
||||
// configured, while different backend pools and unlimited backends continue
|
||||
// independently. An injected [LLMClient] or [ArtifactReader] can therefore
|
||||
// still receive concurrent calls and must be safe for that use.
|
||||
//
|
||||
// NewEngine copies in-memory profiles and backend definitions. Prepare,
|
||||
// ConfigKit captures the selected configuration when NewEngine constructs an
|
||||
// Engine. Prepare,
|
||||
// PrepareExecution, and Run copy request maps, slices, pointer values, and
|
||||
// JSON-compatible extra parameters before using them. InspectPrompt and
|
||||
// InspectProfile return copied inspection values. Returned values and values
|
||||
// passed to extension interfaces are likewise isolated from engine state.
|
||||
// Callers own those copies and may mutate them after the call that supplied or
|
||||
// returned them. Returned structured errors are likewise caller-owned and may
|
||||
// be mutated without affecting engine state or another error.
|
||||
// returned them. [CapacityError] values are caller-owned and may be mutated
|
||||
// without affecting engine state or another error. Immutable [GenerationError]
|
||||
// values are also caller-owned and do not retain shared engine state.
|
||||
//
|
||||
// # Security and sensitive data
|
||||
//
|
||||
@@ -51,11 +55,18 @@
|
||||
// used by those values.
|
||||
//
|
||||
// Construction, inspection, handle, and error values, including [Config],
|
||||
// [Backend], [RunRequest], [ArtifactRef], [ExecutionTargetOverride], [Profile],
|
||||
// [OpenAICompatibleProfileConfig], [ProfileInspection],
|
||||
// [PromptInputDefinition], [PromptInspection], [PreparedExecution], and
|
||||
// [CapacityError], do not have stable JSON representations. Direct API keys
|
||||
// are nevertheless excluded from JSON for every public value.
|
||||
// [ConfigurationSources], [ConfigurationInspection], [BackendConfig],
|
||||
// [ProfileConfig], [RunRequest],
|
||||
// [ArtifactRef], [ExecutionTargetOverride], [ProfileInspection],
|
||||
// [PromptInputDefinition], [PromptInspection], [PreparedExecution],
|
||||
// [CapacityError], [GenerationError], [RetryExhaustedError], and
|
||||
// [ExecutionEvent], do not have stable JSON
|
||||
// representations. Direct API keys are nevertheless excluded from JSON for
|
||||
// every public value.
|
||||
|
||||
// Provider-derived [GenerationError] accessor values are untrusted and can
|
||||
// contain sensitive request or schema fragments. Applications must apply their
|
||||
// own disclosure policy before logging, displaying, or returning them.
|
||||
//
|
||||
// JSON timestamps use time.Time's RFC 3339 encoding and are omitted when zero.
|
||||
// PreparedRun and RunResult durations are encoded as integer milliseconds in
|
||||
|
||||
60
docs/adr/0001-configuration-boundary.md
Normal file
60
docs/adr/0001-configuration-boundary.md
Normal file
@@ -0,0 +1,60 @@
|
||||
# 0001: Configuration Ownership At The ConfigKit Handoff
|
||||
|
||||
## Status
|
||||
|
||||
Accepted.
|
||||
|
||||
## Date
|
||||
|
||||
2026-09-30
|
||||
|
||||
## Context
|
||||
|
||||
PromptKit is adopting ConfigKit to standardize configuration semantics and
|
||||
simplify its loading machinery. Configuration includes prompts, profiles, and
|
||||
backend definitions contributed by maintained catalogs and consuming applications.
|
||||
ConfigKit supplies loading, composition, precedence, inheritance, and inspection
|
||||
capabilities. PromptKit supplies LLM-specific interpretation and execution.
|
||||
|
||||
A clear handoff assigns each library responsibility for the configuration it
|
||||
owns and keeps validation aligned with the values an application actually uses.
|
||||
|
||||
## Decision
|
||||
|
||||
ConfigKit owns configuration processing up to the handoff of the final effective
|
||||
configuration. PromptKit takes ownership at that handoff and applies its domain
|
||||
interpretation and validation to the final configuration.
|
||||
|
||||
The handoff includes effective presence/null information and resource context
|
||||
supplied by ConfigKit. PromptKit uses this information to interpret effective
|
||||
settings and resolve referenced resources.
|
||||
|
||||
PromptKit passes through ConfigKit's inspection capabilities for explicit
|
||||
operator use. Configuration history and source explanations serve that
|
||||
inspection purpose; domain validation operates on the final effective values.
|
||||
|
||||
This boundary applies uniformly to application configuration and maintained
|
||||
catalog configuration, including backend definitions.
|
||||
|
||||
## Alternatives Considered
|
||||
|
||||
- **Independent source or catalog preflight validation:** preserves checks on
|
||||
individual contributions, but duplicates configuration processing and evaluates
|
||||
values that may be replaced before handoff. Raw-source and intermediate-state
|
||||
validation are outside PromptKit's scope, including equivalent test-only checks.
|
||||
- **History-based source auditing:** can identify overwritten contributions, but
|
||||
couples domain enforcement to configuration history. Historical inspection is
|
||||
an operator capability rather than a validation mechanism.
|
||||
|
||||
## Consequences
|
||||
|
||||
PromptKit's configuration logic and tests focus on the final effective values,
|
||||
their domain meaning, and resulting execution behavior. ConfigKit owns failures
|
||||
arising during its processing, which PromptKit preserves through its error
|
||||
boundary. Catalog maintainers own their repositories' publishing checks.
|
||||
|
||||
The adoption work routes backend configuration through the same handoff and
|
||||
retires PromptKit's source-level and intermediate-configuration validation.
|
||||
Effective resource interpretation and explicit operator inspection remain
|
||||
supported. This concentrates configuration semantics in ConfigKit and reduces
|
||||
duplicated machinery and maintenance in PromptKit.
|
||||
@@ -21,54 +21,114 @@ programs.
|
||||
|
||||
## Construct An Engine
|
||||
|
||||
Create an engine with
|
||||
[`NewEngine`](../../engine.go). A directory-backed setup supplies a prompt
|
||||
directory and may supply profile and schema directories:
|
||||
Create an engine with [`NewEngine`](../../engine.go). Select each directory
|
||||
explicitly and place its inputs in an ordered ConfigKit underlay. A group
|
||||
strictly composes its files; later groups can overlay earlier ones:
|
||||
|
||||
```go
|
||||
promptInputs, err := promptkit.DirectoryInputs("prompts")
|
||||
if err != nil { return err }
|
||||
profileInputs, err := promptkit.DirectoryInputs("profiles")
|
||||
if err != nil { return err }
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{
|
||||
PromptDir: "prompts",
|
||||
ProfileDir: "profiles",
|
||||
SchemaDir: "schemas",
|
||||
Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{
|
||||
{Inputs: promptInputs},
|
||||
{Inputs: profileInputs},
|
||||
}},
|
||||
SchemaDir: "schemas",
|
||||
})
|
||||
```
|
||||
|
||||
Options support single-file or `fs.FS` sources, in-memory profiles,
|
||||
engine-scoped backends, and injected artifact or model clients. Consult the
|
||||
[constructor and option GoDoc](../../engine.go) for composition, precedence,
|
||||
validation, and default transport behavior. Source discovery, format
|
||||
validation, and profile precedence are defined by the
|
||||
[framework format reference](../formats.md).
|
||||
For a single highest-priority document, set `Sources.Main` to a
|
||||
`*configkit.Input`; omit it for ordinary layered construction. A single file
|
||||
can use `configkit.File` directly. `ProfileSource`, `BackendSource`, and raw
|
||||
`configkit.Mapping` supply programmatic definitions. Schema and runtime
|
||||
collaborator options remain available. Construction validates the final
|
||||
effective configuration and captures referenced prompt content; construct a
|
||||
new engine to observe configuration edits. See the [constructor GoDoc](../../engine.go)
|
||||
for the exact public contract and the [framework format reference](../formats.md)
|
||||
for definition fields.
|
||||
|
||||
## Supply Embedded Application Defaults
|
||||
|
||||
Use `WithFallbackProfileFS` when an application packages profile definitions
|
||||
that should apply unless an operator provides an ordinary configured profile
|
||||
with the same ID. For example, an application can embed its defaults while
|
||||
continuing to use `ProfileDir` for operator overrides:
|
||||
Use `FilesystemInputs` for embedded defaults and `DirectoryInputs` for an
|
||||
explicitly chosen operator directory. A later programmatic group can override
|
||||
both. No directory name has built-in precedence:
|
||||
|
||||
```go
|
||||
//go:embed profiles/*.yaml
|
||||
var applicationProfiles embed.FS
|
||||
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{
|
||||
PromptDir: "prompts",
|
||||
ProfileDir: operatorProfileDir,
|
||||
},
|
||||
promptkit.WithFallbackProfileFS(applicationProfiles, "profiles"),
|
||||
)
|
||||
defaults, err := promptkit.FilesystemInputs("embedded defaults", applicationFS, "config")
|
||||
if err != nil { return err }
|
||||
operator, err := promptkit.DirectoryInputs(operatorDirectory)
|
||||
if err != nil { return err }
|
||||
profile := promptkit.ProfileSource("application choices", map[string]promptkit.ProfileConfig{
|
||||
"summary": {Model: &selectedModel},
|
||||
})
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||
Underlays: []configkit.Underlay{
|
||||
{Inputs: defaults},
|
||||
{Inputs: operator},
|
||||
{Inputs: []configkit.Input{{Source: profile}}},
|
||||
},
|
||||
}})
|
||||
```
|
||||
|
||||
Keep application-owned profile IDs and definitions in the embedded source.
|
||||
Use the ordinary configured profile source for operator overrides. Leave
|
||||
`operatorProfileDir` empty when the operator did not configure an override
|
||||
directory; a non-empty path names an authoritative higher-precedence source,
|
||||
so an unavailable or unreadable directory is an error rather than a reason to
|
||||
fall back. The
|
||||
[framework format reference](../formats.md#source-and-profile-precedence)
|
||||
owns the exact profile format and lookup order; the
|
||||
[`WithFallbackProfileFS` GoDoc](../../engine.go) owns its option contract and
|
||||
validation rules.
|
||||
Only add the operator group when the application selects that directory; an
|
||||
explicitly selected missing or unreadable root is an error. Each helper finds
|
||||
`.yaml` and `.yml` files recursively and disables imports by default. The
|
||||
caller may enable imports on individual inputs. A nil `Sources.Main` is an
|
||||
empty main source, not another directory lookup.
|
||||
|
||||
For a programmatic document with explicit null or a prompt definition, use a
|
||||
labeled mapping as an input. A mapping owns a snapshot of supported values:
|
||||
|
||||
```go
|
||||
document := configkit.Mapping("application configuration", map[string]any{
|
||||
"profiles": map[string]any{
|
||||
"local-summary": map[string]any{
|
||||
"backend": "local", "model": "example-model",
|
||||
"temperature": nil, // Suppress an inherited provider control.
|
||||
},
|
||||
},
|
||||
})
|
||||
input := configkit.Input{Source: document}
|
||||
// Add input to a later underlay, or set Sources.Main = &input.
|
||||
```
|
||||
|
||||
`ProfileSource` and `BackendSource` are smaller typed conveniences when null
|
||||
is not needed. A caller underlay can overlay catalog backend fields just as it
|
||||
can overlay profiles; there is no registration-only backend namespace.
|
||||
|
||||
When configuration or message files change, build and validate a new engine,
|
||||
then switch the application to that engine for subsequent work. Existing
|
||||
engines and prepared handles retain their captured definitions; schema and
|
||||
request artifact resources still follow their runtime lifecycle. Inspection
|
||||
does not watch or reload sources automatically.
|
||||
|
||||
## Inspect The Captured Configuration
|
||||
|
||||
Use [`Engine.InspectConfiguration`](../../configuration_inspection.go) for an
|
||||
operator-selected report of effective configuration, source descriptions,
|
||||
diagnostics, and path history. For a literal ID containing a slash, construct
|
||||
the semantic path with `configkit.Keys`:
|
||||
|
||||
```go
|
||||
view, err := engine.InspectConfiguration()
|
||||
if err != nil { return err }
|
||||
path := configkit.Keys("profiles", "team/summary", "model")
|
||||
explanation, found := view.Explain(path)
|
||||
if found && explanation.Effective.Redacted {
|
||||
// Apply the application's disclosure policy before sharing the report.
|
||||
}
|
||||
```
|
||||
|
||||
`WriteEffective` and `WriteExplanation` produce ConfigKit-redacted reports;
|
||||
the application decides whether those reports may be disclosed. Credential-
|
||||
shaped provider keys are redacted, but arbitrary prompt and provider content
|
||||
is not necessarily safe to share. Inspection uses the engine's captured result
|
||||
and does not reopen sources or resolve credentials. `InspectProfile` instead
|
||||
shows backend-completed execution settings for one profile, and `Prepare` or
|
||||
`PreparedExecution.Details` shows request-specific rendered messages, inputs,
|
||||
and validation resources. None of those views is a substitute for the others.
|
||||
|
||||
## Inspect A Prompt Before Preparation
|
||||
|
||||
@@ -151,6 +211,14 @@ handling policy. The
|
||||
[engine operation GoDoc](../../engine.go) own exact lifecycle, engine-binding,
|
||||
credential, cancellation, timing, and error semantics.
|
||||
|
||||
If the request supplied `Observer`, successful preparation retains that
|
||||
function for `RunPrepared`; preparation itself emits nothing. Observation
|
||||
starts only when the owning engine successfully claims the handle. Reused,
|
||||
discarded, foreign, and invalid handles emit nothing. `Discard` clears the
|
||||
retained callback without invoking it. The function's closure is retained by
|
||||
identity, so synchronize any shared state it uses. Prepared start events can
|
||||
include identities already resolved in the frozen snapshot.
|
||||
|
||||
## Execute And Validate
|
||||
|
||||
[`Engine.Run`](../../engine.go) performs the same preparation, invokes the
|
||||
@@ -173,6 +241,98 @@ semantics. The
|
||||
[OpenAI-compatible integration contract](../integrations/openai-compatible-chat.md)
|
||||
owns the built-in client's outbound HTTP behavior.
|
||||
|
||||
### Observe An Ordinary Run
|
||||
|
||||
Set `RunRequest.Observer` to receive synchronous metadata events from `Run`:
|
||||
|
||||
```go
|
||||
request.Observer = func(event promptkit.ExecutionEvent) {
|
||||
// Translate selected metadata into an application-owned record.
|
||||
}
|
||||
result, err := engine.Run(ctx, request)
|
||||
```
|
||||
|
||||
An ordinary run emits `started` before request conversion, followed by any
|
||||
attempt and retry-delay completions, then one `finished` event on a normal
|
||||
return. `Prepare` and `PrepareExecution` emit no events. A start event has no
|
||||
resolved prompt, profile, or model; later events gain identities only as
|
||||
selection succeeds. The terminal event has dispatched-attempt, transient-retry,
|
||||
repair-call, duration, and presence-aware usage totals. Its usage is an
|
||||
aggregate, not an additional billable attempt. A known zero differs from an
|
||||
unknown count, and incomplete totals must not be treated as complete cost
|
||||
accounting. A failed operation still returns no partial `RunResult`.
|
||||
An injected client that leaves `UsagePresence` nil can report known nonzero
|
||||
counts, but its zero counts remain unknown to observations; supply explicit
|
||||
presence when measured zeros matter. Existing `RunResult.Usage` sums returned
|
||||
values independently of observation presence.
|
||||
|
||||
Callbacks run synchronously, in order for one operation, and can overlap for
|
||||
different operations. Synchronize shared callback state and avoid blocking on
|
||||
work that needs the same run's admission slot. Callback panics propagate;
|
||||
there is no durable event history or promise of a terminal event after panic.
|
||||
Events omit content and error objects, but selected identifiers may still be
|
||||
sensitive. See the [observation GoDoc](../../observations.go) for exact fields.
|
||||
The [offline proposal and assessment example](../../examples/go-library/observe/main.go)
|
||||
shows application-owned event collection and a separate semantic acceptance
|
||||
decision using the built-in HTTP path without network access.
|
||||
|
||||
### Repair A Structured Result
|
||||
|
||||
Set a small additional-call budget when a structurally invalid result can be
|
||||
corrected automatically:
|
||||
|
||||
```go
|
||||
request.Validation = &promptkit.OutputContract{
|
||||
Format: promptkit.FormatJSON,
|
||||
ValidationMode: promptkit.ValidationJSONSchema,
|
||||
SchemaPath: "events.schema.json",
|
||||
RepairAttempts: 1,
|
||||
}
|
||||
```
|
||||
|
||||
Each repair attempt is another model generation, so it can increase latency and
|
||||
usage; `RunResult.Usage` is cumulative and `Validation.RepairAttempts` reports
|
||||
repair generations rather than transient HTTP attempts. Exhaustion still
|
||||
returns the final failed validation result. `basic` validation can also repair
|
||||
an empty candidate, but structural
|
||||
validity is not evidence of factual or domain correctness. See the
|
||||
[output-contract format reference](../formats.md#output-contract) and
|
||||
[`OutputContract` GoDoc](../../types.go) for the exact budget and eligibility
|
||||
rules.
|
||||
|
||||
The built-in client also retries eligible temporary HTTP statuses. By default,
|
||||
each generation can make two additional attempts; a repair generation receives
|
||||
its own budget. Set `Config.TransientRetries` to a pointer to zero for the
|
||||
previous single-attempt behavior, or set `RunRequest.TransientRetries` to
|
||||
override one execution. A custom HTTP transport still uses this policy;
|
||||
injected `LLMClient` implementations retain their own retry policy. Application
|
||||
semantic retries are separate runs and can multiply provider traffic further.
|
||||
The caller context covers capacity and retry waiting as well as attempts;
|
||||
without a deadline a valid provider delay can be long. Promptkit does not
|
||||
infer exactly-once or complete cost accounting from a retry outcome.
|
||||
See the [provider integration contract](../integrations/openai-compatible-chat.md#transient-status-retries)
|
||||
for eligible statuses, waits, timeouts, and provider timing.
|
||||
|
||||
### Append Already-Rendered Messages
|
||||
|
||||
An application can include an earlier assistant response and its own corrective
|
||||
instruction in a fresh request without changing the configured prompt:
|
||||
|
||||
```go
|
||||
request.AppendedMessages = []promptkit.RenderedMessage{
|
||||
{Role: promptkit.RoleAssistant, Content: previousResponse},
|
||||
{Role: promptkit.RoleUser, Content: correction},
|
||||
}
|
||||
result, err := engine.Run(ctx, request)
|
||||
```
|
||||
|
||||
These messages are already rendered: Promptkit does not template or resolve
|
||||
files in them, and they can contain sensitive model output or application
|
||||
feedback. Every `Run` call uses the engine's captured configuration; the
|
||||
application owns any semantic retry budget. When a
|
||||
pre-execution equality check is required, use `PrepareExecution` and compare
|
||||
its opaque rendered-prompt hash before invoking `RunPrepared`.
|
||||
|
||||
## Inputs, Profiles, And Overrides
|
||||
|
||||
Use `File`, `Inline`, or `InlineWithURI` to construct request inputs. A request
|
||||
@@ -182,12 +342,116 @@ replace execution settings or the complete output contract.
|
||||
The [public value GoDoc](../../types.go) defines nil, empty, zero, replacement,
|
||||
copy, and credential behavior. The
|
||||
[framework format reference](../formats.md) defines how those request values
|
||||
interact with prompt definitions, file-backed and application fallback
|
||||
profiles, built-ins, schemas, and framework defaults.
|
||||
interact with prompt definitions, layered profiles, built-ins, schemas, and
|
||||
framework defaults.
|
||||
|
||||
For programmatic profiles,
|
||||
[`OpenAICompatibleProfile`](../../profiles.go) converts ordinary
|
||||
OpenAI-compatible settings into a value accepted by `WithProfiles`.
|
||||
For programmatic profiles, use [`ProfileSource`](../../profiles.go) with keyed
|
||||
`ProfileConfig` values. Nil fields are omitted; pointers preserve explicit
|
||||
zero, false, and blank values. Raw `configkit.Mapping` expresses explicit null.
|
||||
|
||||
### Migrate From The Previous Configuration API
|
||||
|
||||
Replace `Config.PromptDir` and `Config.ProfileDir` with selected ConfigKit
|
||||
inputs in `Config.Sources`. The removed `WithPromptFS`, `WithPromptFile`,
|
||||
`WithProfileFS`, `WithProfileFile`, `WithFallbackProfileFS`, `WithProfiles`,
|
||||
and `WithBackend` options have no compatibility wrappers. Use
|
||||
`DirectoryInputs` or `FilesystemInputs` for a strict-composition file group,
|
||||
ConfigKit file inputs for individual files, and `ProfileSource`,
|
||||
`BackendSource`, or `configkit.Mapping` for programmatic values. Backend IDs
|
||||
are keys in configuration, not registrations. Choose caller underlays in
|
||||
fallback-to-operator-to-programmatic order; use `Sources.Main` for one final
|
||||
highest-priority input.
|
||||
|
||||
Move embedded `id` and `version` fields into keyed `profiles` and `prompts`
|
||||
maps; convert `base_profile` to ConfigKit `extends`. IDs and references are
|
||||
exact literal keys, without trimming. A supplied zero, false, or empty string
|
||||
is now distinguishable from omission; null can clear or suppress an inherited
|
||||
field according to the [field table](../formats.md#defaults-and-overrides).
|
||||
Maps merge recursively across inheritance and layers. `{}` does not clear an
|
||||
inherited map: use null to clear the whole map. At request time, a literal
|
||||
`extra_params["extends"]` is ordinary provider data, not an inheritance
|
||||
instruction; a nonempty request ExtraParams map replaces configured params.
|
||||
Changing a model or backend does not clear unrelated explicit fields, though
|
||||
backend-derived defaults are applied after inheritance to the final backend.
|
||||
|
||||
Invalid final definitions, including unused ones, now fail `NewEngine`; only
|
||||
request-dependent failures remain at preparation or execution. Prompt text
|
||||
and static templates are captured at construction, not automatically reloaded
|
||||
between runs. Replace the engine to pick up changed definitions or message
|
||||
files. The [format reference](../formats.md) owns exact field and presence
|
||||
rules; the maintained examples show complete runnable workflows.
|
||||
|
||||
### Choose A Profile Ownership Model
|
||||
|
||||
Choose the profile source that matches the application's continuity needs:
|
||||
|
||||
- Select `promptkit-openrouter-standard` or `promptkit-openrouter-auto` when
|
||||
the maintained literal name is the desired consumer contract. Their current
|
||||
selections are in the [format reference](../formats.md#built-in-profile-catalog).
|
||||
- Select an external catalog profile directly when the application accepts that
|
||||
profile's catalog volatility. The selected external inventories are linked
|
||||
from the [format reference](../formats.md#external-profile-inventories).
|
||||
- Define a complete application-owned profile with an explicit backend or
|
||||
endpoint, model, and settings when the application owns those choices.
|
||||
|
||||
An application profile that inherits an external base follows later changes to
|
||||
that base and fails if the base disappears; it is not a frozen copy. Promptkit
|
||||
tests the catalog versions selected in its module graph. Independently advancing
|
||||
a catalog dependency can remove an inherited base and produce an untested
|
||||
combination. Pinning the resolved module graph preserves the embedded data for
|
||||
a build, but cannot preserve provider availability. Running binaries neither
|
||||
download catalog updates nor automatically replace withdrawn models. See the
|
||||
[release policy](../policy/release.md#catalog-profile-compatibility) for the exact
|
||||
cross-version guarantees.
|
||||
|
||||
### Alias A Built-In Profile
|
||||
|
||||
Give an application-owned profile ID a built-in base when prompts should select
|
||||
the application ID while inheriting the built-in target. The child can override
|
||||
only the setting it owns:
|
||||
|
||||
```go
|
||||
profile := promptkit.ProfileSource("application profiles", map[string]promptkit.ProfileConfig{
|
||||
"weather-light": {
|
||||
Extends: &standardID,
|
||||
ReasoningEffort: &high,
|
||||
},
|
||||
})
|
||||
```
|
||||
|
||||
Place `profile` in a ConfigKit underlay, with `standardID` set to
|
||||
`"promptkit-openrouter-standard"` and `high` to `"high"`. Select
|
||||
`weather-light` in a prompt or `RunRequest.ProfileID`; it remains the
|
||||
reported selected profile. Definition IDs and inheritance references are exact
|
||||
keys. If the base is an external profile, apply the volatility guidance
|
||||
above when deciding whether to use an application-owned complete definition
|
||||
instead.
|
||||
|
||||
### Use The Rakestrawhome Built-In Profile
|
||||
|
||||
Set `RAKESTRAWHOME_INFERENCE_API_KEY` in the application environment, then
|
||||
select `rakestrawhome-gemma-4-31b` as an ordinary profile ID. For example, a
|
||||
prepared result identifies the selected built-in through
|
||||
`BackendRakestrawHome`:
|
||||
|
||||
```go
|
||||
prepared, err := engine.Prepare(ctx, promptkit.RunRequest{
|
||||
PromptID: "meeting.summary",
|
||||
ProfileID: "rakestrawhome-gemma-4-31b",
|
||||
Inputs: inputs,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if prepared.SelectedBackendID != promptkit.BackendRakestrawHome {
|
||||
return fmt.Errorf("unexpected backend %q", prepared.SelectedBackendID)
|
||||
}
|
||||
```
|
||||
|
||||
The maintained catalog supplies this backend, and callers may overlay its
|
||||
effective configuration through a later ConfigKit source. Direct request credentials
|
||||
and runtime endpoint overrides remain supported under their ordinary GoDoc and
|
||||
format contracts.
|
||||
|
||||
### Inspect A Profile Before Prompt Work
|
||||
|
||||
@@ -204,7 +468,7 @@ if err != nil {
|
||||
|
||||
target := inspection.EffectiveModelParams
|
||||
if target.APIKeyEnv != "" {
|
||||
// Apply application policy for the named environment variable.
|
||||
// This is a configured optional environment lookup source.
|
||||
} else if inspection.APIKeyRequired {
|
||||
// Arrange a direct credential before later execution.
|
||||
}
|
||||
@@ -213,9 +477,11 @@ if target.APIKeyEnv != "" {
|
||||
Use this configuration-time boundary when only the profile and its target need
|
||||
checking. Use `Prepare` when the application also needs prompt, input, schema,
|
||||
or rendering work; use prepared execution when that work must remain tied to a
|
||||
later execution. Inspection reports credential requirements but leaves the
|
||||
timing of credential enforcement to the application. The method's
|
||||
[GoDoc](../../engine.go) owns its exact result and error contract.
|
||||
later execution. A reported `APIKeyEnv` is a configured optional source, while
|
||||
`APIKeyRequired` is the explicit local requirement. The
|
||||
[credential format reference](../formats.md#credentials) and the method's
|
||||
[GoDoc](../../engine.go) own the exact precedence, timing, result, and error
|
||||
contracts.
|
||||
|
||||
### Set A Per-Run Session And Reasoning
|
||||
|
||||
@@ -250,19 +516,15 @@ Choose the smallest configuration that fits how the endpoint will be reused.
|
||||
|
||||
#### Use An Endpoint-Only Profile
|
||||
|
||||
Put the endpoint directly on an in-memory profile when only that profile needs
|
||||
Put the endpoint directly on a programmatic profile when only that profile needs
|
||||
it and shared backend identity or capacity policy is unnecessary:
|
||||
|
||||
```go
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{
|
||||
PromptDir: "prompts",
|
||||
},
|
||||
promptkit.WithProfiles(promptkit.Profile{
|
||||
ID: "local-summary",
|
||||
Endpoint: "http://localhost:8000/v1",
|
||||
Model: "example-model",
|
||||
}),
|
||||
)
|
||||
endpoint, model := "http://localhost:8000/v1", "example-model"
|
||||
profile := promptkit.ProfileSource("local profile", map[string]promptkit.ProfileConfig{
|
||||
"local-summary": {Endpoint: &endpoint, Model: &model},
|
||||
})
|
||||
// Add configkit.Input{Source: profile} to a caller underlay.
|
||||
```
|
||||
|
||||
Endpoint-only profiles have an empty backend ID and remain unrestricted by
|
||||
@@ -274,59 +536,44 @@ Use `LocalBackend` when profiles should share the conventional `local`
|
||||
identity, endpoint, and concurrency limit:
|
||||
|
||||
```go
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{
|
||||
PromptDir: "prompts",
|
||||
},
|
||||
promptkit.WithBackend(
|
||||
promptkit.LocalBackend("http://localhost:8000/v1", 2),
|
||||
),
|
||||
promptkit.WithProfiles(promptkit.Profile{
|
||||
ID: "local-summary",
|
||||
BackendID: promptkit.BackendLocal,
|
||||
Model: "example-model",
|
||||
}),
|
||||
)
|
||||
backendID, model := promptkit.BackendLocal, "example-model"
|
||||
backend := promptkit.BackendSource("local backend", map[string]promptkit.BackendConfig{
|
||||
backendID: promptkit.LocalBackend("http://localhost:8000/v1", 2),
|
||||
})
|
||||
profile := promptkit.ProfileSource("local profile", map[string]promptkit.ProfileConfig{
|
||||
"local-summary": {Backend: &backendID, Model: &model},
|
||||
})
|
||||
// Add both sources as inputs in a caller underlay.
|
||||
```
|
||||
|
||||
The helper is explicit: it does not pre-register a backend or read environment
|
||||
variables. Supplying a positive limit leaves queue capacity omitted, so normal
|
||||
backend registration selects the existing default waiting capacity of 1024.
|
||||
The returned value still enters the engine through `WithBackend`.
|
||||
The helper does not choose an ID or read environment variables. A positive
|
||||
concurrency limit with omitted queue capacity selects the default waiting
|
||||
capacity of 1024.
|
||||
|
||||
#### Configure A Complete Backend
|
||||
|
||||
Use a keyed `Backend` value for authentication, extra request parameters, an
|
||||
Use a keyed `BackendConfig` value for authentication, extra request parameters, an
|
||||
explicit queue capacity, a custom ID, or multiple local endpoints:
|
||||
|
||||
```go
|
||||
noWaiting := 0
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{
|
||||
PromptDir: "prompts",
|
||||
},
|
||||
promptkit.WithBackend(promptkit.Backend{
|
||||
ID: "local-gpu",
|
||||
Endpoint: "http://gpu-host:8000/v1",
|
||||
APIKeyEnv: "LOCAL_GPU_API_KEY",
|
||||
ExtraParams: map[string]any{"provider_option": "enabled"},
|
||||
ConcurrencyLimit: 2,
|
||||
QueueCapacity: &noWaiting,
|
||||
}),
|
||||
promptkit.WithProfiles(promptkit.Profile{
|
||||
ID: "gpu-summary",
|
||||
BackendID: "local-gpu",
|
||||
Model: "example-model",
|
||||
}),
|
||||
)
|
||||
endpoint, keyName, limit, noWaiting := "http://gpu-host:8000/v1", "LOCAL_GPU_API_KEY", 2, 0
|
||||
backend := promptkit.BackendSource("GPU backend", map[string]promptkit.BackendConfig{
|
||||
"local-gpu": {
|
||||
Endpoint: &endpoint, APIKeyEnv: &keyName,
|
||||
ConcurrencyLimit: &limit, QueueCapacity: &noWaiting,
|
||||
ExtraParams: map[string]any{"provider_option": "enabled"},
|
||||
},
|
||||
})
|
||||
```
|
||||
|
||||
Use distinct custom IDs when registering multiple local endpoints.
|
||||
Registrations belong to one engine and custom IDs cannot replace built-ins.
|
||||
The [`Backend`, `LocalBackend`, and `WithBackend` GoDoc](../../backends.go)
|
||||
defines exact construction, validation, copying, uniqueness, concurrency, and
|
||||
request-default behavior.
|
||||
Place `backend` in a caller underlay and select `"local-gpu"` from a
|
||||
`ProfileConfig.Backend` pointer or a keyed file profile. All backend
|
||||
contributions use the same ConfigKit layering rules, including overlays of
|
||||
maintained definitions. The [`BackendConfig` and `LocalBackend` GoDoc](../../backends.go)
|
||||
defines exact omission and construction behavior.
|
||||
|
||||
Both file-backed and in-memory profiles select a registration through
|
||||
`backend` or `Profile.BackendID`. Profile and request endpoint overrides retain
|
||||
Both file-backed and programmatic profiles select a backend through
|
||||
`backend` or `ProfileConfig.Backend`. Profile and request endpoint overrides retain
|
||||
that routing and capacity identity. `PreparedRun.SelectedBackendID`,
|
||||
`RunResult.SelectedBackendID`, and the effective `ExecutionTarget.BackendID`
|
||||
expose it to consumers and injected model clients. Endpoint-only profiles
|
||||
@@ -334,7 +581,7 @@ remain supported and expose an empty backend ID.
|
||||
|
||||
### Limit Backend Concurrency
|
||||
|
||||
Set `Backend.ConcurrencyLimit` when a backend needs protection from too many
|
||||
Set `BackendConfig.ConcurrencyLimit` when a backend needs protection from too many
|
||||
simultaneous model calls. Leaving `QueueCapacity` nil, as in the local-backend
|
||||
example above, selects the default waiting capacity of 1024.
|
||||
|
||||
@@ -343,24 +590,23 @@ zero:
|
||||
|
||||
```go
|
||||
noWaiting := 0
|
||||
backend := promptkit.Backend{
|
||||
ID: "local-gpu",
|
||||
Endpoint: "http://gpu-host:8000/v1",
|
||||
ConcurrencyLimit: 2,
|
||||
QueueCapacity: &noWaiting,
|
||||
backend := promptkit.BackendConfig{
|
||||
Endpoint: &endpoint,
|
||||
ConcurrencyLimit: &limit,
|
||||
QueueCapacity: &noWaiting,
|
||||
}
|
||||
```
|
||||
|
||||
The pointer distinguishes an explicit zero from omission. Keep using keyed
|
||||
`Backend` literals so additive configuration fields remain source-compatible.
|
||||
The pointer distinguishes an explicit zero from omission. Use keyed
|
||||
`BackendConfig` literals inside `BackendSource` for programmatic definitions.
|
||||
Capacity belongs to one engine and the selected backend ID; endpoint-only
|
||||
profiles and custom backends without a configured limit remain unrestricted.
|
||||
Exact validation, defaulting, ownership, and concurrency semantics belong to
|
||||
the [`Backend` GoDoc](../../backends.go).
|
||||
the [`BackendConfig` GoDoc](../../backends.go).
|
||||
|
||||
## Credentials
|
||||
|
||||
File-backed profiles name an environment variable; in-memory profiles can
|
||||
File-backed and programmatic profiles can name an environment variable or
|
||||
require a direct request key. Direct keys are request-scoped and are excluded
|
||||
from supported JSON values and the package's `String` and `GoString`
|
||||
summaries. The exact precedence and redaction guarantees belong to
|
||||
@@ -394,12 +640,13 @@ the application policy that motivated the injection.
|
||||
|
||||
Use `errors.Is` with the
|
||||
[public error sentinels and operation GoDoc](../../engine.go). The declarations
|
||||
distinguish invalid construction, invalid requests, absent sources,
|
||||
source-loading failures, collaborator failures, and operational validation
|
||||
distinguish invalid construction, invalid requests, absent selections,
|
||||
collaborator failures, and operational validation
|
||||
failures. Specific request conditions may also match the broader
|
||||
`ErrInvalidRequest`, and injected collaborator identities are preserved where
|
||||
documented. Invalid or duplicate backend registrations match
|
||||
`ErrInvalidConfig`; selecting an unknown backend matches `ErrProfileLoad`.
|
||||
documented. Invalid effective definitions, including unknown backend
|
||||
references, match `ErrInvalidConfig` during construction. ConfigKit load
|
||||
errors remain available through `errors.As`.
|
||||
|
||||
When a limited backend has admitted all active and waiting calls, handle
|
||||
`ErrCapacityExceeded` separately from request errors and provider failures:
|
||||
@@ -423,6 +670,38 @@ status; those choices remain with the consuming application. The
|
||||
contract, while the [`Engine.Run` and error GoDoc](../../engine.go) owns broad
|
||||
error and cancellation identities.
|
||||
|
||||
For a non-2xx response from the built-in OpenAI-compatible client, inspect the
|
||||
status and deliberately selected provider diagnostic when useful:
|
||||
|
||||
```go
|
||||
var generationErr *promptkit.GenerationError
|
||||
if errors.As(err, &generationErr) {
|
||||
status := generationErr.StatusCode()
|
||||
message := generationErr.ProviderMessage()
|
||||
_, _ = status, message // Apply application presentation policy.
|
||||
}
|
||||
```
|
||||
|
||||
All provider fields are untrusted and can contain sensitive request or schema
|
||||
fragments. Do not log, display, or return them without an application-specific
|
||||
disclosure policy. Promptkit does not assign presentation behavior.
|
||||
The [`GenerationError` GoDoc](../../generation_error.go) owns the exact typed
|
||||
error contract.
|
||||
|
||||
Generation failures also match `ErrTransport` when the built-in HTTP request
|
||||
fails, or `ErrMalformedResponse` when a successful provider response cannot be
|
||||
decoded or an injected client returns a nil successful response. Both retain
|
||||
`ErrLLMGenerate`; transport failures retain their original causes for
|
||||
`errors.Is` and `errors.As`. Caller cancellation and deadlines retain their
|
||||
standard context identities. Arbitrary injected-client errors retain their own
|
||||
identities without Promptkit guessing a failure category from their text.
|
||||
An eligible built-in status that consumes a positive retry budget additionally
|
||||
matches `ErrRetriesExhausted`; use `errors.As` to read
|
||||
`RetryExhaustedError.Attempts()` and the final `GenerationError` status. With
|
||||
explicit zero retries, the first eligible status is an ordinary generation
|
||||
error, not exhaustion. Completed structural rejection remains a normal result
|
||||
with failed validation, while an operational error returns no partial result.
|
||||
|
||||
## Application Boundary
|
||||
|
||||
Promptkit is an importable library. It does not own a command, inbound HTTP
|
||||
|
||||
@@ -1,8 +1,7 @@
|
||||
# Development
|
||||
|
||||
This is the contributor entry point for Promptkit, a reusable Go library. All
|
||||
contributors must read the
|
||||
[architecture policy](policy/architecture.md) before making changes.
|
||||
This is the contributor entry point for Promptkit, a reusable Go library.
|
||||
Use the Policy Reading Guide below to select policies applicable to the task.
|
||||
|
||||
## Initial Orientation
|
||||
|
||||
@@ -15,17 +14,27 @@ Before starting work:
|
||||
4. keep documentation limited to implemented behavior unless an accepted
|
||||
decision or temporary roadmap explicitly owns future work.
|
||||
|
||||
Start with:
|
||||
## Policy Reading Guide
|
||||
|
||||
- the [architecture policy](policy/architecture.md) for library boundaries,
|
||||
dependency direction, invariants, and non-goals;
|
||||
- the [internal component overview](internal/overview.md) for the current
|
||||
package and component inventory;
|
||||
- the [documentation policy](policy/documentation.md) before changing
|
||||
documentation;
|
||||
- the [testing policy](policy/testing.md) before adding, rewriting, or deleting
|
||||
tests; and
|
||||
- the [release procedure](release.md) for version and publication work.
|
||||
| Policy | Read before… |
|
||||
| --- | --- |
|
||||
| [Policy set provenance](policy/meta.md) | Adopting policies, checking upstream revisions, or updating the adopted policy set. |
|
||||
| [Agents](policy/agents.md) | Recommending, planning, or implementing repository changes. |
|
||||
| [Architecture](policy/architecture.md) | Planning or changing library code or its structure. |
|
||||
| [Testing](policy/testing.md) | Selecting validation for code changes, or adding, changing, removing, or evaluating tests. |
|
||||
| [Documentation](policy/documentation.md) | Writing, revising, or reorganizing documentation. |
|
||||
| [OSS](policy/oss.md) | Adding or updating dependencies or incorporated third-party material, or changing licensing or attribution. |
|
||||
| [Release](policy/release.md) | Preparing or publishing a release, changing release procedures, or making versioning and compatibility decisions. |
|
||||
|
||||
- Read applicable policies before making decisions they govern; multiple policies
|
||||
may apply.
|
||||
- Select policies using the task triggers above. Follow cross-references to
|
||||
relevant sections when needed to resolve a concrete question.
|
||||
- Reuse policies already available in context; reread when changed or no longer
|
||||
available.
|
||||
- Expand reading when the task's scope expands.
|
||||
- When revising a policy itself, read that policy and any provisions elsewhere
|
||||
affected by the revision.
|
||||
|
||||
## Task-Specific Reading Guide
|
||||
|
||||
@@ -34,12 +43,14 @@ Start with:
|
||||
| Root public API | The [architecture policy](policy/architecture.md), [consumer guide](consumers/pkg-promptkit.md), [testing policy](policy/testing.md), and existing GoDoc. |
|
||||
| Prompt, profile, or schema formats | The [framework format reference](formats.md), owning parser or validator package, and [documentation policy](policy/documentation.md). |
|
||||
| Source loading or validation | The [framework format reference](formats.md), [internal source document](internal/sources.md), and owning package tests. |
|
||||
| Maintained external catalogs or built-ins | The [internal source document](internal/sources.md), [framework format reference](formats.md), [testing policy](policy/testing.md), both catalog module repositories, each catalog's release procedure, and the selected module versions in `go.mod`. |
|
||||
| Model-client behavior | The [OpenAI-compatible integration contract](integrations/openai-compatible-chat.md), [internal model-client document](internal/llm.md), and owning package tests. |
|
||||
| Internal package implementation | The [architecture policy](policy/architecture.md), [internal component overview](internal/overview.md), and focused internal document listed for that package. |
|
||||
| Tests or test fixtures | The [testing policy](policy/testing.md), owning package, and focused internal document listed by the component overview. |
|
||||
| Maintained example | The [example](../examples/go-library/prepare/main.go), [consumer guide](consumers/pkg-promptkit.md), [framework format reference](formats.md), and [documentation policy](policy/documentation.md). |
|
||||
| Maintained example | The [preparation](../examples/go-library/prepare/main.go), [execution](../examples/go-library/run/main.go), and [observation](../examples/go-library/observe/main.go) examples, [consumer guide](consumers/pkg-promptkit.md), [framework format reference](formats.md), and [documentation policy](policy/documentation.md). |
|
||||
| Documentation | The [documentation policy](policy/documentation.md) and canonical owner of every affected contract. |
|
||||
| Release preparation or publication | The [release procedure](release.md). |
|
||||
| Licensing, dependencies/catalog updates, copied material, or attribution | The [OSS policy](policy/oss.md), [project grant](../COPYRIGHT), [license texts](../LICENSE), and [attribution entries](oss/attribution.md); inspect the selected upstream source's notices. |
|
||||
| Release preparation or publication, including release notes | The [release policy](policy/release.md), [release procedure](release.md), and [OSS discoverability rules](policy/oss.md#discoverability). |
|
||||
|
||||
For cross-cutting changes, follow every applicable row. Do not create
|
||||
placeholder documents for packages, APIs, or integrations that do not yet
|
||||
@@ -47,187 +58,40 @@ exist.
|
||||
|
||||
## Maintainer Validation
|
||||
|
||||
This section is the canonical local validation workflow for Promptkit. Run
|
||||
every command from the repository root before accepting a change. The test
|
||||
suite and maintained examples are deterministic, offline, and require no real
|
||||
provider credentials.
|
||||
|
||||
### Tests, Analysis, Build, And Examples
|
||||
|
||||
Run the ordinary and race-enabled suites, static analysis, the build, and both
|
||||
maintained consumer examples:
|
||||
For ordinary work, select validation proportionate to the change. Do not repeat
|
||||
successful checks for unchanged content merely to create a commit. The full
|
||||
release-candidate checks have one executable entry point from the repository root:
|
||||
|
||||
```sh
|
||||
go test ./...
|
||||
go test -race ./...
|
||||
go vet ./...
|
||||
go build ./...
|
||||
go run ./examples/go-library/prepare
|
||||
go run ./examples/go-library/run
|
||||
./scripts/release-check.sh
|
||||
```
|
||||
|
||||
Both examples must exit successfully. Review their JSON output: preparation
|
||||
must report the selected offline prompt, profile, model, and message count;
|
||||
execution must report the deterministic generated output, successful
|
||||
validation, selected offline model, and usage. Neither command may contact a
|
||||
provider or require credentials.
|
||||
The [script](../scripts/release-check.sh) owns the executable checks:
|
||||
race-enabled tests, vet, the three offline consumer examples with semantic output
|
||||
assertions, tracked Go formatting, local Markdown links, module/workspace
|
||||
hygiene, whitespace, and the credential-pattern scan. It does not publish,
|
||||
change files, validate release-note structure, or retire roadmaps. It requires Go,
|
||||
Git, a POSIX shell, and Python 3. Tests and examples require no provider
|
||||
credentials or live inference calls.
|
||||
|
||||
### Go Formatting
|
||||
The script can run on intended uncommitted changes during development.
|
||||
Formatting and documentation checks use tracked files; stage intended new
|
||||
files before final validation so those checks include them. The release
|
||||
procedure separately enforces a clean, committed candidate.
|
||||
|
||||
Check every tracked Go file. The final command must succeed and the captured
|
||||
list must be empty:
|
||||
|
||||
```sh
|
||||
unformatted=$(
|
||||
git ls-files '*.go' |
|
||||
while IFS= read -r go_file
|
||||
do
|
||||
gofmt -l "$go_file"
|
||||
done
|
||||
)
|
||||
test -z "$unformatted"
|
||||
```
|
||||
|
||||
### Local Markdown Links
|
||||
|
||||
Use the Python standard library to verify every repository-relative Markdown
|
||||
target and local heading fragment. The check is offline and prints nothing on
|
||||
success:
|
||||
|
||||
```sh
|
||||
python3 - <<'PY'
|
||||
from pathlib import Path
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from urllib.parse import unquote
|
||||
|
||||
root = Path.cwd().resolve()
|
||||
markdown_files = [
|
||||
root / name
|
||||
for name in subprocess.check_output(
|
||||
["git", "ls-files", "*.md"], text=True
|
||||
).splitlines()
|
||||
]
|
||||
link_pattern = re.compile(r"!?\[[^]]*\]\(([^)]+)\)")
|
||||
heading_pattern = re.compile(r"^#{1,6}\s+(.+?)\s*#*\s*$")
|
||||
scheme_pattern = re.compile(r"^[a-z][a-z0-9+.-]*:", re.IGNORECASE)
|
||||
|
||||
|
||||
def markdown_lines(path):
|
||||
in_fence = False
|
||||
fence = ""
|
||||
for line in path.read_text(encoding="utf-8").splitlines():
|
||||
stripped = line.lstrip()
|
||||
marker = stripped[:3]
|
||||
if marker in {"```", "~~~"}:
|
||||
if not in_fence:
|
||||
in_fence = True
|
||||
fence = marker
|
||||
elif marker == fence:
|
||||
in_fence = False
|
||||
fence = ""
|
||||
continue
|
||||
if not in_fence:
|
||||
yield line
|
||||
|
||||
|
||||
anchor_cache = {}
|
||||
|
||||
|
||||
def anchors(path):
|
||||
if path in anchor_cache:
|
||||
return anchor_cache[path]
|
||||
found = set()
|
||||
counts = {}
|
||||
for line in markdown_lines(path):
|
||||
match = heading_pattern.match(line)
|
||||
if not match:
|
||||
continue
|
||||
heading = re.sub(r"<[^>]+>", "", match.group(1)).replace("`", "")
|
||||
base = re.sub(r"[^\w\- ]", "", heading.lower()).replace(" ", "-")
|
||||
count = counts.get(base, 0)
|
||||
counts[base] = count + 1
|
||||
found.add(base if count == 0 else f"{base}-{count}")
|
||||
anchor_cache[path] = found
|
||||
return found
|
||||
|
||||
|
||||
failures = []
|
||||
for source in markdown_files:
|
||||
text = "\n".join(markdown_lines(source))
|
||||
for match in link_pattern.finditer(text):
|
||||
target = match.group(1).strip()
|
||||
if target.startswith("<") and target.endswith(">"):
|
||||
target = target[1:-1]
|
||||
if scheme_pattern.match(target) or target.startswith("//"):
|
||||
continue
|
||||
path_text, separator, fragment = target.partition("#")
|
||||
destination = source if not path_text else source.parent / unquote(path_text)
|
||||
try:
|
||||
destination = destination.resolve()
|
||||
destination.relative_to(root)
|
||||
except ValueError:
|
||||
failures.append(f"{source.relative_to(root)}: escapes repository: {target}")
|
||||
continue
|
||||
if not destination.exists():
|
||||
failures.append(f"{source.relative_to(root)}: missing target: {target}")
|
||||
continue
|
||||
if separator and destination.suffix.lower() == ".md":
|
||||
fragment = unquote(fragment).lower()
|
||||
if fragment not in anchors(destination):
|
||||
failures.append(f"{source.relative_to(root)}: missing anchor: {target}")
|
||||
|
||||
if failures:
|
||||
print("\n".join(failures), file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
PY
|
||||
```
|
||||
|
||||
### Repository Hygiene And Review
|
||||
|
||||
Reject an active Go workspace, tracked workspace files, a vendor tree, or a
|
||||
module replacement:
|
||||
|
||||
```sh
|
||||
case "$(go env GOWORK)" in
|
||||
''|off) ;;
|
||||
*) printf '%s\n' 'an active Go workspace is not allowed' >&2; exit 1 ;;
|
||||
esac
|
||||
test -z "$(git ls-files go.work go.work.sum)"
|
||||
test ! -e vendor
|
||||
if grep -Eq '^[[:space:]]*replace([[:space:]]|\()' go.mod
|
||||
then
|
||||
printf '%s\n' 'go.mod contains a replacement' >&2
|
||||
exit 1
|
||||
fi
|
||||
```
|
||||
|
||||
Check whitespace in both unstaged and staged changes. List ignored files and
|
||||
scan tracked content for common credential forms:
|
||||
|
||||
```sh
|
||||
git diff --check
|
||||
git diff --cached --check
|
||||
test -z "$(git ls-files --others --ignored --exclude-standard)"
|
||||
credential_pattern='-----BEGIN ([A-Z0-9]+ )?PRIV''ATE KEY-----|AKI''A[0-9A-Z]{16}|gh[pousr]_[A-Za-z0-9]{36,}|sk-[A-Za-z0-9]{32,}'
|
||||
if git grep -nEI -e "$credential_pattern" -- .
|
||||
then
|
||||
printf '%s\n' 'possible credential found' >&2
|
||||
exit 1
|
||||
fi
|
||||
```
|
||||
### Change Review
|
||||
|
||||
Inspect `git status --short --untracked-files=all` and the complete diff before
|
||||
accepting a change. The status may contain only the intended source changes
|
||||
during development. Reject credentials, private keys, environment files,
|
||||
generated binaries, test or coverage output, downloaded assets, template
|
||||
residue, and any other artifact that does not belong in source control. The
|
||||
credential scan catches common forms but does not replace inspection of the
|
||||
actual change.
|
||||
accepting a change. Reject unintended files, credentials, private data,
|
||||
generated binaries, test output, downloaded assets, and template residue.
|
||||
The automated credential scan does not replace review of the actual change.
|
||||
|
||||
After committing the accepted change, require a clean candidate:
|
||||
For releases, follow the [release procedure](release.md). Its publication
|
||||
command runs the full local checks once on the final committed candidate and
|
||||
automates release-note structure and Git checks. Do not repeat ordinary change
|
||||
review as a release sign-off. Roadmap retirement remains preparation housekeeping.
|
||||
|
||||
```sh
|
||||
test -z "$(git status --porcelain)"
|
||||
```
|
||||
Follow the [agent commit policy](policy/agents.md#git-commit-policy) for local
|
||||
commits. Ordinary commits may leave unrelated work in the workspace; preserve
|
||||
and report it. A clean worktree and index are required for release candidates,
|
||||
as enforced by the publication command, not for every ordinary commit.
|
||||
|
||||
509
docs/formats.md
509
docs/formats.md
@@ -2,322 +2,309 @@
|
||||
|
||||
## Purpose
|
||||
|
||||
This document is the canonical contract for Promptkit prompt-definition,
|
||||
profile, and schema files. The [Go package consumer guide](consumers/pkg-promptkit.md)
|
||||
explains how to select these sources and invoke the engine. The
|
||||
This is the canonical reference for Promptkit's backend, profile, prompt, and
|
||||
schema definitions. The [Go consumer guide](consumers/pkg-promptkit.md) shows
|
||||
how to select sources and use an engine; the
|
||||
[OpenAI-compatible integration contract](integrations/openai-compatible-chat.md)
|
||||
owns the resulting outbound wire behavior.
|
||||
owns outbound wire behavior. [ConfigKit](https://gitea.maximumdirect.net/eric/ConfigKit)
|
||||
owns the generic YAML grammar, imports, layering, inheritance, and diagnostics.
|
||||
This reference describes what Promptkit does with the final effective values.
|
||||
|
||||
Prompt and profile sources recursively discover files ending in `.yaml` or
|
||||
`.yml`. Each prompt-definition and profile file contains exactly one YAML
|
||||
document; comments and trailing whitespace are allowed. YAML decoding is
|
||||
strict: unknown fields are errors for the selected definition. Definitions are
|
||||
selected by their YAML `id`, not their file name or directory.
|
||||
A configuration document has keyed top-level `backends`, `profiles`, and
|
||||
`prompts` mappings. Their keys are literal IDs, not filename-derived identities.
|
||||
Prompt versions are a second keyed mapping. Keys and references must be
|
||||
nonblank and remain exact: Promptkit does not trim or re-key them. Old embedded
|
||||
`id`, `version`, and `base_profile` fields are invalid. Unknown fields are
|
||||
errors.
|
||||
|
||||
```yaml
|
||||
backends:
|
||||
local:
|
||||
endpoint: http://localhost:8000/v1
|
||||
concurrency_limit: 2
|
||||
profiles:
|
||||
local-summary:
|
||||
backend: local
|
||||
model: example-model
|
||||
prompts:
|
||||
meeting.summary:
|
||||
"1.0.0":
|
||||
default_profile: local-summary
|
||||
messages:
|
||||
- role: user
|
||||
content: Summarize this note.
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
```
|
||||
|
||||
Every effective backend, profile, and prompt definition is validated when
|
||||
`NewEngine` constructs the engine, even if it is never selected. A profile may
|
||||
be partial: an endpoint and model need only be complete after request overrides
|
||||
are applied. A prompt's default profile similarly need not be runnable without
|
||||
an overriding request. Missing request data, artifacts, schemas, credentials,
|
||||
and provider work remain runtime concerns.
|
||||
|
||||
The engine captures effective definitions, referenced message-file contents,
|
||||
and static template syntax at construction. It does not reread configuration or
|
||||
message files on ordinary runs; construct a new engine to observe edits.
|
||||
Schema and request artifact reads still happen during preparation. Use
|
||||
`InspectConfiguration` for an explicit effective-configuration or provenance
|
||||
report, `InspectProfile` for backend-completed profile settings, and preparation
|
||||
for request-specific rendered details.
|
||||
|
||||
## Prompt Definitions
|
||||
|
||||
A prompt definition describes inputs, Go-template messages, an optional
|
||||
default profile, and an output contract.
|
||||
The `prompts` mapping selects a literal prompt ID, then a string version key:
|
||||
|
||||
```yaml
|
||||
id: meeting.summary
|
||||
version: "1.0.0"
|
||||
default_profile: local-summary
|
||||
description: Summarize a synthetic meeting note.
|
||||
session_id: '{{.session}}'
|
||||
inputs:
|
||||
- name: note
|
||||
required: true
|
||||
content_type: text/plain
|
||||
description: Meeting note to summarize.
|
||||
messages:
|
||||
- role: system
|
||||
content: Return a concise summary.
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
ttl: 1h
|
||||
- role: user
|
||||
content_file: ./summary.user.md
|
||||
output:
|
||||
format: markdown
|
||||
validation_mode: basic
|
||||
repair_attempts: 0
|
||||
prompts:
|
||||
meeting.summary:
|
||||
"1.0.0":
|
||||
default_profile: local-summary
|
||||
description: Summarize a synthetic meeting note.
|
||||
session_id: '{{.session}}'
|
||||
inputs:
|
||||
- name: note
|
||||
required: true
|
||||
content_type: text/plain
|
||||
description: Meeting note to summarize.
|
||||
messages:
|
||||
- role: system
|
||||
content: Return a concise summary.
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
ttl: 1h
|
||||
- role: user
|
||||
content_file: ./summary.user.md
|
||||
output:
|
||||
format: markdown
|
||||
validation_mode: basic
|
||||
repair_attempts: 0
|
||||
```
|
||||
|
||||
| Field | Required | Meaning |
|
||||
| --- | --- | --- |
|
||||
| `id` | yes | Non-empty prompt identifier used by `RunRequest.PromptID`. |
|
||||
| `version` | yes | Non-empty version selected by an optional `RunRequest.PromptVersion`. |
|
||||
| `default_profile` | no | Non-empty profile ID used when the request omits `ProfileID`. |
|
||||
| `description` | no | Human-readable description. |
|
||||
| `session_id` | no | Go template rendered from request variables and input helpers. |
|
||||
| `inputs` | no | Declared input metadata. |
|
||||
| `messages` | yes | One or more chat-message templates. |
|
||||
| `output` | yes | Output format and validation settings. |
|
||||
|
||||
When a request omits a version, the selected prompt ID must identify exactly
|
||||
one definition. When it supplies a version, the ID and version pair must be
|
||||
unique.
|
||||
|
||||
Exact prompt inspection uses this same configured source, strict decoding,
|
||||
referenced content-file resolution, and ID/version selection. It reports the
|
||||
selected definition's declared metadata without changing the prompt format or
|
||||
executing the definition.
|
||||
`messages` must be nonempty and `output` must be a valid mapping. Optional
|
||||
`default_profile`, `description`, `session_id`, `inputs`, and cache-control
|
||||
fields may be absent or cleared with null where appropriate. A null whole
|
||||
definition, required role, output format, or validation mode is invalid.
|
||||
`required` and `repair_attempts` default to false and zero, respectively, and
|
||||
are not nullable. A request that omits a prompt version succeeds only if that
|
||||
prompt ID has exactly one effective version. `InspectPrompt` selects from the
|
||||
same captured definitions without rendering or resolving a profile.
|
||||
|
||||
### Inputs
|
||||
|
||||
Each `inputs` item has these fields:
|
||||
|
||||
| Field | Required | Meaning |
|
||||
| --- | --- | --- |
|
||||
| `name` | yes | Non-empty name used by the request input map and `input` template helper. Names must be unique. |
|
||||
| `required` | no | When true, preparation fails if the request omits the input. The default is false. |
|
||||
| `content_type` | no | Expected media-type metadata. |
|
||||
| `description` | no | Human-readable input description. |
|
||||
|
||||
Requests supply inputs as inline or file-backed `ArtifactRef` values. Declared
|
||||
required inputs must be present. A template reference also requires the named
|
||||
input to exist, whether or not it was declared. Extra request inputs are
|
||||
allowed.
|
||||
Each `inputs` item may contain `name`, `required`, `content_type`, and
|
||||
`description`. `name` is required, nonblank, and unique within the prompt.
|
||||
Omitted `required` means false. Null `inputs` means no declarations. Requests
|
||||
supply inline or file-backed `ArtifactRef` values; declared required inputs
|
||||
must be present. A template reference also needs its named input, even when
|
||||
undeclared. Extra request inputs are allowed.
|
||||
|
||||
### Messages And Templates
|
||||
|
||||
Each message has a non-empty `role` and exactly one of:
|
||||
Each message needs a role and exactly one nonnull `content` or `content_file`.
|
||||
An explicitly empty inline `content` is valid. Null clears either content
|
||||
selection. Roles are trimmed and lowercased, then must be `developer`,
|
||||
`system`, `user`, or `assistant`. Static template syntax is checked during
|
||||
construction; missing variables and named inputs fail when rendered.
|
||||
Request variables are template data (`{{.audience}}`), and
|
||||
`{{input "note"}}` renders an input body.
|
||||
|
||||
- `content`, containing an inline Go template; or
|
||||
- `content_file`, naming a file whose contents are the Go template.
|
||||
`content_file` uses the effective field's own source context, including when
|
||||
inherited or imported. Its relative path is resolved against that source's
|
||||
base directory within its rooted filesystem. Absolute paths and lexical
|
||||
escapes from the selected root fail. Host-file sources also prevent symlink
|
||||
escapes. Promptkit preserves the path text, including meaningful whitespace,
|
||||
after checking that it is not blank. A source without a unique usable resource
|
||||
context must supply one explicitly or override the field. Content is captured
|
||||
at construction, not reopened for each request.
|
||||
|
||||
`content_file` must be a relative path. It resolves from the directory that
|
||||
contains the prompt file and must remain within the configured prompt source
|
||||
root; parent components are allowed only when the resolved target remains
|
||||
inside that root. Absolute paths and paths that escape the root are rejected.
|
||||
Operating-system directory and single-file sources also reject symlink targets
|
||||
outside the root, while injected `fs.FS` sources apply containment in that
|
||||
filesystem's relative path namespace. For `WithPromptFile`, the source root is
|
||||
the directory containing the selected prompt file. Promptkit uses the parsed
|
||||
path text exactly after checking separately that it is not blank, so leading
|
||||
and trailing whitespace can name real filesystem entries.
|
||||
|
||||
Request variables are the template data, so a variable named `audience` is
|
||||
referenced as `{{.audience}}`. The `{{input "note"}}` helper renders the body
|
||||
of a named input. Missing variables and input references are errors.
|
||||
|
||||
The optional `session_id` uses the same template data and input helper. Its
|
||||
rendered value is trimmed, omitted when empty, and limited to 256 Unicode code
|
||||
points. A nonblank direct request session ID bypasses this template completely;
|
||||
a blank direct value leaves the template behavior unchanged.
|
||||
Optional `session_id` uses the same template variables and input helper. Its
|
||||
rendered value is trimmed, omitted if empty, and limited to 256 Unicode code
|
||||
points. A nonblank direct request session ID bypasses the template.
|
||||
|
||||
### Cache Control
|
||||
|
||||
`cache_control` is optional and has these fields:
|
||||
|
||||
| Field | Required | Values |
|
||||
| --- | --- | --- |
|
||||
| `type` | yes | `ephemeral` |
|
||||
| `ttl` | no | Empty or `1h` |
|
||||
|
||||
Promptkit preserves cache-control metadata on the rendered message. The
|
||||
outbound integration determines its wire representation.
|
||||
Optional `cache_control` has required `type: ephemeral` and optional `ttl`,
|
||||
which may be empty or `1h`. Promptkit retains this metadata on rendered
|
||||
messages; the integration contract defines its wire representation.
|
||||
|
||||
### Output Contract
|
||||
|
||||
| Field | Required | Values or behavior |
|
||||
| --- | --- | --- |
|
||||
| `format` | yes | `text`, `markdown`, or `json`. |
|
||||
| `validation_mode` | yes | `none`, `basic`, `json`, or `json_schema`. |
|
||||
| `schema_path` | for `json_schema` | Path to a schema in the configured schema source. |
|
||||
| `repair_attempts` | no | Integer zero or greater; omitted means zero. |
|
||||
| `format` | yes | `text`, `markdown`, or `json` |
|
||||
| `validation_mode` | yes | `none`, `basic`, `json`, or `json_schema` |
|
||||
| `schema_path` | for `json_schema` | Path in the selected schema source |
|
||||
| `repair_attempts` | no | Integer 0–3; default 0. Positive values need `basic`, `json`, or `json_schema` |
|
||||
|
||||
The validation modes behave as follows:
|
||||
`none` skips content validation; `basic` requires nonempty content; `json`
|
||||
accepts any valid JSON value; `json_schema` checks the selected schema.
|
||||
`format` determines output artifact metadata. JSON Schema mode also supplies
|
||||
structured-output metadata to compatible clients. Plain `json` does not
|
||||
request a provider-native JSON-object constraint. A positive repair budget
|
||||
counts additional generations after failed validation; exhaustion returns the
|
||||
last candidate and its failed validation result. Operational errors remain
|
||||
errors. A request-level `OutputContract` replaces the whole configured one,
|
||||
not individual fields; an empty request format defaults to `text`.
|
||||
|
||||
- `none` skips content validation;
|
||||
- `basic` requires non-empty generated content;
|
||||
- `json` requires valid JSON; and
|
||||
- `json_schema` requires valid JSON that satisfies the selected schema.
|
||||
## Built-In Backends
|
||||
|
||||
`format` controls output artifact metadata. JSON Schema mode also supplies the
|
||||
schema to compatible model clients as structured-output metadata. The public
|
||||
engine does not install an output repairer, so its validation is single-pass
|
||||
even when a positive `repair_attempts` value is present.
|
||||
The selected catalog modules contribute these backend definitions as the
|
||||
earliest underlay. Their settings can be overlaid by caller sources; these IDs
|
||||
are not reserved against configuration changes.
|
||||
|
||||
A request-level `OutputContract` replaces the complete prompt output contract.
|
||||
It does not merge individual fields. If its format is empty, Promptkit uses
|
||||
`text`.
|
||||
| ID | Catalog endpoint | API-key environment name | Active generation limit |
|
||||
| --- | --- | --- | ---: |
|
||||
| `openrouter` | `https://openrouter.ai/api/v1` | `OPENROUTER_API_KEY` | 16 |
|
||||
| `rakestrawhome` | `https://inference.ai.rakestrawhome.com/v1` | `RAKESTRAWHOME_INFERENCE_API_KEY` | 4 |
|
||||
|
||||
A backend definition may have `endpoint`, `api_key_env`,
|
||||
`concurrency_limit`, `queue_capacity`, and `extra_params`. The final endpoint
|
||||
is required and must be an absolute HTTP(S) URL with a host, no user
|
||||
information, query, or fragment. An omitted or null API-key environment name
|
||||
means no lookup. Omitted/null concurrency is zero (unlimited); a supplied
|
||||
negative value is invalid. For positive concurrency, omitted/null queue
|
||||
capacity defaults to 1024. An explicit queue capacity, including zero,
|
||||
requires positive concurrency and must be nonnegative. Unlimited concurrency
|
||||
has no queue. Provider maps contain JSON-compatible values; null entries are
|
||||
ordinary JSON null. Unknown fields, including a raw `api_key`, fail.
|
||||
|
||||
## Profile Definitions
|
||||
|
||||
A profile supplies model execution settings:
|
||||
Profiles are keyed by literal ID and may inherit a named parent with
|
||||
`extends`:
|
||||
|
||||
```yaml
|
||||
id: local-summary
|
||||
backend: openrouter
|
||||
model: example-model
|
||||
temperature: 0.2
|
||||
max_tokens: 500
|
||||
top_p: 0.95
|
||||
timeout_seconds: 90
|
||||
service_tier: flex
|
||||
reasoning_effort: medium
|
||||
extra_params:
|
||||
provider_option: enabled
|
||||
profiles:
|
||||
local-summary:
|
||||
backend: local
|
||||
model: example-model
|
||||
temperature: 0.2
|
||||
max_tokens: 500
|
||||
timeout_seconds: 90
|
||||
extra_params:
|
||||
provider_option: enabled
|
||||
local-summary-fast:
|
||||
extends: local-summary
|
||||
timeout_seconds: 30
|
||||
reasoning_effort: low
|
||||
```
|
||||
|
||||
| Field | Required | Meaning |
|
||||
| --- | --- | --- |
|
||||
| `id` | yes | Profile identifier, trimmed before selection and publication. It must be non-empty after trimming and unique within one source after normalization. |
|
||||
| `backend` | unless `endpoint` is present | Backend registry ID. It is trimmed and registry membership is checked when the profile is prepared or inspected. |
|
||||
| `endpoint` | unless `backend` is present | OpenAI-compatible base URL, including an API version path when required. A nonempty value is trimmed and must be absolute HTTP or HTTPS with a host and without user information, a query, or a fragment. When both connection fields are present, this overrides the backend endpoint without changing backend identity. |
|
||||
| `model` | yes | Non-empty provider model name. |
|
||||
| `temperature` | no | Number from 0 through 2. |
|
||||
| `max_tokens` | no | Integer zero or greater. |
|
||||
| `top_p` | no | Number from 0 through 1. |
|
||||
| `timeout_seconds` | no | Per-generation deadline in whole seconds; integer zero or greater. |
|
||||
| `service_tier` | no | Provider-specific request tier. |
|
||||
| `reasoning_effort` | no | Provider-specific reasoning setting. |
|
||||
| `api_key_env` | no | Name of an environment variable containing the API key. |
|
||||
| `extra_params` | no | JSON-compatible provider-specific outbound fields. |
|
||||
|
||||
Raw `api_key` is prohibited in profile YAML. Store only an environment
|
||||
variable name in `api_key_env`.
|
||||
|
||||
Promptkit does not infer a backend from a model or endpoint. Endpoint-only
|
||||
profiles remain supported and have no effective backend ID.
|
||||
The engine always provides the built-in `openrouter` ID. Consumers can add
|
||||
engine-scoped IDs with
|
||||
[`WithBackend`](../backends.go); exact registration validation belongs to its
|
||||
GoDoc.
|
||||
|
||||
`extra_params` accepts null, booleans, finite numbers, strings, arrays, and
|
||||
objects with string keys. Keys must be non-empty. With the built-in client,
|
||||
they also cannot collide with the standard fields listed in the
|
||||
[outbound request contract](integrations/openai-compatible-chat.md#request-body).
|
||||
Excessively deep or large JSON-shaped values are rejected for safety.
|
||||
Profiles may also supply `endpoint`, `top_p`, `service_tier`, `api_key_env`,
|
||||
and `api_key_required`. Promptkit does not infer a backend from an endpoint or
|
||||
model. A profile with only an endpoint has no backend capacity identity.
|
||||
Nonblank endpoints follow the backend URL rules; temperature is 0–2, top_p
|
||||
is 0–1, max_tokens and timeout_seconds are nonnegative. Provider maps accept
|
||||
JSON-compatible values with nonempty keys and cannot collide with the built-in
|
||||
client's [standard request fields](integrations/openai-compatible-chat.md#request-body).
|
||||
Credential values do not belong in profiles: raw `api_key` is invalid.
|
||||
|
||||
### Defaults And Overrides
|
||||
|
||||
Execution settings resolve in this order:
|
||||
After effective profile inheritance, Promptkit applies the selected backend's
|
||||
defaults once, then request `ExecutionTargetOverride` values. Backend-derived
|
||||
settings are not inheritance input: changing a child profile's backend does
|
||||
not carry the parent's derived endpoint or credential source into it. Changing
|
||||
only a model or backend does not clear unrelated explicitly inherited fields.
|
||||
|
||||
1. the framework timeout baseline;
|
||||
2. the selected backend, when the profile names one;
|
||||
3. the selected profile; and
|
||||
4. request `ExecutionTargetOverride` values.
|
||||
| Profile field | Absent | Explicit value | Explicit null |
|
||||
| --- | --- | --- | --- |
|
||||
| `backend` | No backend | Exact backend ID; blank means none | No backend |
|
||||
| `endpoint` | Selected backend endpoint | Valid nonblank URL; empty clears | Clear, without backend fallback |
|
||||
| `model` | Unspecified | Nonblank model; empty clears | Unspecified |
|
||||
| `temperature`, `top_p`, `max_tokens` | Unspecified | Valid value, including outbound zero | Suppress outbound control |
|
||||
| `service_tier`, `reasoning_effort` | Unspecified | Normalized value; blank suppresses | Suppress outbound control |
|
||||
| `timeout_seconds` | 600 seconds | Nonnegative; zero disables generation deadline | Reset to 600 seconds |
|
||||
| `api_key_env` | Backend lookup name, if any | Trimmed name; blank disables | Disable backend lookup |
|
||||
| `api_key_required` | False | Boolean, including false | False |
|
||||
| `extra_params` | Backend defaults | Recursively overlay profile map on backend map | Suppress configured provider map |
|
||||
|
||||
The framework baseline is:
|
||||
|
||||
| Setting | Default |
|
||||
| --- | --- |
|
||||
| `temperature` | Unspecified and omitted from compatible provider requests unless a profile or runtime override selects it. |
|
||||
| `max_tokens` | Unspecified and omitted from compatible provider requests unless a profile or runtime override selects it. |
|
||||
| `top_p` | Unspecified and omitted from compatible provider requests unless a profile or runtime override selects it. |
|
||||
| `timeout_seconds` | `600` |
|
||||
|
||||
Numeric zero in a file or in-memory profile does not select a numeric value.
|
||||
For `temperature`, `max_tokens`, and `top_p`, it leaves the provider control
|
||||
unspecified. For `timeout_seconds`, it retains the framework deadline. Numeric
|
||||
request overrides use pointers, so an explicit zero is retained and sent to
|
||||
compatible providers. In particular, an explicit request `timeout_seconds` of
|
||||
zero disables the per-generation deadline while leaving the caller context and
|
||||
transport timeout intact.
|
||||
|
||||
Non-empty profile strings replace backend defaults, and non-empty request
|
||||
strings replace both. Request reasoning is the exception: a nil
|
||||
`ReasoningEffort` pointer inherits the profile, a pointer to a nonblank string
|
||||
trims and replaces it, and a pointer to a blank string clears it. Backend
|
||||
identity is retained when either layer overrides the endpoint, so the override
|
||||
also retains any engine-local capacity policy configured for that backend.
|
||||
Capacity configuration belongs to the Go
|
||||
[`Backend` API](../backends.go), not prompt or profile YAML. A non-empty
|
||||
`extra_params` map at each layer replaces the entire lower-precedence map
|
||||
rather than merging keys.
|
||||
The [outbound integration contract](integrations/openai-compatible-chat.md)
|
||||
defines how the effective settings are serialized.
|
||||
An empty mapping does not clear inherited map entries; null clears the whole
|
||||
map. Nested maps merge recursively, while scalar and array values replace.
|
||||
Runtime nonempty `ExtraParams` overrides replace the entire configured map.
|
||||
Request overrides otherwise follow the [public value GoDoc](../types.go).
|
||||
For example, a runtime `extra_params["extends"]` key is ordinary provider
|
||||
data, not a configuration inheritance instruction. An explicit zero timeout
|
||||
does not disable the caller context or HTTP transport timeout.
|
||||
|
||||
### Source And Profile Precedence
|
||||
|
||||
An explicit request profile ID takes precedence over the prompt's
|
||||
`default_profile`. If neither is present, preparation fails. Exact profile
|
||||
inspection instead takes one explicit profile ID and does not use a prompt
|
||||
default.
|
||||
`NewEngine` composes catalog definitions first, Promptkit-owned definitions
|
||||
next, then caller underlays in their given order, with an optional main input
|
||||
last. Later tiers overlay earlier ones. Inputs within one underlay group are
|
||||
strictly composed: conflicting contributions are errors, not priority
|
||||
signals. The directory helpers discover `.yaml` and `.yml` files recursively
|
||||
in sorted order, but that order does not give one file higher precedence.
|
||||
An explicitly selected missing/unreadable root fails; an empty root contributes
|
||||
nothing. Imports are disabled by these helpers unless the caller enables them
|
||||
on an input. Single files can use ConfigKit inputs directly.
|
||||
|
||||
Profile sources resolve matching IDs in this order:
|
||||
Backends, profiles, and prompts all follow these same source rules. A later
|
||||
profile or backend can change individual fields of a lower definition. A
|
||||
request `ProfileID` takes precedence over a prompt `default_profile`; if
|
||||
neither is selected, preparation fails. Exact profile inspection selects one
|
||||
explicit ID and does not consult a prompt default.
|
||||
|
||||
1. in-memory profiles supplied with `WithProfiles`;
|
||||
2. the ordinary configured source selected by a profile file, `fs.FS`, or
|
||||
configured profile directory;
|
||||
3. application fallback profiles supplied with `WithFallbackProfileFS`; and
|
||||
4. embedded built-in profiles.
|
||||
### Profile Inheritance
|
||||
|
||||
A profile source supplies a complete definition; definitions and their fields
|
||||
are not merged across sources. A higher-precedence source falls back only when
|
||||
the requested profile ID is absent. An invalid matching profile is an error and
|
||||
does not fall back. In-memory `Profile` values follow the same ranges as YAML
|
||||
profiles. They use `APIKeyRequired` for request-scoped credentials instead of
|
||||
`api_key_env`. Preparation and exact profile inspection use this same source
|
||||
precedence.
|
||||
ConfigKit resolves `extends` on effective profile definitions. One profile
|
||||
may inherit another keyed profile; presence, explicit null, and recursively
|
||||
merged maps determine the final fields. References are literal IDs, including
|
||||
IDs containing punctuation. There is no Promptkit-specific depth limit or
|
||||
second base-chain resolver. Missing references and cycles fail at construction.
|
||||
For generic inheritance and layer mechanics, consult
|
||||
[ConfigKit](https://gitea.maximumdirect.net/eric/ConfigKit).
|
||||
|
||||
## Built-In Profile Catalog
|
||||
|
||||
Every built-in selects the `openrouter` backend. The engine's built-in backend
|
||||
registry supplies `https://openrouter.ai/api/v1` and the environment-variable
|
||||
name `OPENROUTER_API_KEY`, so individual profiles contain only model and
|
||||
generation settings. Built-in profile files do not repeat those connection
|
||||
values. A configured, application fallback, or in-memory profile with the same
|
||||
profile ID takes precedence.
|
||||
Promptkit contributes two opt-in definitions; neither is selected by default:
|
||||
|
||||
| Provider | ID | Model |
|
||||
| --- | --- | --- |
|
||||
| aion-labs | `aion-2` | `aion-labs/aion-2.0` |
|
||||
| anthropic | `claude-fable-latest` | `~anthropic/claude-fable-latest` |
|
||||
| anthropic | `claude-haiku-latest` | `~anthropic/claude-haiku-latest` |
|
||||
| anthropic | `claude-opus-latest` | `~anthropic/claude-opus-latest` |
|
||||
| anthropic | `claude-sonnet-latest` | `~anthropic/claude-sonnet-latest` |
|
||||
| deepseek | `deepseek-3-2` | `deepseek/deepseek-v3.2` |
|
||||
| deepseek | `deepseek-4-flash` | `deepseek/deepseek-v4-flash` |
|
||||
| deepseek | `deepseek-4-pro` | `deepseek/deepseek-v4-pro` |
|
||||
| google | `gemini-2-flash` | `google/gemini-2.5-flash` |
|
||||
| google | `gemini-2-flash-lite` | `google/gemini-2.5-flash-lite` |
|
||||
| google | `gemini-2-pro` | `google/gemini-2.5-pro` |
|
||||
| google | `gemini-3-flash-lite` | `google/gemini-3.1-flash-lite` |
|
||||
| google | `gemini-flash-latest` | `~google/gemini-flash-latest` |
|
||||
| google | `gemini-pro-latest` | `~google/gemini-pro-latest` |
|
||||
| google | `gemma-4-31b` | `google/gemma-4-31b-it:exacto` |
|
||||
| minimax | `minimax-m2` | `minimax/minimax-m2.5` |
|
||||
| minimax | `minimax-m3` | `minimax/minimax-m3` |
|
||||
| mistral | `mistral-large-2512` | `mistralai/mistral-large-2512` |
|
||||
| mistral | `mistral-medium-3-5` | `mistralai/mistral-medium-3-5` |
|
||||
| mistral | `mistral-small-3` | `mistralai/mistral-small-3.2-24b-instruct` |
|
||||
| mistral | `mistral-small-4` | `mistralai/mistral-small-2603` |
|
||||
| nvidia | `nemotron-3-ultra` | `nvidia/nemotron-3-ultra-550b-a55b` |
|
||||
| openai | `gpt-5-mini` | `openai/gpt-5.4-mini` |
|
||||
| openai | `gpt-5-nano` | `openai/gpt-5.4-nano` |
|
||||
| ID | Current definition |
|
||||
| --- | --- |
|
||||
| `promptkit-openrouter-standard` | Extends the selected OpenRouter catalog's `gemini-flash-latest` profile. |
|
||||
| `promptkit-openrouter-auto` | Selects backend `openrouter` and model `openrouter/auto`. |
|
||||
|
||||
Select one through request `ProfileID` or prompt `default_profile`.
|
||||
Caller layers may overlay these definitions or their catalog bases. The
|
||||
[release policy](policy/release.md#catalog-profile-compatibility) owns their
|
||||
cross-version compatibility boundary.
|
||||
|
||||
## External Profile Inventories
|
||||
|
||||
The selected catalog modules, not this document, own their profile inventories:
|
||||
|
||||
- [OpenRouter catalog v3.20260930.0 profiles](https://gitea.maximumdirect.net/eric/promptkit-backend-openrouter/src/tag/v3.20260930.0/catalog/profiles)
|
||||
- [Rakestrawhome catalog v3.20260930.0 profiles](https://gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/src/tag/v3.20260930.0/catalog/profiles)
|
||||
|
||||
These are tested local data, not a promise of live provider availability.
|
||||
The [release policy](policy/release.md#catalog-profile-compatibility) owns
|
||||
the compatibility boundary.
|
||||
|
||||
## Schemas
|
||||
|
||||
Schemas are JSON documents selected by a prompt or request
|
||||
`schema_path`. For a directory or `fs.FS` source, paths resolve within the
|
||||
configured source root. Referenced nested schemas resolve relative to the
|
||||
owning schema document. `WithSchemaFile` exposes one schema, addressed by its
|
||||
base name.
|
||||
|
||||
An unreadable, invalid, or unresolvable schema produces an operational
|
||||
validation error. Generated content that is valid JSON but does not satisfy the
|
||||
schema produces a failed validation result.
|
||||
Schemas are JSON documents selected by a prompt or request `schema_path`.
|
||||
Directory and `fs.FS` schema paths resolve within the configured root;
|
||||
referenced nested schemas resolve relative to the owning schema document.
|
||||
`WithSchemaFile` exposes one schema under its base name. An unreadable,
|
||||
invalid, or unresolvable schema causes an operational validation error.
|
||||
Valid JSON that fails the schema instead yields a failed validation result.
|
||||
|
||||
## Credentials
|
||||
|
||||
Credential values belong at the request or environment boundary, never in
|
||||
prompt, profile, schema, or example files:
|
||||
Only environment-variable names belong in backend/profile definitions, never
|
||||
credential values. A request may supply a direct `APIKey` or an `APIKeyEnv`
|
||||
override. A direct key takes precedence, followed by a request environment
|
||||
source, an explicit profile source, and an inherited backend source. A true
|
||||
profile `api_key_required` suppresses the backend-derived environment source,
|
||||
but an explicit profile or request environment source remains usable. False
|
||||
restores ordinary optional-credential behavior. Missing or blank optional
|
||||
environment values leave the built-in client's Authorization header absent;
|
||||
required availability is checked at preparation and again during prepared
|
||||
execution. Injected clients own their credential-resolution behavior.
|
||||
|
||||
- a file profile names an environment variable with `api_key_env`;
|
||||
- an in-memory profile may set `APIKeyRequired`;
|
||||
- a request can provide a direct `APIKey` or override `APIKeyEnv`; and
|
||||
- a direct request key takes precedence over environment lookup.
|
||||
|
||||
After a direct request key, the credential-source precedence is request
|
||||
`APIKeyEnv`, profile `api_key_env`, then the backend default. An in-memory
|
||||
profile with `APIKeyRequired` clears an inherited backend environment name and
|
||||
requires a direct key unless the request explicitly supplies `APIKeyEnv`.
|
||||
Promptkit validates required credential availability during preparation.
|
||||
Direct keys are excluded from JSON results and redacted by public string
|
||||
formatters. Environment-variable names may appear in prepared metadata, but
|
||||
their values do not.
|
||||
Credential values are excluded from JSON results and redacted by public
|
||||
string formatters. ConfigKit inspection redacts credential-shaped dynamic
|
||||
provider keys, but does not make arbitrary prompt/provider content safe to
|
||||
disclose. Applications own report disclosure and secret handling.
|
||||
|
||||
@@ -31,11 +31,13 @@ does not serialize it in the provider request.
|
||||
|
||||
## Authentication
|
||||
|
||||
A non-empty API key supplied directly on the execution target takes
|
||||
precedence. Otherwise, when an API-key environment-variable name is supplied,
|
||||
the client reads that variable and requires a non-empty value. The selected
|
||||
key is sent as `Authorization: Bearer <key>`. No authorization header is sent
|
||||
when neither mechanism is configured.
|
||||
A usable API key supplied directly on the execution target takes precedence.
|
||||
Otherwise, when an API-key environment-variable name is supplied, the client
|
||||
reads and trims that variable. A bearer header is sent only when the resolved
|
||||
direct or environment credential is non-empty. When neither source is usable,
|
||||
the client omits `Authorization` and handles the provider response normally.
|
||||
An explicitly required target with no usable source is rejected before
|
||||
transport.
|
||||
|
||||
The target contains the already resolved environment-variable name: an
|
||||
explicit request override takes precedence over profile metadata, which takes
|
||||
@@ -53,6 +55,12 @@ Each ordinary message contains its `role` and string `content`. A
|
||||
cache-controlled message instead uses a text content block containing `type`,
|
||||
`text`, and `cache_control`; an empty cache-control TTL is omitted.
|
||||
|
||||
Promptkit sends only `developer`, `system`, `user`, and `assistant` roles and
|
||||
does so without provider-specific translation. Tool and deprecated function
|
||||
payloads are outside this text-message contract. A backend or model that
|
||||
rejects an otherwise supported role or context returns its ordinary provider
|
||||
error, which follows the normal generation-error path.
|
||||
|
||||
The effective direct or prompt-rendered session ID is trimmed, limited to 256
|
||||
Unicode code points, and sent when nonempty as top-level `session_id`. It is
|
||||
never also sent as a session header.
|
||||
@@ -65,7 +73,8 @@ The client conditionally includes:
|
||||
- non-empty `service_tier` and effective `reasoning_effort`; an explicitly
|
||||
disabled reasoning setting is empty and therefore omitted; and
|
||||
- `response_format` for JSON Schema structured output, including its name,
|
||||
strict flag, and schema document.
|
||||
strict flag, and schema document. Plain JSON validation does not add an
|
||||
object-only response constraint.
|
||||
|
||||
The engine resolves backend, profile, and request extra-parameter maps by
|
||||
whole-map replacement rather than key merging. The resulting effective map is
|
||||
@@ -97,21 +106,74 @@ drained.
|
||||
|
||||
The bounded body must contain exactly one OpenAI-compatible JSON response
|
||||
object followed only by JSON whitespace and EOF. The client returns the first
|
||||
choice's non-empty message content and maps prompt, completion, total, cached,
|
||||
and cache-write token counts. Invalid or truncated JSON, trailing non-whitespace
|
||||
data, a second JSON value, absent choices, empty first-choice content, and size
|
||||
overflow are malformed responses and return no partial result.
|
||||
choice's explicitly present string message content, including an empty or
|
||||
whitespace-only string, and maps prompt, completion, total, cached, and
|
||||
cache-write token counts. Invalid or truncated JSON, trailing non-whitespace
|
||||
data, a second JSON value, absent choices, missing content, `null` content,
|
||||
non-string content, and size overflow are malformed responses and return no
|
||||
partial result.
|
||||
|
||||
For a non-2xx status, the error includes the status code but never the provider
|
||||
response body. Promptkit does not yet parse provider error envelopes; bounded
|
||||
non-success parsing belongs to the
|
||||
[structured-generation-error roadmap](../roadmap/structured-generation-errors.md).
|
||||
Each of the five token fields retains whether the provider reported a
|
||||
non-null integer, including zero. Omitted and `null` fields are unknown for
|
||||
observations. A strictly decoded successful envelope can yield observed usage
|
||||
even when a later missing choice or content makes the attempt malformed.
|
||||
Invalid framing and non-2xx error bodies supply no observed usage; provider
|
||||
work may still have occurred. Reported counts are not inferred, reconciled,
|
||||
or proof of complete cost accounting.
|
||||
|
||||
For a non-2xx status, Promptkit recognizes one JSON document with a top-level
|
||||
object-valued `error` member. Its optional `message` and `type` fields must be
|
||||
strings, and `code` may be a string or JSON number. Valid supported fields are
|
||||
handled independently, numeric codes retain their JSON number text, and
|
||||
unknown fields are ignored. Missing, invalid, malformed, or multiply framed
|
||||
envelopes contribute no provider detail.
|
||||
|
||||
Non-success bodies have a 65,536-byte limit. A larger declared
|
||||
`Content-Length` is not read; otherwise the client reads at most one additional
|
||||
byte to detect streamed or underreported overflow. Empty, unreadable,
|
||||
oversized, malformed, and unrecognized bodies retain only the received status.
|
||||
The body is always closed and no oversized stream is drained beyond that probe.
|
||||
|
||||
Extracted strings are made valid UTF-8, trimmed, and converted to one line by
|
||||
collapsing Unicode whitespace, control, and format-character runs. Blank
|
||||
values are omitted. Codes and types longer than 256 Unicode code points are
|
||||
omitted; messages longer than 4,096 code points are truncated at a code-point
|
||||
boundary with an ellipsis inside the limit. Promptkit never exposes raw bodies,
|
||||
headers, endpoints, credentials, request data, schemas, generated content, or
|
||||
unsupported provider metadata through this handling.
|
||||
|
||||
An outbound `http.Client.Do` failure retains both Promptkit's request-failure
|
||||
identity and the exact transport error for `errors.Is` and `errors.As` checks.
|
||||
The rendered error does not include the selected endpoint, request headers,
|
||||
request content, credentials, or provider body.
|
||||
|
||||
## Transient Status Retries
|
||||
|
||||
The built-in path retries only HTTP 429, 500, 502, 503, and 504. It makes two
|
||||
additional attempts by default; engine configuration or an individual run can
|
||||
choose a different nonnegative count, including zero for single-attempt
|
||||
behavior. The same budget applies separately to initial generation and each
|
||||
structural repair generation. An injected `LLMClient` is dispatched once by
|
||||
Promptkit regardless of this setting. A custom `http.Client` on the built-in
|
||||
path remains subject to these retries.
|
||||
|
||||
Each retry uses equal jitter between half and all of an exponential local
|
||||
delay, starting at 500 milliseconds and capped at 8 seconds. The first
|
||||
`Retry-After` header value sets a minimum not-before time measured from
|
||||
response receipt if it is valid unsigned decimal seconds or an HTTP date.
|
||||
Invalid values use local delay; past dates impose no extra delay. The provider
|
||||
minimum is not capped at 8 seconds. Retry waiting is cancellable and holds no
|
||||
active backend permit. A caller without a deadline can wait a long time for
|
||||
provider timing.
|
||||
|
||||
The request body, selected URL, and resolved authorization are fixed across
|
||||
attempts of one generation. Each attempt creates a fresh HTTP request and
|
||||
receives its own configured generation and HTTP-client timeout after capacity
|
||||
admission. The caller context covers every attempt and wait without a reset.
|
||||
Transport failures, malformed responses, noneligible statuses, and invalid
|
||||
generated content are not transient retries. A positive budget ending in
|
||||
eligible statuses reports retry exhaustion with the final provider cause.
|
||||
|
||||
## Timeout And Cancellation
|
||||
|
||||
Timeouts are layered:
|
||||
|
||||
@@ -14,7 +14,7 @@ stable JSON values.
|
||||
|
||||
## Construction And Pool Lifecycle
|
||||
|
||||
Each root `NewEngine` call obtains a normalized capacity-policy snapshot from
|
||||
Each internal engine construction obtains a normalized capacity-policy snapshot from
|
||||
its immutable backend registry and constructs a new `Manager`. The manager
|
||||
creates one pool for each limited backend ID. It has no package-global mutable
|
||||
state, background workers, shutdown protocol, or persistence, so engines with
|
||||
@@ -54,38 +54,48 @@ reserve another bounded slot.
|
||||
|
||||
## FIFO Generation Permits
|
||||
|
||||
`NewClient` wraps the engine's selected internal model client after public
|
||||
client adaptation or built-in client construction. Initial generation and the
|
||||
default repairer receive the same wrapper. Their requests retain the same
|
||||
effective backend, credential, numeric-presence metadata, and structured-output
|
||||
settings, so scheduling does not change provider omission semantics between
|
||||
calls.
|
||||
The engine supplies one [generation coordinator](generation.md) to initial
|
||||
generation and the default repairer. Their requests retain the same effective
|
||||
backend, credential, numeric-presence metadata, structured-output settings,
|
||||
and retry budget. Scheduling does not change provider omission semantics
|
||||
between calls.
|
||||
|
||||
For each `Generate` call, the wrapper selects a pool from the request's
|
||||
effective backend ID. An unlimited call passes directly to the next client. A
|
||||
limited call acquires an active permit, invokes the next client, and defers
|
||||
permit release so ordinary returns and panic unwinding both restore capacity.
|
||||
For each model attempt, the coordinator selects a pool from the request's
|
||||
effective backend ID. An unlimited call uses the unrestricted path. A limited
|
||||
call acquires an active permit and defers release around that attempt, so
|
||||
ordinary returns and panic unwinding both restore capacity.
|
||||
Preparation and validation never hold an active permit.
|
||||
|
||||
The manager's `Acquire` operation owns that active-permit acquisition and
|
||||
returns an idempotent release function. The coordinator uses it around
|
||||
individual attempts. The earlier one-shot wrapper still delegates to the same
|
||||
operation for focused internal callers and tests, but engine assembly no
|
||||
longer uses it. There is one queue and one active limit per limited backend pool.
|
||||
|
||||
When all active permits are occupied, calls join a mutex-protected FIFO waiter
|
||||
list. Releasing a permit transfers it directly to the oldest remaining waiter
|
||||
before making it generally available. Pools do not order work relative to
|
||||
other backend IDs.
|
||||
|
||||
The wrapper passes generation requests, responses, and collaborator errors
|
||||
through unchanged. It owns scheduling only; the concrete model client remains
|
||||
responsible for provider transport behavior. The runner, rather than the
|
||||
The coordinator does not retry injected clients; it normalizes a nil successful
|
||||
response as malformed generation. The built-in HTTP client remains responsible
|
||||
for provider transport behavior. The runner, rather than the
|
||||
capacity layer, sums all five usage fields from the initial response and every
|
||||
completed repair response into the successful run result.
|
||||
|
||||
An invocation-local observer receives attempt events after active permit
|
||||
release, including canceled acquisition without dispatch. A retry wait holds
|
||||
the logical admission lease but no active permit. Observer callbacks must not
|
||||
block on work that requires the same run's admission slot.
|
||||
|
||||
## Cancellation And Release
|
||||
|
||||
Admission checks the caller context before reserving a slot. A call canceled
|
||||
while waiting for an active permit removes its waiter under the same pool lock
|
||||
used to grant permits. If cancellation removes the waiter first, the wrapped
|
||||
client is not invoked. If a concurrent grant wins first, the call owns the
|
||||
permit and invokes the client with the original context, allowing the client
|
||||
to observe cancellation normally.
|
||||
used to grant permits. If cancellation removes the waiter first, no model
|
||||
attempt is dispatched. If a concurrent grant wins first, the call owns the
|
||||
permit; the coordinator checks the context before dispatch and releases the
|
||||
permit if cancellation has already occurred.
|
||||
|
||||
This grant-or-cancel decision prevents lost and double-released permits.
|
||||
Admission leases and active permits are released after success, collaborator
|
||||
@@ -111,6 +121,8 @@ release. The
|
||||
[external package capacity tests](../../capacity_contract_test.go) own the
|
||||
assembled public-engine behavior for configured limits, capacity errors,
|
||||
endpoint identity, engine independence, and injected clients. The
|
||||
[reliable execution contract tests](../../reliable_execution_contract_test.go)
|
||||
cover shared and independent backend pools during retry and repair. The
|
||||
[prepared-execution contract tests](../../prepared_execution_contract_test.go)
|
||||
own the public prepared-capacity boundary. The
|
||||
[root error-boundary tests](../../errors_internal_test.go) own preservation of
|
||||
|
||||
50
docs/internal/generation.md
Normal file
50
docs/internal/generation.md
Normal file
@@ -0,0 +1,50 @@
|
||||
# Internal Generation Coordination
|
||||
|
||||
`internal/generation` coordinates one model-generation request through the
|
||||
existing engine-local [capacity manager](capacity.md) and the
|
||||
[built-in HTTP client](llm.md). The engine supplies the same coordinator to
|
||||
initial generation and the default output repairer.
|
||||
|
||||
The coordinator has explicit built-in and injected-client constructors. The
|
||||
built-in path prepares the provider request once, then acquires and releases an
|
||||
active backend permit around each actual HTTP attempt. Only HTTP 429, 500, 502,
|
||||
503, and 504 can lead to another attempt. The injected-client path acquires a
|
||||
permit and invokes its client once, regardless of the internal retry budget.
|
||||
The runner continues to own the logical execution's admission lease. A request
|
||||
can override the engine's retry budget; the runner freezes that choice for
|
||||
prepared execution and passes it to each repair generation.
|
||||
|
||||
Retry waits hold no active permit. The local delay uses equal jitter between
|
||||
half and all of a capped exponential base, beginning at 500 milliseconds and
|
||||
capping at 8 seconds. A valid provider not-before time can extend that delay;
|
||||
the caller context interrupts it. The next attempt reacquires the ordinary
|
||||
FIFO permit. Preparation failures dispatch nothing. When a positive retry
|
||||
budget ends with eligible failures, the internal exhaustion error retains the
|
||||
final cause and actual dispatched-attempt count.
|
||||
|
||||
When an invocation-local [observation recorder](observation.md) is present in
|
||||
the context, each acquisition emits an attempt event, including canceled
|
||||
capacity waits without dispatch. Each actual retry wait emits a delay event,
|
||||
including interrupted waits. Active permits are released before callbacks;
|
||||
generation duration ends before callback delivery. Built-in successful-envelope
|
||||
usage retains reported-field presence, while injected clients use explicit
|
||||
presence when supplied and otherwise treat zero fields as unknown.
|
||||
Malformed successful envelopes can retain strictly decoded usage; error
|
||||
response bodies are never mined for it. Arbitrary injected failures carry
|
||||
unknown usage. Transport, timeout, malformed, and noneligible status failures
|
||||
stop the sequence without retry exhaustion. Context is checked before each
|
||||
wait, acquisition, and dispatch.
|
||||
|
||||
The public client adapter translates explicitly supplied public transport and
|
||||
malformed-response identities into internal categories while retaining the
|
||||
original error chain. This keeps injected attempt classifications consistent
|
||||
with terminal observations without parsing diagnostic text or retrying injected
|
||||
clients.
|
||||
|
||||
Focused coordinator tests use controlled transport and wait functions. They
|
||||
cover eligibility, exhaustion, cancellation, backoff, request freezing, permit
|
||||
release, and the injected-client path. The capacity manager tests own permit
|
||||
acquisition and idempotent release. Public
|
||||
[integration tests](../../reliable_execution_contract_test.go) cover retry,
|
||||
repair, observation, frozen prepared execution, and shared-backend behavior
|
||||
through the assembled facade.
|
||||
@@ -10,7 +10,7 @@ owns the observable outbound HTTP contract. The
|
||||
[framework format reference](../formats.md) owns the profile and prompt
|
||||
settings consumed by the client.
|
||||
|
||||
The concrete client remains under `internal/llm`. The root engine assembles it
|
||||
The concrete client remains under `internal/llm`. The internal engine assembles it
|
||||
as the default implementation behind Promptkit's public client boundary.
|
||||
|
||||
## Components And Flow
|
||||
@@ -21,31 +21,39 @@ uses internal domain values for rendered prompts, execution targets,
|
||||
structured output, responses, and token usage.
|
||||
|
||||
The runner supplies a fully resolved target after applying backend, profile,
|
||||
and request precedence. The client uses its endpoint, credential metadata,
|
||||
generation fields, and extra parameters. `BackendID` remains routing metadata
|
||||
and request precedence, plus canonical provider-bound text messages. The
|
||||
client uses its endpoint, credential metadata, generation fields, and extra
|
||||
parameters. `BackendID` remains routing metadata
|
||||
for the generation boundary and is not mapped into the provider payload.
|
||||
|
||||
Construction trims and validates a nonempty configured base URL and clones any
|
||||
supplied `http.Client` so Promptkit can apply its timeout default without
|
||||
mutating the caller's client. An empty configured base remains valid because a
|
||||
resolved request target may supply the endpoint. Generation then:
|
||||
resolved request target may supply the endpoint. Generation preparation then:
|
||||
|
||||
1. validates shared execution-setting invariants and the final selected base
|
||||
endpoint;
|
||||
2. maps the internal request into the OpenAI-compatible chat payload;
|
||||
3. validates and merges extra parameters;
|
||||
4. composes `/chat/completions` through parsed URL path operations;
|
||||
5. resolves authentication;
|
||||
6. performs the outbound request under the applicable deadlines; and
|
||||
7. decodes one strictly framed, size-bounded response object and maps its first
|
||||
choice and token usage.
|
||||
5. resolves authentication and freezes the URL, body, and optional credential.
|
||||
|
||||
Each execution of that prepared request creates a fresh HTTP request, body
|
||||
reader, and attempt timeout context. It closes the response body and returns
|
||||
the HTTP status, decoded response or error, reported token values and presence,
|
||||
and any valid provider `Retry-After` not-before time. A strictly decoded
|
||||
successful envelope can retain usage even if its first choice or content is
|
||||
missing. Invalid framing and non-success error bodies supply no usage. The
|
||||
ordinary `Generate` method prepares and executes once and returns no partial
|
||||
response on failure.
|
||||
|
||||
`internal/llm` owns the set of reserved OpenAI-compatible request fields used
|
||||
when validating extra parameters. Backend registration consumes the same rule
|
||||
without making the model client depend on registry configuration.
|
||||
|
||||
The implementation has no retry loop, tool-call support, provider catalog,
|
||||
inbound HTTP behavior, or durable session store.
|
||||
This client does not schedule retries; the [generation coordinator](generation.md)
|
||||
does so around its prepared attempts. The client has no tool-call support,
|
||||
provider catalog, inbound HTTP behavior, or durable session store.
|
||||
|
||||
## Prepared Generation
|
||||
|
||||
@@ -54,12 +62,14 @@ the target, rendered messages, and structured-output constraint retained by
|
||||
executable preparation. Execution does not reopen or rerender consumer
|
||||
sources.
|
||||
|
||||
Before backend admission, the runner rechecks that the frozen credential
|
||||
environment-variable name is available. The handle does not retain the
|
||||
environment value; the model client resolves the value visible when generation
|
||||
begins. A direct request key remains in private execution state only until the
|
||||
claimed execution finishes or an unclaimed handle is discarded. Exact public
|
||||
ownership and redaction semantics belong to the
|
||||
Before backend admission, the runner rechecks a frozen credential
|
||||
environment-variable name only when the target explicitly requires a
|
||||
credential. The handle does not retain the environment value; the model client
|
||||
resolves the value visible when generation begins. For optional sources with no
|
||||
usable value, the built-in client omits `Authorization` and continues to the
|
||||
provider. A direct request key remains in private execution state only until
|
||||
the claimed execution finishes or an unclaimed handle is discarded. Exact
|
||||
public ownership and redaction semantics belong to the
|
||||
[`PreparedExecution` GoDoc](../../prepared_execution.go).
|
||||
|
||||
## Failure Categories
|
||||
@@ -67,21 +77,44 @@ ownership and redaction semantics belong to the
|
||||
The package preserves distinct error identities for invalid client
|
||||
configuration, invalid generation requests, request execution failures,
|
||||
non-success provider statuses, and malformed successful responses. Provider
|
||||
response bodies are not included in non-success errors.
|
||||
response bodies are never exposed in raw form through non-success errors.
|
||||
|
||||
Invalid nonempty configured endpoints are configuration failures. A missing or
|
||||
invalid final selected endpoint is an invalid generation request and is
|
||||
rejected before transport.
|
||||
|
||||
Authentication resolves a trimmed direct key before a trimmed configured
|
||||
environment value. Optional missing, empty, or whitespace-only sources do not
|
||||
block transport and produce no `Authorization` header. An explicitly required
|
||||
target with no usable source is rejected before transport with the existing
|
||||
invalid-request diagnostics.
|
||||
|
||||
Successful response bodies have a fixed 16 MiB limit enforced by declared
|
||||
length and by reading at most one byte beyond the boundary. The decoder accepts
|
||||
exactly one JSON object plus trailing whitespace and EOF. Size overflow,
|
||||
truncation, malformed JSON, trailing data, and a second value are malformed
|
||||
responses with no partial result or provider content in the error. Every body
|
||||
is closed, and an unbounded oversized stream is not drained. Non-success
|
||||
responses remain status-only; bounded provider error-envelope parsing belongs
|
||||
to the
|
||||
[structured-generation-error roadmap](../roadmap/structured-generation-errors.md).
|
||||
is closed, and an unbounded oversized stream is not drained.
|
||||
|
||||
After framing succeeds, the first choice must contain an explicitly present
|
||||
string `message.content`. The string is returned exactly, including empty or
|
||||
whitespace-only content. Missing choices, missing or `null` content, and
|
||||
non-string content are malformed responses. Output validation and correction
|
||||
eligibility remain outside this package.
|
||||
|
||||
Successful-envelope usage is decoded with independent presence bits for each
|
||||
token field: omission and `null` are unknown, while a reported integer zero is
|
||||
known. Strictly decoded usage can still be observed if later choice/content
|
||||
checks fail; invalid framing or a non-success error body contributes none.
|
||||
Existing successful-result accounting remains value-based.
|
||||
|
||||
For a non-success response, `ProviderHTTPError` retains the HTTP status and
|
||||
only normalized detail from the bounded recognized envelope. It retains
|
||||
`ErrUnexpectedStatus` through unwrapping. The client owns response closure;
|
||||
its bounded reader and parser never close or drain a body themselves. The root
|
||||
facade converts this concrete internal error into the public
|
||||
[`GenerationError`](../../generation_error.go), while arbitrary injected-client
|
||||
errors continue through the ordinary generation-error mapping unchanged.
|
||||
|
||||
An `http.Client.Do` failure is represented by a redacting multi-cause error:
|
||||
the package request-failure sentinel and the exact returned transport error are
|
||||
@@ -99,10 +132,14 @@ The
|
||||
own configuration, client cloning, deterministic deadline precedence,
|
||||
authentication, request and response mapping, malformed data, error identity,
|
||||
cancellation, endpoint selection and composition, pre-transport rejection, and
|
||||
bounded single-document response framing, closure, and response-body
|
||||
suppression. The root
|
||||
transport contract tests also verify that resolved backend settings reach this
|
||||
client without serializing backend identity and that ordinary-run cancellation
|
||||
retains its public generation and context identities. All use local test
|
||||
servers or controlled test transports; the default suite makes no live or paid
|
||||
provider requests.
|
||||
bounded single-document successful-response framing, closure, and
|
||||
response-body suppression. The focused
|
||||
[provider HTTP error tests](../../internal/llm/provider_http_error_test.go)
|
||||
own envelope parsing, normalization, and bounded-reader cases; their
|
||||
[transport tests](../../internal/llm/provider_http_error_transport_test.go)
|
||||
own non-success response closure and integration. Root transport contract tests
|
||||
own public `GenerationError` conversion, while also verifying that resolved
|
||||
backend settings reach this client without serializing backend identity and
|
||||
that ordinary-run cancellation retains its public generation and context
|
||||
identities. All use local test servers or controlled test transports; the
|
||||
default suite makes no live or paid provider requests.
|
||||
|
||||
48
docs/internal/observation.md
Normal file
48
docs/internal/observation.md
Normal file
@@ -0,0 +1,48 @@
|
||||
# Internal Execution Observation
|
||||
|
||||
`internal/observation` owns one invocation's run ID, presence-aware token
|
||||
aggregation, measured phase totals, and synchronous metadata event delivery.
|
||||
Its event values and callback type live in `internal/domain` so internal
|
||||
requests can carry them without depending on the recorder package. The
|
||||
recorder supplies run IDs to the runner and receives attempt and
|
||||
retry-delay measurements from the generation coordinator. Ordinary `Run`
|
||||
lifecycle instrumentation spans the facade and runner boundaries.
|
||||
Prepared execution starts only after its private payload is atomically claimed;
|
||||
the facade receives that invocation's recorder privately and emits the terminal
|
||||
event after public error mapping. Failed claims and preparation emit nothing.
|
||||
|
||||
The facade masks inherited recorder and prepared-claim state when entering
|
||||
preparation or execution. Delegated calls from an injected collaborator retain
|
||||
the caller's cancellation, deadline, and unrelated context values, but do not
|
||||
reuse the enclosing invocation's accounting. Only the new request's observer
|
||||
(or the claimed handle's retained observer) enables its observation lifecycle.
|
||||
|
||||
A recorder keeps only counters, resolved identity fields, and the selected
|
||||
callback. It has no global history, queue, goroutine, or mutex held across a
|
||||
callback. The caller supplies normalized failure metadata; the recorder never
|
||||
parses an error string or receives provider bodies, credentials, headers, or
|
||||
other content. Events are values with no mutable containers. Callback panics
|
||||
propagate to the caller.
|
||||
Per-operation callbacks are ordered and synchronous; different operations may
|
||||
invoke the same callback concurrently. The recorder marks each measured phase's
|
||||
end before delivery, so callback work does not inflate provider duration.
|
||||
Terminal phase durations sum measured phases, while wall duration may also
|
||||
include preparation, validation, and callbacks. Panics do not promise a
|
||||
terminal event.
|
||||
|
||||
Attempt usage is known when its explicit presence bit is true. For legacy
|
||||
injected clients without presence metadata, nonzero counts are known and
|
||||
zeros are unknown. Aggregate values sum known contributions; `Known` means
|
||||
at least one reported value, while `Complete` means every dispatched attempt
|
||||
reported the field. Zero dispatched attempts produce known, complete zero
|
||||
totals. Non-dispatched acquisitions contribute capacity wait but no usage or
|
||||
generation duration. Terminal totals count dispatched attempts, transient
|
||||
attempts above the first in a generation, and repair generations with at least
|
||||
one dispatch. A terminal event is an aggregate and must not be added to its
|
||||
attempt events.
|
||||
|
||||
The [recorder tests](../../internal/observation/recorder_test.go) own usage
|
||||
presence, phase sums, event ordering, empty-run accounting, identity snapshots,
|
||||
and callback panic behavior. The [generation coordinator](generation.md) owns
|
||||
attempt and retry-delay measurements and emits their completion events after
|
||||
releasing active permits.
|
||||
@@ -11,25 +11,30 @@ contributor workflow and validation.
|
||||
|
||||
| Component | Implemented responsibility | References |
|
||||
| --- | --- | --- |
|
||||
| Root `promptkit` package | Provides the supported engine facade, source, backend-registration, and injection options, public request, result, prompt-inspection, and profile-inspection values, opaque prepared-execution handles, profile construction, extension interfaces, value conversion, redacted formatting, typed capacity errors, public error mapping, and engine-local profile-source assembly including application fallbacks. | [Package GoDoc](../../doc.go), [prepared execution](../../prepared_execution.go), [backend API](../../backends.go), [engine assembly](../../engine.go) |
|
||||
| `examples/go-library/prepare` | Demonstrates an offline downstream consumer using a prompt file, in-memory profile, inline input, and `Prepare`. It is not a public library package. | [Example program](../../examples/go-library/prepare/main.go) |
|
||||
| `examples/go-library/run` | Demonstrates an offline downstream consumer using a prompt file, in-memory profile, inline input, an injected deterministic model client, and `Run`. It is not a public library package. | [Example program](../../examples/go-library/run/main.go) |
|
||||
| `internal/backend` | Constructs each engine's immutable registry from the built-in OpenRouter definition and consumer additions, validates and defensively copies definitions through the shared JSON-value package, and consumes the LLM-owned OpenAI-compatible reserved request-field rule. | [Backend registry](../../internal/backend/registry.go) |
|
||||
| Root `promptkit` package | Provides the supported engine facade, ConfigKit source helpers, profile and backend mapping conveniences, injection options, public requests and results, explicit inspection, observation values, opaque prepared-execution handles, value conversion, redacted formatting, and typed capacity and generation error mapping. | [Package GoDoc](../../doc.go), [observations](../../observations.go), [prepared execution](../../prepared_execution.go), [backend API](../../backends.go), [engine facade](../../engine.go) |
|
||||
| `examples/go-library/prepare` | Demonstrates offline preparation and explicit configuration inspection with a prompt file, programmatic profile, and inline input. It is not a public library package. | [Example program](../../examples/go-library/prepare/main.go) |
|
||||
| `examples/go-library/run` | Demonstrates an offline downstream consumer using a prompt file, programmatic profile, inline input, an injected deterministic model client, and `Run`. It is not a public library package. | [Example program](../../examples/go-library/run/main.go) |
|
||||
| `examples/go-library/observe` | Demonstrates offline built-in HTTP retry and synchronous metadata observation across structured proposal and assessment runs using embedded sources, two profiles on one finite backend, and an application decision. It is not a public library package. | [Example program](../../examples/go-library/observe/main.go) |
|
||||
| `internal/backend` | Holds a defensive, immutable lookup and capacity-policy index for final backend definitions validated by the configuration adapter. | [Backend registry](../../internal/backend/registry.go) |
|
||||
| `internal/catalog` | Selects released keyed catalog assets and PromptKit-owned profile definitions as ConfigKit underlays; effective validation belongs to the configuration loader. | [Catalog adapter](../../internal/catalog/catalog.go), [internal sources](sources.md#profiles-and-built-ins) |
|
||||
| `internal/configuration` | Loads keyed backend, profile, and prompt configuration through ConfigKit; records effective field presence, validates final values, captures indexed prompt definitions and referenced message content, constructs backend and presence-aware profile snapshots, and retains an explicit read-only inspection view. | [Typed loader](../../internal/configuration/load.go), [internal sources](sources.md#typed-configuration-loading) |
|
||||
| `internal/capacity` | Owns engine-local bounded execution admission and FIFO model-generation permits for limited backend IDs, including cancellation-safe waiter removal and client wrapping. | [Internal capacity management](capacity.md) |
|
||||
| `internal/domain` | Defines internal framework values for requests, artifacts, prompt definitions, profiles, execution targets, rendering, generation, and validation, and owns source-neutral invariants for shared execution settings, OpenAI-compatible base endpoints, session identifiers, and output contracts. Source parsing, required fields, other source-specific normalization, defaulting, and boundary-specific error classification remain with their callers. | [Domain declarations](../../internal/domain/domain.go), [endpoint invariant](../../internal/domain/endpoint.go) |
|
||||
| `internal/engine` | Assembles selected catalog and caller configuration into one snapshot, then constructs capacity, rendering, validation, generation, repair, and the use-case runner for each engine. | [Engine assembly](../../internal/engine/engine.go), [internal sources](sources.md#profiles-and-built-ins), [internal runner](runner.md) |
|
||||
| `internal/generation` | Coordinates frozen built-in HTTP attempts and one-shot injected calls through active backend permits, including bounded status retries, cancellable backoff, and attempt observations. | [Internal generation coordination](generation.md) |
|
||||
| `internal/defaults` | Defines application-neutral framework constants and constructs the default execution target. It contains no CLI, server, or inbound HTTP limits. | [Framework defaults](../../internal/defaults/defaults.go) |
|
||||
| `internal/filecatalog` | Provides deterministic YAML discovery and path helpers for operating-system filesystems and `fs.FS` sources. | [File catalog](../../internal/filecatalog/catalog.go) |
|
||||
| `internal/jsonvalue` | Validates and deeply copies bounded JSON-compatible extra-parameter and prepared-schema trees while preserving supported concrete value types and rejecting cycles or excessive depth and work. | [JSON values](../../internal/jsonvalue/jsonvalue.go) |
|
||||
| `internal/promptdef` | Loads strictly decoded, validated prompt definitions from filesystem and `fs.FS` sources, including version selection and contained file-backed message content. | [Framework formats](../formats.md), [prompt-definition repository](../../internal/promptdef/filesystem_repository.go) |
|
||||
| `internal/profile` | Loads strictly decoded, validated execution profiles, including backend selection, from filesystem and `fs.FS` sources and composes repositories with error-preserving fallback. | [Framework formats](../formats.md), [profile repositories](../../internal/profile/filesystem_repository.go) |
|
||||
| `internal/profile/builtin` | Embeds the built-in profile catalog, whose entries select OpenRouter. | [Built-in catalog](../formats.md#built-in-profile-catalog), [repository](../../internal/profile/builtin/repository.go) |
|
||||
| `internal/promptdef` | Provides a narrow captured-prompt lookup contract, not-found identity, and execution-ready definition hash. | [Prompt lookup](../../internal/promptdef/repository.go), [framework formats](../formats.md) |
|
||||
| `internal/profile` | Provides a narrow captured-profile lookup contract and not-found identity. | [Profile lookup](../../internal/profile/repository.go), [framework formats](../formats.md) |
|
||||
| `internal/prompt` | Renders prompt messages from Go templates with artifact, variable, session, and cache-control data. | [Go-template renderer](../../internal/prompt/go_renderer.go) |
|
||||
| `internal/artifact` | Resolves ordinary inline and unrestricted caller-selected file references into copied artifacts with metadata and hashes. | [Internal sources and validation](sources.md) |
|
||||
| `internal/validate` | Validates basic, JSON, and JSON Schema output using operating-system filesystem or `fs.FS` schema sources and creates operation-local validation plans with canonical contained schema resources. | [Framework formats](../formats.md#schemas), [internal sources and validation](sources.md) |
|
||||
| `internal/llm` | Defines the internal generation boundary and implements outbound OpenAI-compatible chat requests from resolved execution targets, including response decoding, authentication, deadline handling, and ownership of the OpenAI-compatible reserved request-field policy. | [Internal model client](llm.md) |
|
||||
| `internal/usecase` | Resolves prompt definitions and hashes, profiles, backends, and targets for exact inspection and request settings for preparation, and coordinates ordinary execution and one-attempt prepared execution across internal sources, rendering, artifact loading, operation-local validation plans, generation, capacity, and optional repair. | [Internal runner](runner.md), [prepared-execution implementation](../../internal/usecase/prepared_execution.go) |
|
||||
| `internal/llm` | Defines the internal generation boundary and implements outbound OpenAI-compatible chat requests from resolved execution targets, including bounded structured non-success response decoding, successful-response decoding, authentication, deadline handling, and ownership of the OpenAI-compatible reserved request-field policy. | [Internal model client](llm.md) |
|
||||
| `internal/observation` | Owns invocation-local run IDs, presence-aware usage and phase accounting, and synchronous metadata event delivery for ordinary and claimed prepared runs and generation attempts. | [Internal execution observation](observation.md) |
|
||||
| `internal/usecase` | Selects captured prompts and profiles, applies request overrides and completion checks, and coordinates ordinary and single-claim prepared execution across rendering, artifact loading, operation-local validation plans, generation, capacity, and bounded repair. | [Internal runner](runner.md), [prepared-execution implementation](../../internal/usecase/prepared_execution.go) |
|
||||
|
||||
The root package assembles these internal components without exposing their
|
||||
The internal engine assembles these internal components without exposing their
|
||||
representations. Consumers depend only on the root facade.
|
||||
|
||||
## Maintenance
|
||||
|
||||
@@ -4,27 +4,28 @@
|
||||
|
||||
This document describes Promptkit's implemented internal orchestration. The
|
||||
[architecture policy](../policy/architecture.md) owns dependency and consumer
|
||||
boundaries. The [source and validation document](sources.md) owns repository,
|
||||
boundaries. The [source and validation document](sources.md) owns configuration,
|
||||
artifact, rendering, and validation behavior, while the
|
||||
[model-client document](llm.md) owns generation behavior and failure
|
||||
categories.
|
||||
|
||||
The runner remains under `internal/usecase` and is assembled by the root
|
||||
Promptkit engine. Its concrete type is not part of the public API.
|
||||
The runner remains under `internal/usecase` and is assembled by
|
||||
`internal/engine`. Its concrete type is not part of the public API.
|
||||
The [framework format reference](../formats.md) owns prompt, profile, schema,
|
||||
and override semantics consumed by the runner.
|
||||
|
||||
## Collaborators
|
||||
|
||||
`Runner` coordinates narrow internal interfaces for prompt definitions,
|
||||
profiles, backend resolution, artifacts, rendering, model generation, and
|
||||
validation. The root engine supplies one immutable registry containing the
|
||||
built-in backend and validated consumer additions, one engine-local run
|
||||
admitter, and a model client wrapped by the same capacity manager. Validation
|
||||
`Runner` coordinates narrow internal interfaces for captured prompt definitions
|
||||
and resolved profiles, artifacts, rendering, model generation, and validation.
|
||||
The internal engine supplies one engine-local run admitter and a generation
|
||||
coordinator using the same capacity manager. Validation
|
||||
plans and provider-facing schema metadata come from the validator's preparation
|
||||
interface.
|
||||
An output repairer can be injected internally, but the ordinary runner
|
||||
constructor does not enable one.
|
||||
The internal engine supplies one default output repairer through the explicit
|
||||
runner constructor, using the same coordinator as initial
|
||||
generation. The no-repair runner constructor remains available for focused
|
||||
internal callers and tests.
|
||||
|
||||
Each invocation carries its state in request, prepared-run, and result values.
|
||||
The runner has no durable run or session store.
|
||||
@@ -33,11 +34,11 @@ The runner has no durable run or session store.
|
||||
|
||||
The runner uses one prompt-selection and hashing boundary for ordinary
|
||||
preparation and exact prompt inspection. Preparation retains its early
|
||||
request-ID check before direct-session normalization; both operations then use
|
||||
the configured prompt repository to select one definition, load referenced
|
||||
message content, and calculate the same prompt hash.
|
||||
request-ID check before direct-session normalization; both operations select
|
||||
one captured definition and its construction-time hash. Referenced message
|
||||
content and static template syntax were captured when the engine was built.
|
||||
|
||||
Inspection stops after that structural lookup. It does not parse templates or
|
||||
Inspection stops after that structural lookup. It does not render templates or
|
||||
touch profile, artifact, schema, renderer, validator, admission, or model
|
||||
collaborators. The root [`Engine.InspectPrompt`](../../engine.go) GoDoc owns
|
||||
the public operation's exact contract.
|
||||
@@ -47,12 +48,11 @@ the public operation's exact contract.
|
||||
The runner uses one profile-selection and target-resolution boundary for
|
||||
ordinary preparation and exact profile inspection. Preparation first selects a
|
||||
request profile or a prompt default; inspection begins with its required
|
||||
explicit profile ID. Both then apply the ordinary source precedence, resolve a
|
||||
named backend, and construct the effective target from framework, backend, and
|
||||
profile values.
|
||||
|
||||
Inspection stops after the resulting endpoint and model are structurally
|
||||
validated. It does not check credential availability or perform prompt,
|
||||
explicit profile ID. Both use the captured, presence-aware target constructed
|
||||
from the final effective profile and backend configuration. Request overrides
|
||||
apply afterward; preparation checks endpoint and model completeness only after
|
||||
those overrides. Inspection can report a partial profile without requiring a
|
||||
runnable target. Neither path checks credential availability or performs prompt,
|
||||
artifact, schema, rendering, admission, or model-client work. The root
|
||||
[`Engine.InspectProfile`](../../engine.go) GoDoc owns the public operation's
|
||||
exact contract.
|
||||
@@ -64,11 +64,10 @@ where a run can be assigned to its selected backend pool. The resolution phase
|
||||
performs only the work needed to validate routing and admission:
|
||||
|
||||
1. validate the required prompt selection and normalize any direct session ID;
|
||||
2. load the prompt definition and hash the original definition;
|
||||
2. select the captured prompt definition and hash;
|
||||
3. select the request profile or the prompt's default profile;
|
||||
4. resolve the profile's backend ID, when present;
|
||||
5. resolve application-neutral defaults, backend defaults, profile values,
|
||||
and explicit request overrides in that order;
|
||||
4. retain the profile's selected backend identity, when present;
|
||||
5. apply explicit request overrides to its captured backend-completed target;
|
||||
6. validate endpoint, model, numeric overrides, and credential requirements;
|
||||
7. resolve and validate the effective output contract without loading its
|
||||
schema; and
|
||||
@@ -82,7 +81,8 @@ profile, or backend:
|
||||
schema metadata from it when required;
|
||||
2. load and hash input artifacts;
|
||||
3. render messages and the prompt-defined session;
|
||||
4. apply any direct session ID;
|
||||
4. apply any direct session ID, then append already-normalized request messages
|
||||
after the rendered definition messages;
|
||||
5. hash the effective rendered prompt; and
|
||||
6. construct the prepared value and preparation timing.
|
||||
|
||||
@@ -99,23 +99,26 @@ Pointer-based numeric overrides preserve an explicit zero. Invalid negative or
|
||||
out-of-range values fail as invalid requests. Endpoint overrides do not change
|
||||
the selected backend identity. Non-empty extra-parameter maps replace whole
|
||||
lower-precedence maps. A direct API key takes precedence over environment
|
||||
lookup; otherwise request, profile, and backend environment-variable names
|
||||
apply in that order. A profile requiring a direct key clears an inherited
|
||||
lookup; otherwise request, explicit profile, and backend environment-variable
|
||||
names apply in that order. A profile requiring a direct key suppresses an inherited
|
||||
backend environment name unless the request supplies its own name. Secret
|
||||
values remain excluded from serialized metadata.
|
||||
|
||||
Reasoning overrides are tri-state: nil inherits the profile, a pointer to a
|
||||
nonblank string trims and replaces it, and a pointer to a blank string clears
|
||||
it. A nonblank direct session is normalized before source loading, bypasses
|
||||
it. A nonblank direct session is normalized before prompt selection, bypasses
|
||||
the prompt session template, and is applied after ordinary message rendering.
|
||||
A blank direct value retains prompt-template behavior. The runner clears the
|
||||
template only on a value copy of the definition, so the definition hash always
|
||||
describes the original source while the rendered-prompt hash includes the
|
||||
effective direct or rendered session.
|
||||
describes captured effective content while the rendered-prompt hash includes the
|
||||
effective direct or rendered session and the complete effective message
|
||||
sequence. The rendered-prompt hash uses a versioned, length-framed SHA-256
|
||||
encoding of that session, every role and content value, and cache-control
|
||||
presence and values; its hexadecimal value is opaque.
|
||||
|
||||
The registry is read-only after engine construction. Concurrent `Prepare` and
|
||||
`Run` calls resolve independent defensive backend values and keep all
|
||||
invocation state local.
|
||||
The configuration snapshot is read-only after engine construction. Concurrent
|
||||
`Prepare` and `Run` calls copy their resolved targets and keep all invocation
|
||||
state local.
|
||||
|
||||
## Run Flow
|
||||
|
||||
@@ -125,8 +128,17 @@ admitter is an internal unlimited fallback. After successful admission, `Run`
|
||||
immediately defers the returned release function, performs the completion
|
||||
phase, makes one initial generation call, builds the named output artifact,
|
||||
and validates that artifact with the plan compiled during completion. Invalid
|
||||
generated content remains a validation result; an inability to perform
|
||||
validation is an operational error.
|
||||
generated content remains a validation result; an inability to generate or
|
||||
perform validation is an operational error.
|
||||
|
||||
When the public facade installs an invocation-local observation recorder,
|
||||
ordinary `Run` reuses its run ID and carries it to the generation coordinator
|
||||
through context. Prompt and profile identities are enriched only after their
|
||||
respective resolutions succeed, before admission. The facade starts observation
|
||||
before request conversion and emits its terminal event after public error
|
||||
mapping and after the runner's deferred admission release. Preparation and
|
||||
prepared execution follows the same terminal mapping after a successful claim;
|
||||
invalid or discarded handles do not acquire an observer.
|
||||
|
||||
Validation preparation and execution honor cancellation at every
|
||||
Promptkit-controlled boundary and do not publish a partial plan or result.
|
||||
@@ -139,20 +151,30 @@ method must return before cancellation can take precedence over its outcome.
|
||||
The admission lease covers completion-phase preparation, initial generation,
|
||||
validation, every repair, and every exit. It bounds accepted work without
|
||||
serializing preparation or validation behind the active-generation limit.
|
||||
The wrapped model client separately acquires a FIFO active permit only around
|
||||
each actual generation call.
|
||||
The coordinator separately acquires a FIFO active permit only around each
|
||||
actual model attempt. It releases the permit before a transient retry wait.
|
||||
|
||||
When an internal repairer is present, a JSON or JSON Schema content failure can
|
||||
trigger bounded repair attempts. Repair receives the effective execution
|
||||
target, explicit numeric-presence bits, credential, backend identity, session
|
||||
ID, validation errors, prior output, and structured-output specification. One
|
||||
request constructor supplies those common fields to initial and repair
|
||||
generation while their rendered prompts remain intentionally distinct. The
|
||||
default repairer uses the same wrapped client as initial generation, so each
|
||||
repair reacquires the selected backend's active permit while remaining inside
|
||||
its original admission lease. Repair never performs a second bounded
|
||||
admission, and repaired outputs use the operation's existing validation plan.
|
||||
This capability remains internal and is not a public option.
|
||||
After a failed `basic`, JSON, or JSON Schema validation with a positive frozen
|
||||
budget, the installed repairer can make a bounded corrective call. Each request
|
||||
starts with a fresh copy of the complete effective message sequence (the
|
||||
configured prefix followed by the request suffix), includes only the latest
|
||||
nonempty candidate as an assistant message, and appends one corrective user
|
||||
message. Empty candidates omit that assistant message. The
|
||||
correction carries validation diagnostics as JSON data bounded to 64 KiB; the
|
||||
full diagnostics remain in the validation result.
|
||||
|
||||
Repair receives the effective execution target, explicit numeric-presence bits,
|
||||
credential, backend identity, session ID, structured-output specification, and
|
||||
frozen transient retry budget.
|
||||
The same request constructor supplies those common fields to initial and repair
|
||||
generation. The default repairer uses the same coordinator as initial
|
||||
generation, so each repair reacquires the selected backend's active permit
|
||||
while remaining inside its original admission lease. Repair never performs a
|
||||
second bounded admission, and repaired outputs use the operation's existing
|
||||
validation plan. The runner stops at the first valid candidate, sums completed
|
||||
generation usage, reports calls actually started, and returns the final failed
|
||||
validation result on exhaustion. A repair generation failure follows the
|
||||
ordinary generation-error category rather than becoming a validation error.
|
||||
|
||||
A successful result includes the output artifact and raw output, validation
|
||||
state, effective session ID, prompt and rendered-prompt hashes, selected
|
||||
@@ -164,7 +186,8 @@ credential during execution, reaches generation and every repair attempt.
|
||||
Result usage is the field-wise sum of all five usage values from the initial
|
||||
response and every completed repair response. Final raw output, artifact, and
|
||||
validation state still come from the last candidate. A repair error returns no
|
||||
partial run result or partial usage.
|
||||
partial run result or partial result usage, although the invocation-local
|
||||
observer retains completed-attempt usage and reports it in the terminal event.
|
||||
|
||||
## Failure Categories
|
||||
|
||||
@@ -189,7 +212,7 @@ generation, validation, repair, and cancellation failures.
|
||||
Other context cancellation propagates through the invoked collaborator and is
|
||||
classified by the owning operation. In particular, cancellation observed by
|
||||
validation retains the context identity through the validation error category.
|
||||
An overlong direct session is an invalid request before source loading, while
|
||||
An overlong direct session is an invalid request before prompt selection, while
|
||||
an invalid or overlong prompt session template remains a prompt-render failure.
|
||||
An unknown selected backend, or a selected backend with no configured resolver,
|
||||
is classified as a profile-load failure.
|
||||
@@ -203,8 +226,9 @@ behavior, hashing, generation and validation outcomes, backend propagation,
|
||||
bounded repair progression, initial/repair request parity, cumulative usage,
|
||||
shared initial/repair capacity, credentials and redaction, error categories,
|
||||
artifact metadata, and timing. The
|
||||
[capacity subsystem document](capacity.md) identifies the focused pool,
|
||||
waiter, and wrapped-client tests.
|
||||
[capacity subsystem document](capacity.md) identifies the focused pool and
|
||||
waiter tests. The [generation document](generation.md) owns attempt and retry
|
||||
coordination tests.
|
||||
|
||||
Changes to orchestration should continue to use the existing package
|
||||
interfaces, keep request state local to an invocation, and preserve the shared
|
||||
|
||||
@@ -2,80 +2,73 @@
|
||||
|
||||
## Purpose
|
||||
|
||||
This document describes Promptkit's implemented internal source, artifact,
|
||||
This document describes Promptkit's implemented configuration, artifact,
|
||||
rendering, and output-validation behavior. The
|
||||
[architecture policy](../policy/architecture.md) owns the library boundary and
|
||||
dependency rules. None of these internal packages is a supported consumer API,
|
||||
and the root engine assembles them behind its public source options and values.
|
||||
The [framework format reference](../formats.md) owns the exact file fields,
|
||||
validation modes, built-in catalog, and source precedence.
|
||||
[architecture policy](../policy/architecture.md) owns dependency rules, while
|
||||
the [framework format reference](../formats.md) owns fields, defaults, and
|
||||
source semantics. Internal packages are not consumer APIs.
|
||||
|
||||
## Typed Configuration Loading
|
||||
|
||||
`internal/configuration` is the single adapter from ConfigKit's final
|
||||
effective configuration to execution-owned definitions. It performs one
|
||||
layered load, captures effective presence (including explicit null), records
|
||||
normalization and defaults through resolution, and validates all effective
|
||||
backend, profile, and prompt values. Replaced lower-layer domain values are
|
||||
not separately validated. It builds an immutable backend index and
|
||||
presence-aware profile targets, applying final backend defaults only after
|
||||
profile inheritance. It retains ConfigKit's result privately for explicit
|
||||
inspection, not for per-run resolution. The runtime/domain packages do not
|
||||
depend on ConfigKit.
|
||||
|
||||
The root `DirectoryInputs` and `FilesystemInputs` helpers discover sorted YAML
|
||||
files under an explicit root, provide resource context, and leave imports
|
||||
disabled. Each returned slice belongs to one strict-composition group; file
|
||||
order does not confer priority. The engine prepends selected catalog and
|
||||
Promptkit-owned definitions, then caller underlays and optional main. The
|
||||
[configuration tests](../../internal/configuration/load_test.go) exercise the
|
||||
real loader and final-value handoff.
|
||||
|
||||
## Prompt Definitions
|
||||
|
||||
`internal/promptdef` uses one source-neutral flow for prompt selection and
|
||||
normalization. That flow scans normalized YAML ID and version metadata,
|
||||
requires one strictly decoded document per file, classifies errors for the
|
||||
selected definition, detects duplicates, and normalizes the exact match.
|
||||
Small operating-system and `fs.FS` adapters own discovery, byte reads, display
|
||||
paths, content opening, and root containment. Each lookup remains a
|
||||
point-in-time scan: definitions and catalogs are not cached, and file-backed
|
||||
message content is opened only for the exact selected candidate.
|
||||
The configuration adapter indexes each prompt by exact ID and version,
|
||||
materializes file-backed message content using the exact effective field's
|
||||
origin and resource context, and checks static message/session template syntax
|
||||
at construction. It stores captured content, compiled templates, declared
|
||||
input metadata, and a definition hash. It closes content-file handles before
|
||||
returning and retains no source capability for public inspection.
|
||||
|
||||
Operating-system sources enforce containment against canonical roots and
|
||||
targets so symlinks cannot escape. Injected `fs.FS` sources enforce containment
|
||||
in their clean relative path namespace. A single-file source uses the selected
|
||||
prompt file's containing directory as its root. Every content path must be
|
||||
relative and is opened from its exact parsed text after a separate blank check;
|
||||
contained parent components and whitespace-bearing names remain valid.
|
||||
Role and cache-control normalization is applied to execution-owned message
|
||||
copies. The retained ConfigKit configuration keeps effective input message
|
||||
metadata and its resource origins: recording a sequence-wide normalization in
|
||||
ConfigKit v0.5.0 would replace origins of unchanged content-file fields.
|
||||
|
||||
Exact prompt inspection performs one point-in-time lookup through that same
|
||||
repository and validates referenced message content before returning declared
|
||||
metadata. It does not parse templates or read profile, input, or schema
|
||||
sources, and it does not retain the definition for a later execution.
|
||||
|
||||
Its package tests own prompt selection, strict decoding, definition validation,
|
||||
duplicate detection, and source containment:
|
||||
[prompt-definition repository tests](../../internal/promptdef/repository_test.go).
|
||||
The runner selects from this index for preparation and `InspectPrompt`.
|
||||
Neither path rescans configuration or reopens message files. `InspectPrompt`
|
||||
returns declared metadata without rendering or resolving profile, input, or
|
||||
schema resources. Runtime rendering still checks request variables and input
|
||||
helpers. The [prompt adapter tests](../../internal/configuration/prompts_test.go)
|
||||
cover resource origins, context containment, capture, and selection.
|
||||
|
||||
## Profiles And Built-Ins
|
||||
|
||||
`internal/profile` loads and validates execution profiles from an
|
||||
operating-system filesystem or an `fs.FS`. A file contains exactly one YAML
|
||||
document and its trimmed YAML `id` is its only selection identity; filenames do
|
||||
not confer authority. Each point lookup reads discovered files once for their
|
||||
metadata and reuses the selected file's bytes for strict decoding; unrelated
|
||||
profiles are not fully decoded. Strict selected decoding recognizes the
|
||||
optional `backend` field, trims its value, and requires a model plus at least
|
||||
one non-blank backend or endpoint. File-backed `extra_params` values are
|
||||
validated and defensively copied through the shared bounded JSON-value owner
|
||||
before a profile is published. OpenAI-compatible reserved-field policy remains
|
||||
with the model-client and backend-registry owners.
|
||||
`internal/catalog` selects the released keyed OpenRouter and Rakestrawhome
|
||||
assets as one ConfigKit underlay and PromptKit-owned profiles as the next.
|
||||
It discovers YAML inputs and supplies passive filesystem context without
|
||||
parsing or pre-validating catalog definitions. The configuration loader
|
||||
validates only the final effective backend and profile values, including
|
||||
consumer overlays of maintained bases or backends. There is no catalog-only
|
||||
preflight, reserved source namespace, or replay of backend registration.
|
||||
|
||||
The overlay repository consults the next repository only when the
|
||||
higher-precedence repository reports that a profile is absent. A reliably
|
||||
selected malformed profile stops fallback, while an unrelated malformed file
|
||||
does not become authoritative through its filename. Loading does not check
|
||||
backend registry membership because the available registry belongs to the
|
||||
assembled engine; the runner checks membership during preparation and exact
|
||||
profile inspection.
|
||||
|
||||
The root engine assembles profile repositories in precedence order: in-memory
|
||||
profiles, one ordinary configured source, an application fallback source, then
|
||||
the embedded built-in catalog. An explicit file or `fs.FS` profile source
|
||||
replaces `Config.ProfileDir` within the ordinary configured-source category.
|
||||
|
||||
Exact profile inspection performs one point-in-time lookup through those
|
||||
profile sources and checks the resolved target without reading prompt, input,
|
||||
or schema sources. It does not retain that lookup for a later execution.
|
||||
|
||||
`internal/profile/builtin` embeds the maintained built-in profile catalog.
|
||||
Every embedded profile selects `openrouter` and inherits its endpoint and
|
||||
credential environment-variable name from the built-in backend registry rather
|
||||
than repeating those values. Profile loading and overlay behavior are owned by
|
||||
the [profile repository tests](../../internal/profile/repository_test.go),
|
||||
while catalog completeness, the backend-selection invariant, and duplicate IDs
|
||||
are owned by the
|
||||
[built-in repository tests](../../internal/profile/builtin/repository_test.go).
|
||||
ConfigKit resolves profile inheritance before the handoff. The configuration
|
||||
adapter validates effective references and field values, then builds target
|
||||
snapshots with backend defaults and presence/suppression state. A profile can
|
||||
remain partial until runtime overrides complete it. `InspectProfile` selects
|
||||
that captured target without prompt, artifact, schema, credential lookup, or
|
||||
provider work. Preparation applies runtime overrides and then checks target
|
||||
completeness. The [target tests](../../internal/configuration/targets_test.go)
|
||||
cover effective target interpretation, while the [catalog tests](../../internal/catalog/catalog_test.go)
|
||||
cover selected released assets.
|
||||
|
||||
## Ordinary Artifacts
|
||||
|
||||
@@ -100,18 +93,20 @@ implemented reader behavior and failures.
|
||||
|
||||
## Rendering
|
||||
|
||||
`internal/prompt` renders definition messages as Go templates using named
|
||||
artifacts and variables. Within one render, each referenced artifact body is
|
||||
converted to text lazily and cached by input name for reuse across the session
|
||||
and every message; the cache is not shared across renders. Conversion uses
|
||||
bounded chunks and preserves the artifact bytes exactly.
|
||||
`internal/prompt` renders captured, pre-parsed definition templates using named
|
||||
artifacts and variables. It clones templates to bind request-specific input
|
||||
helpers without mutating shared state. Within one render, each referenced
|
||||
artifact body is converted to text lazily and cached by input name for reuse
|
||||
across the session and every message; the cache is not shared across renders.
|
||||
Conversion uses bounded chunks and preserves the artifact bytes exactly.
|
||||
|
||||
Session and message parsing and execution remain synchronous. The renderer
|
||||
checks cancellation before and after each parse and execution boundary,
|
||||
Template parsing occurs during engine construction. Runtime execution remains
|
||||
synchronous. The renderer checks cancellation around execution boundaries,
|
||||
between artifact conversion chunks, around each message, and before publishing
|
||||
the complete prompt. It cannot interrupt template work already in progress and
|
||||
never publishes a partial prompt after observing cancellation. It carries
|
||||
message roles, session IDs, and cache control into the rendered prompt. The
|
||||
never publishes a partial prompt after observing cancellation. It validates and
|
||||
canonicalizes message roles before carrying roles, session IDs, and cache
|
||||
control into the rendered prompt. The
|
||||
[renderer tests](../../internal/prompt/renderer_test.go) own rendering behavior.
|
||||
|
||||
## Schemas And Output Validation
|
||||
@@ -143,9 +138,10 @@ not detach dependency work into background goroutines.
|
||||
`Prepare` discards its validation plan after returning metadata. `Run` retains
|
||||
its plan for initial and repaired-output validation, then discards it with the
|
||||
operation. `PrepareExecution` retains the plan in its private frozen payload;
|
||||
`RunPrepared` uses that plan without reopening prompt, profile, input, or
|
||||
schema sources or rerendering the request. A later ordinary `Run` always
|
||||
performs fresh source resolution and preparation.
|
||||
`RunPrepared` uses that plan without reopening configuration, message, input,
|
||||
or schema resources or rerendering the request. A later ordinary `Run` uses
|
||||
the same captured configuration but prepares request artifacts and schemas
|
||||
afresh.
|
||||
|
||||
The [validator tests](../../internal/validate/standard_validator_test.go) own
|
||||
basic, JSON, JSON Schema, source resolution, schema loading, compilation,
|
||||
|
||||
126
docs/oss/attribution.md
Normal file
126
docs/oss/attribution.md
Normal file
@@ -0,0 +1,126 @@
|
||||
# Open Source Attribution Notices
|
||||
|
||||
## Standard Repository Policies
|
||||
|
||||
Material: policy documents under `docs/policy/` and the Policy Reading Guide in
|
||||
`docs/development.md`, adapted for this repository; root `AGENTS.md`, copied
|
||||
without modification from upstream `agents/AGENTS.md`.
|
||||
|
||||
Copyright (C) 2026 Eric Rakestraw
|
||||
|
||||
SPDX-License-Identifier: CC-BY-SA-4.0
|
||||
|
||||
License and notices: [CC BY-SA 4.0](licenses/CC-BY-SA-4.0.txt) |
|
||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/COPYRIGHT) |
|
||||
[LICENSE](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/LICENSE).
|
||||
|
||||
[Source](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843).
|
||||
|
||||
## ConfigKit
|
||||
|
||||
Selected module: `gitea.maximumdirect.net/eric/ConfigKit` at `v0.5.0`.
|
||||
|
||||
Copyright (C) 2026 Eric Rakestraw
|
||||
|
||||
SPDX-License-Identifier: LGPL-3.0-or-later
|
||||
|
||||
License and notices: [LGPL 3.0](../../LICENSE) |
|
||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/ConfigKit/src/commit/dc09959cca4e105f49ec316c6d4ba585bd2340bd/COPYRIGHT) |
|
||||
[LICENSE](https://gitea.maximumdirect.net/eric/ConfigKit/src/commit/dc09959cca4e105f49ec316c6d4ba585bd2340bd/LICENSE).
|
||||
|
||||
[Source](https://gitea.maximumdirect.net/eric/ConfigKit/src/commit/dc09959cca4e105f49ec316c6d4ba585bd2340bd).
|
||||
|
||||
## Promptkit OpenRouter Catalog v3
|
||||
|
||||
Selected module: `gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v3`
|
||||
at `v3.20260930.0`.
|
||||
|
||||
Copyright (C) 2026 Eric Rakestraw
|
||||
|
||||
SPDX-License-Identifier: MIT
|
||||
|
||||
License and notices: [MIT](licenses/MIT.txt) |
|
||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit-backend-openrouter/src/commit/f17d208cb7ad1fb043cda2266cd3011d11b40aa0/COPYRIGHT) |
|
||||
[LICENSE](https://gitea.maximumdirect.net/eric/promptkit-backend-openrouter/src/commit/f17d208cb7ad1fb043cda2266cd3011d11b40aa0/LICENSE).
|
||||
|
||||
[Source](https://gitea.maximumdirect.net/eric/promptkit-backend-openrouter/src/commit/f17d208cb7ad1fb043cda2266cd3011d11b40aa0).
|
||||
|
||||
## Promptkit Rakestrawhome Catalog v3
|
||||
|
||||
Selected module: `gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v3`
|
||||
at `v3.20260930.0`.
|
||||
|
||||
Copyright (C) 2026 Eric Rakestraw
|
||||
|
||||
SPDX-License-Identifier: MIT
|
||||
|
||||
License and notices: [MIT](licenses/MIT.txt) |
|
||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/src/commit/993443d88c451901f4c917fa00c204fd3b54d28d/COPYRIGHT) |
|
||||
[LICENSE](https://gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/src/commit/993443d88c451901f4c917fa00c204fd3b54d28d/LICENSE).
|
||||
|
||||
[Source](https://gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/src/commit/993443d88c451901f4c917fa00c204fd3b54d28d).
|
||||
|
||||
## jsonschema
|
||||
|
||||
Selected module: `github.com/santhosh-tekuri/jsonschema/v6` at `v6.0.2`.
|
||||
|
||||
SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
License and notices: [Apache 2.0](licenses/Apache-2.0.txt) |
|
||||
[LICENSE](https://github.com/santhosh-tekuri/jsonschema/blob/29cbed948d24a04700eb94436416b18a07953b71/LICENSE).
|
||||
|
||||
[Source](https://github.com/santhosh-tekuri/jsonschema/tree/29cbed948d24a04700eb94436416b18a07953b71).
|
||||
|
||||
## Go Text
|
||||
|
||||
Selected module: `golang.org/x/text` at `v0.14.0` (transitive dependency).
|
||||
|
||||
Copyright (c) 2009 The Go Authors. All rights reserved.
|
||||
|
||||
SPDX-License-Identifier: BSD-3-Clause
|
||||
|
||||
License and notices: [BSD 3-Clause](licenses/BSD-3-Clause.txt) |
|
||||
[LICENSE](https://github.com/golang/text/blob/6c97a165dd661335ff7bce6104a008558123c353/LICENSE) |
|
||||
[PATENTS](https://github.com/golang/text/blob/6c97a165dd661335ff7bce6104a008558123c353/PATENTS).
|
||||
|
||||
[Source](https://github.com/golang/text/tree/6c97a165dd661335ff7bce6104a008558123c353).
|
||||
|
||||
## Go YAML v3
|
||||
|
||||
Selected module: `gopkg.in/yaml.v3` at `v3.0.1` (transitive dependency).
|
||||
|
||||
Copyright (c) 2006-2010 Kirill Simonov
|
||||
|
||||
Copyright (c) 2006-2011 Kirill Simonov
|
||||
|
||||
Copyright (c) 2011-2019 Canonical Ltd
|
||||
|
||||
Copyright 2011-2016 Canonical Ltd.
|
||||
|
||||
SPDX-License-Identifier: MIT AND Apache-2.0
|
||||
|
||||
License and notices: [MIT](licenses/MIT.txt) |
|
||||
[Apache 2.0](licenses/Apache-2.0.txt) |
|
||||
[LICENSE](https://github.com/go-yaml/yaml/blob/f6f7691b1fdeb513f56608cd2c32c51f8194bf51/LICENSE) |
|
||||
[NOTICE](https://github.com/go-yaml/yaml/blob/f6f7691b1fdeb513f56608cd2c32c51f8194bf51/NOTICE).
|
||||
|
||||
[Source](https://github.com/go-yaml/yaml/tree/f6f7691b1fdeb513f56608cd2c32c51f8194bf51).
|
||||
|
||||
## Go YAML v4
|
||||
|
||||
Selected module: `go.yaml.in/yaml/v4` at `v4.0.0-rc.6` (transitive dependency).
|
||||
|
||||
Copyright 2006-2010 Kirill Simonov
|
||||
|
||||
Copyright 2011-2019 Canonical Ltd
|
||||
|
||||
Copyright 2025 The go-yaml Project Contributors
|
||||
|
||||
SPDX-License-Identifier: MIT AND Apache-2.0
|
||||
|
||||
License and notices: [MIT](licenses/MIT.txt) |
|
||||
[Apache 2.0](licenses/Apache-2.0.txt) |
|
||||
[LICENSE](https://github.com/yaml/go-yaml/blob/1c17b9cee72bf81ae73a8be50fbd7dd3a9985125/LICENSE) |
|
||||
[NOTICE](https://github.com/yaml/go-yaml/blob/1c17b9cee72bf81ae73a8be50fbd7dd3a9985125/NOTICE).
|
||||
|
||||
[Source](https://github.com/yaml/go-yaml/tree/1c17b9cee72bf81ae73a8be50fbd7dd3a9985125).
|
||||
202
docs/oss/licenses/Apache-2.0.txt
Normal file
202
docs/oss/licenses/Apache-2.0.txt
Normal file
@@ -0,0 +1,202 @@
|
||||
|
||||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
APPENDIX: How to apply the Apache License to your work.
|
||||
|
||||
To apply the Apache License to your work, attach the following
|
||||
boilerplate notice, with the fields enclosed by brackets "[]"
|
||||
replaced with your own identifying information. (Don't include
|
||||
the brackets!) The text should be enclosed in the appropriate
|
||||
comment syntax for the file format. We also recommend that a
|
||||
file or class name and description of purpose be included on the
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright [yyyy] [name of copyright owner]
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
11
docs/oss/licenses/BSD-3-Clause.txt
Normal file
11
docs/oss/licenses/BSD-3-Clause.txt
Normal file
@@ -0,0 +1,11 @@
|
||||
Copyright (c) <year> <owner>.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
|
||||
|
||||
1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
|
||||
|
||||
2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
|
||||
|
||||
3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
170
docs/oss/licenses/CC-BY-SA-4.0.txt
Normal file
170
docs/oss/licenses/CC-BY-SA-4.0.txt
Normal file
@@ -0,0 +1,170 @@
|
||||
Creative Commons Attribution-ShareAlike 4.0 International
|
||||
|
||||
Creative Commons Corporation (“Creative Commons”) is not a law firm and does not provide legal services or legal advice. Distribution of Creative Commons public licenses does not create a lawyer-client or other relationship. Creative Commons makes its licenses and related information available on an “as-is” basis. Creative Commons gives no warranties regarding its licenses, any material licensed under their terms and conditions, or any related information. Creative Commons disclaims all liability for damages resulting from their use to the fullest extent possible.
|
||||
|
||||
Using Creative Commons Public Licenses
|
||||
|
||||
Creative Commons public licenses provide a standard set of terms and conditions that creators and other rights holders may use to share original works of authorship and other material subject to copyright and certain other rights specified in the public license below. The following considerations are for informational purposes only, are not exhaustive, and do not form part of our licenses.
|
||||
|
||||
Considerations for licensors: Our public licenses are intended for use by those authorized to give the public permission to use material in ways otherwise restricted by copyright and certain other rights. Our licenses are irrevocable. Licensors should read and understand the terms and conditions of the license they choose before applying it. Licensors should also secure all rights necessary before applying our licenses so that the public can reuse the material as expected. Licensors should clearly mark any material not subject to the license. This includes other CC-licensed material, or material used under an exception or limitation to copyright. More considerations for licensors.
|
||||
|
||||
Considerations for the public: By using one of our public licenses, a licensor grants the public permission to use the licensed material under specified terms and conditions. If the licensor’s permission is not necessary for any reason–for example, because of any applicable exception or limitation to copyright–then that use is not regulated by the license. Our licenses grant only permissions under copyright and certain other rights that a licensor has authority to grant. Use of the licensed material may still be restricted for other reasons, including because others have copyright or other rights in the material. A licensor may make special requests, such as asking that all changes be marked or described.
|
||||
|
||||
Although not required by our licenses, you are encouraged to respect those requests where reasonable. More considerations for the public.
|
||||
|
||||
Creative Commons Attribution-ShareAlike 4.0 International Public License
|
||||
|
||||
By exercising the Licensed Rights (defined below), You accept and agree to be bound by the terms and conditions of this Creative Commons Attribution-ShareAlike 4.0 International Public License ("Public License"). To the extent this Public License may be interpreted as a contract, You are granted the Licensed Rights in consideration of Your acceptance of these terms and conditions, and the Licensor grants You such rights in consideration of benefits the Licensor receives from making the Licensed Material available under these terms and conditions.
|
||||
|
||||
Section 1 – Definitions.
|
||||
|
||||
a. Adapted Material means material subject to Copyright and Similar Rights that is derived from or based upon the Licensed Material and in which the Licensed Material is translated, altered, arranged, transformed, or otherwise modified in a manner requiring permission under the Copyright and Similar Rights held by the Licensor. For purposes of this Public License, where the Licensed Material is a musical work, performance, or sound recording, Adapted Material is always produced where the Licensed Material is synched in timed relation with a moving image.
|
||||
|
||||
b. Adapter's License means the license You apply to Your Copyright and Similar Rights in Your contributions to Adapted Material in accordance with the terms and conditions of this Public License.
|
||||
|
||||
c. BY-SA Compatible License means a license listed at creativecommons.org/compatiblelicenses, approved by Creative Commons as essentially the equivalent of this Public License.
|
||||
|
||||
d. Copyright and Similar Rights means copyright and/or similar rights closely related to copyright including, without limitation, performance, broadcast, sound recording, and Sui Generis Database Rights, without regard to how the rights are labeled or categorized. For purposes of this Public License, the rights specified in Section 2(b)(1)-(2) are not Copyright and Similar Rights.
|
||||
|
||||
e. Effective Technological Measures means those measures that, in the absence of proper authority, may not be circumvented under laws fulfilling obligations under Article 11 of the WIPO Copyright Treaty adopted on December 20, 1996, and/or similar international agreements.
|
||||
|
||||
f. Exceptions and Limitations means fair use, fair dealing, and/or any other exception or limitation to Copyright and Similar Rights that applies to Your use of the Licensed Material.
|
||||
|
||||
g. License Elements means the license attributes listed in the name of a Creative Commons Public License. The License Elements of this Public License are Attribution and ShareAlike.
|
||||
|
||||
h. Licensed Material means the artistic or literary work, database, or other material to which the Licensor applied this Public License.
|
||||
|
||||
i. Licensed Rights means the rights granted to You subject to the terms and conditions of this Public License, which are limited to all Copyright and Similar Rights that apply to Your use of the Licensed Material and that the Licensor has authority to license.
|
||||
|
||||
j. Licensor means the individual(s) or entity(ies) granting rights under this Public License.
|
||||
|
||||
k. Share means to provide material to the public by any means or process that requires permission under the Licensed Rights, such as reproduction, public display, public performance, distribution, dissemination, communication, or importation, and to make material available to the public including in ways that members of the public may access the material from a place and at a time individually chosen by them.
|
||||
|
||||
l. Sui Generis Database Rights means rights other than copyright resulting from Directive 96/9/EC of the European Parliament and of the Council of 11 March 1996 on the legal protection of databases, as amended and/or succeeded, as well as other essentially equivalent rights anywhere in the world.
|
||||
|
||||
m. You means the individual or entity exercising the Licensed Rights under this Public License. Your has a corresponding meaning.
|
||||
|
||||
Section 2 – Scope.
|
||||
|
||||
a. License grant.
|
||||
|
||||
1. Subject to the terms and conditions of this Public License, the Licensor hereby grants You a worldwide, royalty-free, non-sublicensable, non-exclusive, irrevocable license to exercise the Licensed Rights in the Licensed Material to:
|
||||
|
||||
A. reproduce and Share the Licensed Material, in whole or in part; and
|
||||
|
||||
B. produce, reproduce, and Share Adapted Material.
|
||||
|
||||
2. Exceptions and Limitations. For the avoidance of doubt, where Exceptions and Limitations apply to Your use, this Public License does not apply, and You do not need to comply with its terms and conditions.
|
||||
|
||||
3. Term. The term of this Public License is specified in Section 6(a).
|
||||
|
||||
4. Media and formats; technical modifications allowed. The Licensor authorizes You to exercise the Licensed Rights in all media and formats whether now known or hereafter created, and to make technical modifications necessary to do so. The Licensor waives and/or agrees not to assert any right or authority to forbid You from making technical modifications necessary to exercise the Licensed Rights, including technical modifications necessary to circumvent Effective Technological Measures. For purposes of this Public License, simply making modifications authorized by this Section 2(a)(4) never produces Adapted Material.
|
||||
|
||||
5. Downstream recipients.
|
||||
|
||||
A. Offer from the Licensor – Licensed Material. Every recipient of the Licensed Material automatically receives an offer from the Licensor to exercise the Licensed Rights under the terms and conditions of this Public License.
|
||||
|
||||
B. Additional offer from the Licensor – Adapted Material. Every recipient of Adapted Material from You automatically receives an offer from the Licensor to exercise the Licensed Rights in the Adapted Material under the conditions of the Adapter’s License You apply.
|
||||
|
||||
C. No downstream restrictions. You may not offer or impose any additional or different terms or conditions on, or apply any Effective Technological Measures to, the Licensed Material if doing so restricts exercise of the Licensed Rights by any recipient of the Licensed Material.
|
||||
|
||||
6. No endorsement. Nothing in this Public License constitutes or may be construed as permission to assert or imply that You are, or that Your use of the Licensed Material is, connected with, or sponsored, endorsed, or granted official status by, the Licensor or others designated to receive attribution as provided in Section 3(a)(1)(A)(i).
|
||||
|
||||
b. Other rights.
|
||||
|
||||
1. Moral rights, such as the right of integrity, are not licensed under this Public License, nor are publicity, privacy, and/or other similar personality rights; however, to the extent possible, the Licensor waives and/or agrees not to assert any such rights held by the Licensor to the limited extent necessary to allow You to exercise the Licensed Rights, but not otherwise.
|
||||
|
||||
2. Patent and trademark rights are not licensed under this Public License.
|
||||
|
||||
3. To the extent possible, the Licensor waives any right to collect royalties from You for the exercise of the Licensed Rights, whether directly or through a collecting society under any voluntary or waivable statutory or compulsory licensing scheme. In all other cases the Licensor expressly reserves any right to collect such royalties.
|
||||
|
||||
Section 3 – License Conditions.
|
||||
|
||||
Your exercise of the Licensed Rights is expressly made subject to the following conditions.
|
||||
|
||||
a. Attribution.
|
||||
|
||||
1. If You Share the Licensed Material (including in modified form), You must:
|
||||
|
||||
A. retain the following if it is supplied by the Licensor with the Licensed Material:
|
||||
|
||||
i. identification of the creator(s) of the Licensed Material and any others designated to receive attribution, in any reasonable manner requested by the Licensor (including by pseudonym if designated);
|
||||
|
||||
ii. a copyright notice;
|
||||
|
||||
iii. a notice that refers to this Public License;
|
||||
|
||||
iv. a notice that refers to the disclaimer of warranties;
|
||||
|
||||
v. a URI or hyperlink to the Licensed Material to the extent reasonably practicable;
|
||||
|
||||
B. indicate if You modified the Licensed Material and retain an indication of any previous modifications; and
|
||||
|
||||
C. indicate the Licensed Material is licensed under this Public License, and include the text of, or the URI or hyperlink to, this Public License.
|
||||
|
||||
2. You may satisfy the conditions in Section 3(a)(1) in any reasonable manner based on the medium, means, and context in which You Share the Licensed Material. For example, it may be reasonable to satisfy the conditions by providing a URI or hyperlink to a resource that includes the required information.
|
||||
|
||||
3. If requested by the Licensor, You must remove any of the information required by Section 3(a)(1)(A) to the extent reasonably practicable.
|
||||
|
||||
b. ShareAlike.In addition to the conditions in Section 3(a), if You Share Adapted Material You produce, the following conditions also apply.
|
||||
|
||||
1. The Adapter’s License You apply must be a Creative Commons license with the same License Elements, this version or later, or a BY-SA Compatible License.
|
||||
|
||||
2. You must include the text of, or the URI or hyperlink to, the Adapter's License You apply. You may satisfy this condition in any reasonable manner based on the medium, means, and context in which You Share Adapted Material.
|
||||
|
||||
3. You may not offer or impose any additional or different terms or conditions on, or apply any Effective Technological Measures to, Adapted Material that restrict exercise of the rights granted under the Adapter's License You apply.
|
||||
|
||||
Section 4 – Sui Generis Database Rights.
|
||||
|
||||
Where the Licensed Rights include Sui Generis Database Rights that apply to Your use of the Licensed Material:
|
||||
|
||||
a. for the avoidance of doubt, Section 2(a)(1) grants You the right to extract, reuse, reproduce, and Share all or a substantial portion of the contents of the database;
|
||||
|
||||
b. if You include all or a substantial portion of the database contents in a database in which You have Sui Generis Database Rights, then the database in which You have Sui Generis Database Rights (but not its individual contents) is Adapted Material, including for purposes of Section 3(b); and
|
||||
|
||||
c. You must comply with the conditions in Section 3(a) if You Share all or a substantial portion of the contents of the database.
|
||||
For the avoidance of doubt, this Section 4 supplements and does not replace Your obligations under this Public License where the Licensed Rights include other Copyright and Similar Rights.
|
||||
|
||||
Section 5 – Disclaimer of Warranties and Limitation of Liability.
|
||||
|
||||
a. Unless otherwise separately undertaken by the Licensor, to the extent possible, the Licensor offers the Licensed Material as-is and as-available, and makes no representations or warranties of any kind concerning the Licensed Material, whether express, implied, statutory, or other. This includes, without limitation, warranties of title, merchantability, fitness for a particular purpose, non-infringement, absence of latent or other defects, accuracy, or the presence or absence of errors, whether or not known or discoverable. Where disclaimers of warranties are not allowed in full or in part, this disclaimer may not apply to You.
|
||||
|
||||
b. To the extent possible, in no event will the Licensor be liable to You on any legal theory (including, without limitation, negligence) or otherwise for any direct, special, indirect, incidental, consequential, punitive, exemplary, or other losses, costs, expenses, or damages arising out of this Public License or use of the Licensed Material, even if the Licensor has been advised of the possibility of such losses, costs, expenses, or damages. Where a limitation of liability is not allowed in full or in part, this limitation may not apply to You.
|
||||
|
||||
c. The disclaimer of warranties and limitation of liability provided above shall be interpreted in a manner that, to the extent possible, most closely approximates an absolute disclaimer and waiver of all liability.
|
||||
|
||||
Section 6 – Term and Termination.
|
||||
|
||||
a. This Public License applies for the term of the Copyright and Similar Rights licensed here. However, if You fail to comply with this Public License, then Your rights under this Public License terminate automatically.
|
||||
|
||||
b. Where Your right to use the Licensed Material has terminated under Section 6(a), it reinstates:
|
||||
|
||||
1. automatically as of the date the violation is cured, provided it is cured within 30 days of Your discovery of the violation; or
|
||||
|
||||
2. upon express reinstatement by the Licensor.
|
||||
|
||||
c. For the avoidance of doubt, this Section 6(b) does not affect any right the Licensor may have to seek remedies for Your violations of this Public License.
|
||||
|
||||
d. For the avoidance of doubt, the Licensor may also offer the Licensed Material under separate terms or conditions or stop distributing the Licensed Material at any time; however, doing so will not terminate this Public License.
|
||||
|
||||
e. Sections 1, 5, 6, 7, and 8 survive termination of this Public License.
|
||||
|
||||
Section 7 – Other Terms and Conditions.
|
||||
|
||||
a. The Licensor shall not be bound by any additional or different terms or conditions communicated by You unless expressly agreed.
|
||||
|
||||
b. Any arrangements, understandings, or agreements regarding the Licensed Material not stated herein are separate from and independent of the terms and conditions of this Public License.
|
||||
|
||||
Section 8 – Interpretation.
|
||||
|
||||
a. For the avoidance of doubt, this Public License does not, and shall not be interpreted to, reduce, limit, restrict, or impose conditions on any use of the Licensed Material that could lawfully be made without permission under this Public License.
|
||||
|
||||
b. To the extent possible, if any provision of this Public License is deemed unenforceable, it shall be automatically reformed to the minimum extent necessary to make it enforceable. If the provision cannot be reformed, it shall be severed from this Public License without affecting the enforceability of the remaining terms and conditions.
|
||||
|
||||
c. No term or condition of this Public License will be waived and no failure to comply consented to unless expressly agreed to by the Licensor.
|
||||
|
||||
d. Nothing in this Public License constitutes or may be interpreted as a limitation upon, or waiver of, any privileges and immunities that apply to the Licensor or You, including from the legal processes of any jurisdiction or authority.
|
||||
|
||||
Creative Commons is not a party to its public licenses. Notwithstanding, Creative Commons may elect to apply one of its public licenses to material it publishes and in those instances will be considered the “Licensor.” Except for the limited purpose of indicating that material is shared under a Creative Commons public license or as otherwise permitted by the Creative Commons policies published at creativecommons.org/policies, Creative Commons does not authorize the use of the trademark “Creative Commons” or any other trademark or logo of Creative Commons without its prior written consent including, without limitation, in connection with any unauthorized modifications to any of its public licenses or any other arrangements, understandings, or agreements concerning use of licensed material. For the avoidance of doubt, this paragraph does not form part of the public licenses.
|
||||
|
||||
Creative Commons may be contacted at creativecommons.org.
|
||||
18
docs/oss/licenses/MIT.txt
Normal file
18
docs/oss/licenses/MIT.txt
Normal file
@@ -0,0 +1,18 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) <year> <copyright holders>
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
|
||||
associated documentation files (the "Software"), to deal in the Software without restriction, including
|
||||
without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the
|
||||
following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all copies or substantial
|
||||
portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
|
||||
LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO
|
||||
EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
|
||||
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
|
||||
USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
96
docs/policy/agents.md
Normal file
96
docs/policy/agents.md
Normal file
@@ -0,0 +1,96 @@
|
||||
# Agent Policy
|
||||
|
||||
**Base Policy Version:** 1.3
|
||||
|
||||
**Last Updated**: 2026-09-27
|
||||
|
||||
**Upstream Source:** [agents.md](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/docs/policy/agents.md)
|
||||
|
||||
## Purpose
|
||||
|
||||
Agents must read and follow these rules.
|
||||
|
||||
## Implementation Discipline
|
||||
|
||||
**KEY PRINCIPLES:** Write the simplest idiomatic code that satisfies the user's
|
||||
request. Do not implicitly strengthen requirements, restrictions, or guarantees
|
||||
beyond what the user actually requested.
|
||||
|
||||
**Keep implementation machinery proportionate.** Use straightforward,
|
||||
idiomatic code and introduce only the machinery needed to support the requested
|
||||
behavior and flexibility. Optimize for conceptual simplicity and
|
||||
maintainability, not for the fewest lines, fewest options, or narrowest product
|
||||
surface.
|
||||
|
||||
**Do not invent restrictions.** Do not reject, prohibit, constrain, cap,
|
||||
harden, or otherwise narrow behavior unless required by the user, an explicit
|
||||
repository contract, or an unavoidable correctness constraint. Preserve
|
||||
operator and caller choice where policy has not been established. If an
|
||||
additional restriction appears advisable, present it as a decision rather than
|
||||
silently implementing it.
|
||||
|
||||
**Do not invent guarantees.** Do not strengthen public contracts, guarantees,
|
||||
or invariants beyond those expressly requested by the user.
|
||||
|
||||
**Use the standard library.** For example: in almost every case, the best
|
||||
approach to copying files is to use the standard library. Writing custom code to
|
||||
implement power-loss protection at the application level, or to guard against
|
||||
bad actors with privileged and concurrent access to the host's filesystem,
|
||||
is wasteful, unnecessary, and not idiomatic. Stick to the simple, standard, and
|
||||
idiomatic approach unless another path is expressly requested by the user.
|
||||
|
||||
**An implementation detail is not repository policy.** Tests and documentation
|
||||
must not elevate incidental behavior into a durable guarantee or restriction
|
||||
unless that behavior was expressly selected as part of the contract.
|
||||
|
||||
## Promptkit Contract Authority
|
||||
|
||||
The following clarification preserves Promptkit's established distinction
|
||||
between implementation choices and authorized contracts.
|
||||
|
||||
A correctness constraint must follow from the requested behavior or an
|
||||
established invariant, not from the convenience of the chosen implementation.
|
||||
Explain the specific failure it prevents and why a less restrictive
|
||||
implementation cannot satisfy the contract.
|
||||
|
||||
Existing explicit contracts remain authoritative; changing or weakening one
|
||||
requires a deliberate decision. Agent-authored plans, tests, documentation,
|
||||
and prior implementations are not evidence that an additional restriction was
|
||||
deliberately approved. Identify the authorizing requirement or decision; when
|
||||
that authority is unclear, surface the ambiguity rather than treating repetition
|
||||
as approval.
|
||||
|
||||
## Git Commit Policy
|
||||
|
||||
By default, commit each completed, nonempty unit of work using a
|
||||
concise, plain-English message describing what changed. This policy is standing
|
||||
authorization to create those local commits; a separate user request is not
|
||||
required. An explicit user instruction to leave work uncommitted takes precedence.
|
||||
|
||||
A unit of work is a focused, reviewable outcome, including its relevant tests and
|
||||
documentation.
|
||||
Complete the appropriate validation before committing. If work is incomplete or
|
||||
validation is blocked, report that state accurately.
|
||||
|
||||
Before committing:
|
||||
|
||||
1. Run `git status --short` and review the relevant diff, including any staged
|
||||
changes. Identify changes that predate the current unit of work.
|
||||
2. Exclude secrets, local environment files, generated junk, and unrelated changes.
|
||||
3. Stage only the files or hunks belonging to the completed unit. Use `git add .`
|
||||
only when every included change belongs to that unit. Preserve unrelated
|
||||
staged and unstaged work.
|
||||
4. Review `git diff --staged` and run appropriate checks for the change. Reuse
|
||||
successful checks for unchanged content.
|
||||
5. Review `git log --oneline -n 10` and use
|
||||
`git commit -m "<concise plain-English commit message>"`.
|
||||
6. Report the commit identifier and any remaining uncommitted work.
|
||||
|
||||
If the unit cannot be safely separated from pre-existing work, leave it uncommitted
|
||||
and explain the overlap rather than silently including that work. If a commit
|
||||
fails, preserve the changes and report the failure; do not bypass hooks.
|
||||
|
||||
Default commit authorization does not authorize pushing, publishing, amending,
|
||||
resetting, reverting, or rewriting history. Those actions require a separate user
|
||||
instruction. Do not use `git commit -a` or `git commit --no-verify` to bypass the
|
||||
staging and validation discipline above.
|
||||
@@ -1,11 +1,41 @@
|
||||
# Architecture Policy
|
||||
|
||||
## Purpose
|
||||
**Base Policy Version:** 1.3
|
||||
|
||||
This document defines Promptkit's current high-level architecture and the
|
||||
durable boundaries that implementation changes must preserve. The
|
||||
[internal component overview](../internal/overview.md) inventories concrete
|
||||
implemented packages without redefining these rules.
|
||||
**Last Updated**: 2026-09-27
|
||||
|
||||
**Upstream Source:** [architecture.md](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/docs/policy/architecture.md)
|
||||
|
||||
## Purpose And Ownership
|
||||
|
||||
The adopted policy defines Promptkit's purpose and the architectural
|
||||
constraints contributors must follow. It owns system boundaries, responsibility
|
||||
and dependency rules, established invariants, and explicit non-goals.
|
||||
|
||||
Describe the concrete package inventory and implementation mechanics in internal
|
||||
documentation. Link to canonical public contracts for their exact behavior and
|
||||
to ADRs for decision rationale.
|
||||
|
||||
## Architectural Approach
|
||||
|
||||
Hexagonal architecture is strongly recommended, including encapsulating external
|
||||
integrations and APIs within internal adapters. Keep domain and application
|
||||
logic independent of external integration details, and make dependency direction
|
||||
clear in the adopted policy.
|
||||
|
||||
Implement that separation with proportionate, idiomatic code. Choose interfaces
|
||||
and package layout to serve the application's actual boundaries. Use adapters
|
||||
for collaborators with external integration responsibilities.
|
||||
|
||||
## Adopted Constraints And Implemented Structure
|
||||
|
||||
Architectural constraints govern current and future development. Record
|
||||
implemented structure accurately, proposals and implementation status in the
|
||||
roadmap, and significant decisions and rationale in ADRs. Describe a target
|
||||
structure as implemented when its changes land.
|
||||
|
||||
Constraints must reflect deliberately selected requirements and decisions.
|
||||
Do not promote incidental implementation choices into architectural invariants.
|
||||
|
||||
## System Shape
|
||||
|
||||
@@ -17,55 +47,26 @@ The module root contains package `promptkit`, which is the public facade. It
|
||||
provides the supported engine, configuration and source options, requests,
|
||||
results, public values, extension interfaces, profiles, and error sentinels.
|
||||
|
||||
The implemented internal components consist of:
|
||||
The [internal component overview](../internal/overview.md) owns the inventory
|
||||
of implemented packages and their concrete responsibilities. This policy owns
|
||||
the boundaries and dependency rules those components must preserve.
|
||||
|
||||
- `internal/domain`, which owns framework data values and source-neutral
|
||||
invariants shared by later internal components;
|
||||
- `internal/backend`, which owns validated immutable OpenAI-compatible backend
|
||||
definitions and the built-in OpenRouter definition;
|
||||
- `internal/capacity`, which owns engine-local bounded run admission and
|
||||
model-generation scheduling for limited backends;
|
||||
- `internal/defaults`, which owns application-neutral framework defaults and
|
||||
constructs the default execution target;
|
||||
- `internal/filecatalog`, which discovers YAML files and provides source-path
|
||||
helpers for filesystem and `fs.FS` consumers;
|
||||
- `internal/jsonvalue`, which validates and defensively copies JSON-compatible
|
||||
extra-parameter trees;
|
||||
- `internal/promptdef`, which loads and validates prompt definitions from
|
||||
filesystem and `fs.FS` sources;
|
||||
- `internal/profile`, which loads, validates, and overlays execution profiles
|
||||
from filesystem and `fs.FS` sources;
|
||||
- `internal/profile/builtin`, which embeds the built-in execution profile
|
||||
catalog;
|
||||
- `internal/prompt`, which renders prompt messages from Go templates;
|
||||
- `internal/artifact`, which resolves ordinary inline and unrestricted
|
||||
caller-selected file references;
|
||||
- `internal/validate`, which validates basic, JSON, and JSON Schema output
|
||||
using filesystem and `fs.FS` schema sources;
|
||||
- `internal/llm`, which defines the provider-neutral generation boundary and
|
||||
implements outbound OpenAI-compatible chat requests; and
|
||||
- `internal/usecase`, which coordinates preparation and execution across the
|
||||
internal framework components.
|
||||
The internal engine composes selected catalog and caller ConfigKit inputs once,
|
||||
then assembles the captured definitions, capacity manager, renderer, validator,
|
||||
outbound client, and use-case runner. The configuration adapter interprets and
|
||||
validates only the final effective values, including presence, null, and
|
||||
resource context. It retains source history for explicit operator inspection,
|
||||
not for validation or a second resolution pass. The root facade translates
|
||||
public values and errors at the library boundary.
|
||||
|
||||
The `examples/go-library/prepare` and `examples/go-library/run` packages are
|
||||
maintained downstream consumers of the root facade. They do not expose library
|
||||
packages or participate in internal assembly.
|
||||
|
||||
The root facade assembles one immutable backend registry, one capacity manager,
|
||||
the internal repositories, renderer, validator, outbound client, and use-case
|
||||
runner while translating public values and errors at the library boundary. The
|
||||
registry contains built-ins plus validated engine-scoped consumer additions.
|
||||
The facade constructs the capacity manager from the registry's immutable
|
||||
policy snapshot, wraps the selected built-in or injected model client, and
|
||||
supplies bounded admission to the runner. The defaults and renderer depend on
|
||||
the domain model. Prompt-definition and profile repositories use the domain
|
||||
model, file catalog, and YAML decoder. The built-in profile repository supplies
|
||||
an embedded `fs.FS` to the profile package. Artifact reading uses the domain
|
||||
model and application-neutral defaults. Validation uses the domain model, file
|
||||
catalog, and JSON Schema implementation. The model client uses the domain
|
||||
model, application-neutral defaults, and an injected or standard-library HTTP
|
||||
client. The use-case runner depends on the narrow interfaces owned by each
|
||||
internal component, including backend lookup and run admission.
|
||||
The engine constructs capacity policy from final backend definitions, wraps
|
||||
the selected built-in or injected model client, and supplies bounded admission
|
||||
to the runner. Runtime/domain components consume execution-owned values, not
|
||||
ConfigKit types. Artifact reading and validation use the domain model and
|
||||
application-neutral defaults; validation also uses file-catalog helpers and
|
||||
JSON Schema. The model client uses the domain model, defaults, and an injected
|
||||
or standard-library HTTP client. The runner coordinates narrow lookup and
|
||||
collaborator interfaces without rereading configuration.
|
||||
|
||||
The current implementation follows this dependency direction:
|
||||
|
||||
@@ -82,9 +83,10 @@ downstream consumers, including Scriptorium
|
||||
narrow injected abstractions
|
||||
```
|
||||
|
||||
The backend registry depends on the domain model and shared JSON-value
|
||||
validation, has no mutation API after construction, and consumes the
|
||||
OpenAI-compatible reserved request-field rule owned by the model client. The
|
||||
The backend index depends on the domain model and shared JSON-value ownership
|
||||
and has no mutation API after construction. The configuration adapter applies
|
||||
the OpenAI-compatible reserved request-field rule owned by the model client
|
||||
when it validates final provider maps. The
|
||||
capacity component depends on the domain model and the narrow internal
|
||||
model-client boundary, not on provider transport implementation. The model
|
||||
client does not depend on registry or capacity configuration. The facade
|
||||
@@ -125,8 +127,10 @@ consumer contract requires a public package.
|
||||
Each package must have one cohesive responsibility and a clear dependency
|
||||
direction. Internal packages must not expose their types merely to simplify
|
||||
wiring, and the public facade must not leak internal representations through
|
||||
exported signatures. New public packages require a durable consumer need that
|
||||
cannot be served cleanly by the root facade.
|
||||
exported signatures. Production packages under `internal/` must not import the
|
||||
module root; the root facade may depend on internal implementation packages.
|
||||
New public packages require a durable consumer need that cannot be served
|
||||
cleanly by the root facade.
|
||||
|
||||
The [internal component overview](../internal/overview.md) must be updated as
|
||||
packages are implemented or their responsibilities change.
|
||||
@@ -177,3 +181,21 @@ state.
|
||||
Do not list planned packages as implemented components. When implementation
|
||||
introduces a package, update the internal inventory and the owning contract or
|
||||
subsystem document in the same change.
|
||||
|
||||
## Trust Boundaries And Threat Scope
|
||||
|
||||
Where useful, state the trust boundaries and threat scenarios relevant to the
|
||||
application's adopted requirements, along with scenarios it explicitly excludes.
|
||||
For example, distinguish handling untrusted application input from defending
|
||||
against a privileged actor concurrently modifying the host filesystem.
|
||||
|
||||
Threat exclusions define the scope of the adopted requirements. Base defenses
|
||||
and guarantees on those requirements and raise unsettled scope for a deliberate
|
||||
decision under the [agent policy](agents.md).
|
||||
|
||||
Keep this discussion proportionate to the application. A separate threat
|
||||
assessment or security document is optional when adopting this template.
|
||||
For this library, the framework format and integration documents own credential
|
||||
supply and external contracts; internal documents own implementation mechanisms.
|
||||
Consumer-specific filesystem and security policy remains outside Promptkit's
|
||||
boundary as described above.
|
||||
|
||||
@@ -1,138 +1,139 @@
|
||||
# Documentation Policy
|
||||
|
||||
**Base Policy Version:** 1.11
|
||||
|
||||
**Last Updated**: 2026-09-27
|
||||
|
||||
**Upstream Source:** [documentation.md](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/docs/policy/documentation.md)
|
||||
|
||||
## Purpose
|
||||
|
||||
This policy assigns each Promptkit documentation topic to one canonical owner.
|
||||
Its goal is to keep documentation for this reusable Go library accurate,
|
||||
concise, discoverable, and resistant to drift for consumers, contributors,
|
||||
maintainers, integrators, and coding agents.
|
||||
This policy assigns each documentation topic to one canonical owner. Its goal is
|
||||
to keep this library's documentation accurate, concise, discoverable, and resistant
|
||||
to drift for users, operators, developers, integrators, and LLM coding agents.
|
||||
|
||||
## Core Rules
|
||||
|
||||
### One Canonical Owner
|
||||
|
||||
Each authoritative fact belongs in one document or source form. A non-owning
|
||||
document may give a short, stable summary for orientation, but it must link to
|
||||
the canonical owner instead of repeating exact contracts.
|
||||
Each authoritative fact belongs in one document. A non-owning document may give
|
||||
a short, stable summary for orientation, but it must link to the canonical owner
|
||||
for volatile details.
|
||||
|
||||
Volatile details include exported declarations, accepted inputs, defaults,
|
||||
schemas, file names, paths, error identities, retry behavior, and runtime
|
||||
guarantees. If readers could reasonably treat a statement as a contract,
|
||||
maintain its exact definition only in the owning source.
|
||||
Volatile details include commands, flags, configuration fields and defaults,
|
||||
module keys, schemas, file names, paths, status codes, retry behavior, and
|
||||
runtime guarantees. If readers could reasonably treat a statement as a
|
||||
contract, maintain it only in the owning document.
|
||||
|
||||
### Current State, Decisions, And Future Work
|
||||
### Current And Future Behavior
|
||||
|
||||
Outside `docs/roadmap/`, documentation describes implemented behavior only.
|
||||
Partial features may be described only to their implemented boundary.
|
||||
|
||||
An accepted architecture decision may describe an approved direction before it
|
||||
is implemented, but acceptance is not evidence that the behavior exists.
|
||||
Current-state documents change when the implementation lands. Temporary
|
||||
roadmaps own future work, sequencing, and implementation status; they do not
|
||||
replace durable policies or current contracts.
|
||||
Architectural constraints may govern future changes. Describe current structure
|
||||
from implemented behavior.
|
||||
|
||||
ADRs may record an accepted architectural decision before implementation. The
|
||||
roadmap owns implementation status and sequencing until the decision is
|
||||
implemented. Current architecture, user, operator, integration, and internal
|
||||
documentation are updated when the behavior lands.
|
||||
|
||||
### Audience And Detail
|
||||
|
||||
Write for the document's stated audience and include only the detail needed for
|
||||
its owned topic. Consumer guidance should not expose incidental implementation
|
||||
detail. Contributor documentation should link to public contracts and durable
|
||||
policies instead of restating them.
|
||||
its owned topic. Keep user and operator docs focused on using and operating the
|
||||
application. Developer docs should link to user-facing and external contracts
|
||||
for their definitions.
|
||||
|
||||
### Links
|
||||
### Examples
|
||||
|
||||
Use descriptive link text and repository-relative links for repository
|
||||
documents. Link to the canonical owner rather than to a duplicate summary.
|
||||
Check every added or changed link and repair or remove links when their target
|
||||
moves or is retired.
|
||||
Complete copyable files belong in `examples/`. Documentation may use the
|
||||
smallest illustrative snippet needed to explain its owned topic, but should link
|
||||
to maintained examples for complete copyable files.
|
||||
|
||||
### Examples And Code Fences
|
||||
|
||||
Complete copyable files belong in `examples/` when maintained examples exist.
|
||||
Documentation may use the smallest illustrative snippet needed for its owned
|
||||
topic, but should link to a maintained example instead of embedding a second
|
||||
complete copy.
|
||||
|
||||
Examples must be valid, secret-free, and tested where practical. Commands,
|
||||
imports, and Go snippets must match the implemented library. Use a language tag
|
||||
on fenced code blocks and make clear when a fragment is illustrative rather
|
||||
than directly runnable.
|
||||
Examples must be valid, secret-free, and tested where practical. Commands and
|
||||
configuration used in documentation should match the application.
|
||||
|
||||
### Security And Privacy
|
||||
|
||||
Documentation and examples must not contain real credentials, private keys,
|
||||
private environment dumps, sensitive source material, or private
|
||||
infrastructure details unless intentionally public. Document secret-handling
|
||||
mechanisms, not secret values.
|
||||
private environment dumps, sensitive source material, or private infrastructure
|
||||
details unless intentionally public. Document secret-handling mechanisms, not
|
||||
secret values.
|
||||
|
||||
## Canonical Ownership
|
||||
|
||||
| Topic | Canonical owner | Owned content | Content owned elsewhere |
|
||||
| --- | --- | --- | --- |
|
||||
| Project orientation | `README.md` | What Promptkit is, its current usability, module identity, license summary, and links onward. | Exact API contracts, contributor procedures, architecture detail, and release steps. |
|
||||
| Contributor workflow | `docs/development.md` | Task-oriented reading guide, local workflow, validation commands, and repository hygiene. | Architecture rules, API semantics, subsystem behavior, and release procedure. |
|
||||
| Current architecture | `docs/policy/architecture.md` | System shape, normative ownership, dependency direction, package boundaries, invariants, and non-goals. | Concrete component inventory, implementation mechanics, contributor procedures, decision history, and future work. |
|
||||
| Documentation organization | `docs/policy/documentation.md` | Documentation ownership, audience boundaries, maintenance rules, and document lifecycle. | Library architecture or runtime behavior. |
|
||||
| Testing policy | `docs/policy/testing.md` | Test philosophy, risk-based sufficiency, test boundaries, doubles, coverage guidance, regression policy, and test maintenance. | Subsystem behavior, exact public contracts, subsystem-specific test inventories, and implementation plans. |
|
||||
| Release procedure | `docs/release.md`, when present | Required release validation, version and tag procedure, release ordering, and post-publication checks. | General contributor workflow, public API semantics, and decision history. |
|
||||
| Product orientation and minimal end-to-end quickstart | `README.md` | Purpose, value, shortest successful invocation, and navigation, including licensing and attribution. | Complete command reference, configuration reference, operational procedures, implementation detail. |
|
||||
| Contributor entry point | `docs/development.md` | Task-oriented reading guide, contributor orientation, baseline validation commands, and canonical links. | Package inventory, architecture rules, subsystem behavior, detailed change recipes. |
|
||||
| Application architecture policy | `docs/policy/architecture.md` | System shape, normative ownership, dependency direction, boundaries, invariants, safety properties, and non-goals. | Concrete package inventory, implementation mechanics, contributor procedures, decision history, future work. |
|
||||
| Policy-set provenance and upstream navigation | `docs/policy/meta.md` | Policy-set origin, local authority, adoption baselines, and upstream adoption and update links. | Task-specific reading triggers, detailed adoption procedures, and formal attribution notices. |
|
||||
| Agent policy | `docs/policy/agents.md` | Proportionate, idiomatic implementation; rules against inventing requirements, restrictions, guarantees, or policy. | Application contracts, architecture, testing policy, and contributor procedures. |
|
||||
| Documentation policy | `docs/policy/documentation.md` | Documentation ownership, audience boundaries, maintenance rules, and ADR/document lifecycle. | Application architecture or product behavior. |
|
||||
| Testing policy | `docs/policy/testing.md` | Test philosophy, risk sufficiency, boundaries, doubles, coverage, regressions, and test maintenance. | Subsystem behavior, application contracts, subsystem-specific test inventories, and implementation plans. |
|
||||
| Release policy | `docs/policy/release.md` | Versioning, compatibility, release integrity and immutability, release documentation, proportionate gates, publication controls, completion, and required versus optional checks. | Repository-specific commands, product installation guidance, CLI version semantics, historical release summaries, and CI implementation detail. |
|
||||
| Open source licensing and attribution policy | `docs/policy/oss.md` | License defaults and author choices, COPYRIGHT and attribution formats, license/notice links, discoverability, maintenance, and concrete exceptions. | Component entries, license texts, release commands, and dependency security review. |
|
||||
| Project copyright and license grant | `COPYRIGHT` | Project identity, copyright holders and years, SPDX expression, grant, warranty disclaimer, and third-party statement. | Complete license texts and upstream component attribution. |
|
||||
| Project license texts | `LICENSE` | Complete project and incorporated companion license texts, with required project-specific fields. | General licensing policy and upstream component attribution. |
|
||||
| Upstream component attribution | `docs/oss/attribution.md` | House Format component entries: identity, material scope and modifications where applicable, copyright, license, required notices, and license, notice, and source links. | General attribution policy, exception reasoning, research narratives, compliance commentary, and complete standard license texts. |
|
||||
| Standard dependency license texts | `docs/oss/licenses/` | Complete, verbatim standard license texts shared by applicable entries. | Project-specific upstream notices, attribution prose, and policy interpretation. |
|
||||
| Repository-specific release procedure | `docs/release.md` | Maintainer prerequisites, candidate preparation, validation and publication commands and automated guards, asynchronous deliverables, delivery status, optional consumer verification, and recovery. | General release requirements, product installation guidance, historical release summaries, and current compatibility contracts. |
|
||||
| Consumer guidance for the public Go API (if applicable) | `docs/consumers/` | Task-oriented public Go API usage, minimal examples, and consumer responsibilities. | HTTP wire semantics, external protocol contracts, internal implementation detail. |
|
||||
| Integration contracts for external components (if applicable) | `docs/integrations/` | External formats and protocols, upstream and downstream contracts, logical output paths and schemas, media types, and compatibility. | Physical runtime placement and lifecycle, internal transformations, CLI syntax, configuration defaults. |
|
||||
| Inventory of implemented internal packages | `docs/internal/overview.md` | Implemented packages and components, responsibilities, and links to focused internal docs. | Normative architecture, contributor reading policy, external contracts. |
|
||||
| Behavior and architecture of internal packages | Other files under `docs/internal/` | Implementation flow, collaborators, state transitions, package-local guarantees and failures, and relevant tests. | Global architecture invariants, configuration definitions and defaults, external schemas, operator procedures. |
|
||||
| Architectural decision records | `docs/adr/` | Significant decisions, context, alternatives, rationale, consequences, and supersession history. | Current behavior reference, implementation status, task sequencing. |
|
||||
| Future work and implementation status | `docs/roadmap/` | Proposed, accepted, deferred, or rejected work; implementation status, sequencing, and task breakdowns. | Implemented behavior reference and architectural decision rationale. |
|
||||
| Release-note history | `docs/releases/` | One checked-in summary per source release under the procedure; the note at its immutable tag is the release record. | Current commands, behavior, contracts, and compatibility definitions. |
|
||||
| Complete copyable artifacts | `examples/` | Maintained configuration, inputs, and other copyable or runnable files. | Field-by-field reference, command reference, prose explanation. |
|
||||
| Exact exported Go API | Go declarations and GoDoc, as APIs are implemented | Exported names, signatures, types, values, errors, and exact behavioral contracts. | Task-oriented consumer walkthroughs, implementation details, and future API proposals. |
|
||||
| Framework file formats | `docs/formats.md` | Prompt-definition and profile YAML fields, schema references, defaults, validation modes, built-in profiles, credentials, and file-to-request precedence. | Exported Go declarations, outbound wire behavior, internal parsing mechanics, and application configuration. |
|
||||
| Consumer guidance | `docs/consumers/`, when consumer workflows require dedicated guidance | Task-oriented use of implemented public APIs, minimal examples, and consumer responsibilities. | Exact exported declarations and internal mechanics. |
|
||||
| Durable integration contracts | `docs/integrations/`, when integrations exist | External formats and protocols, compatibility behavior, and upstream or downstream responsibilities. | Internal transformations and public Go declarations. |
|
||||
| Supplemental release guidance | None. `docs/releases/` may be used when a release benefits from a changelog or migration guide. | No canonical content. These files may briefly summarize release-specific changes, compatibility, and consumer migration paths, and may be corrected, consolidated, archived, or removed when no longer useful. | Public API and behavior contracts, formats, integrations, architecture, release procedure, and the authoritative annotated-tag release record. |
|
||||
| Implemented component inventory | `docs/internal/overview.md` | Current packages and components, their implemented responsibilities, and links to focused internal documents. | Normative architecture, contributor workflow, external contracts, and proposed components. |
|
||||
| Internal subsystem behavior | Other files under `docs/internal/`, when a subsystem needs durable detail | Implementation flow, internal collaborators and state transitions, package-local guarantees and failures, and relevant tests. | Global architecture invariants, public API definitions, and future package plans. |
|
||||
| Architectural decision history | `docs/adr/`, when repository-local decisions require records | Significant decisions, context, alternatives, rationale, consequences, and supersession history. | Current behavior reference, implementation status, and task sequencing. |
|
||||
| Temporary feature roadmaps | `docs/roadmap/`, while planned work needs coordination | Proposed or accepted scope, sequencing, gates, and implementation status. | Implemented behavior reference and durable decision rationale. |
|
||||
| Complete copyable artifacts | `examples/` | Valid inputs, Go programs, and other files intended to be copied or run. | Field-by-field reference, exact API declarations, and prose explanation. |
|
||||
|
||||
Conditional owners do not require placeholder files or directories. Create a
|
||||
consumer, integration, release, subsystem, ADR, roadmap, or example document
|
||||
only when the corresponding implemented interface, release, decision, planned
|
||||
effort, or maintained artifact exists.
|
||||
Create supporting documents for existing interfaces and responsibilities when
|
||||
substantive content is available.
|
||||
|
||||
## Boundary Rules
|
||||
|
||||
### Orientation
|
||||
|
||||
The README owns project orientation. The development guide routes
|
||||
contributors. Architecture owns normative structure. The internal overview
|
||||
owns the current concrete component map. These documents may link to one
|
||||
another but must not maintain parallel package or behavior descriptions.
|
||||
Link among orientation, architecture, and internal documents; maintain each
|
||||
package or behavior description in its canonical owner.
|
||||
|
||||
### Public Contracts And Implementation
|
||||
|
||||
Go declarations and GoDoc own exact exported API contracts once those APIs
|
||||
exist. Consumer and integration documents explain how to use those contracts
|
||||
for a task. Internal documents explain how Promptkit implements them. Internal
|
||||
documentation may identify a public type or external format as a dependency,
|
||||
but must link to its canonical definition rather than restate it.
|
||||
Go declarations and GoDoc own exact exported Go API contracts. The
|
||||
[framework format reference](../formats.md) owns prompt, profile, and schema
|
||||
formats; integration documents own external wire behavior. Consumer and
|
||||
internal documents link to those owners for exact definitions.
|
||||
|
||||
The [framework format reference](../formats.md) owns exact prompt, profile, and
|
||||
schema-file contracts. Integration documents own external wire formats.
|
||||
### Releases
|
||||
|
||||
### Supplemental Release Guidance
|
||||
The release policy defines requirements and invariants that every release must
|
||||
satisfy. The repository-specific release procedure defines the commands and
|
||||
workflow maintainers use to satisfy that policy. Release notes record history
|
||||
at immutable tags. Maintain current behavior in its canonical documentation,
|
||||
update it with the corresponding behavior changes, and keep release procedures
|
||||
consistent with adopted policy.
|
||||
|
||||
Files under `docs/releases/` may provide changelog-style summaries and
|
||||
migration guidance for a particular release. They are navigation and
|
||||
orientation aids, not canonical owners of public APIs, behavior, formats,
|
||||
integrations, architecture, release procedure, or other durable facts. When a
|
||||
reader needs detail beyond a short release-specific note, the release document
|
||||
must link to the applicable canonical documentation rather than reproduce its
|
||||
contract.
|
||||
### Project Licensing And Upstream Attribution
|
||||
|
||||
The annotated tag message required by the
|
||||
[release procedure](../release.md#write-the-release-note) remains the
|
||||
authoritative release record. Supplemental release documents may be corrected,
|
||||
consolidated, archived, or removed at any time when they are no longer useful,
|
||||
provided maintained documentation does not depend on them and the annotated
|
||||
tag record remains intact.
|
||||
README and release documents link to the canonical licensing and attribution
|
||||
materials under the [OSS policy](oss.md). The release procedure owns URL
|
||||
construction and publication mechanics.
|
||||
|
||||
### Contracts And Implementation
|
||||
|
||||
Integration and API documents define externally observable shapes and
|
||||
semantics. Internal documents explain how this application implements or consumes those
|
||||
contracts. Internal docs may name a field, file, or protocol to identify a
|
||||
dependency, but must link to its canonical contract for the definition.
|
||||
|
||||
### Security Topics
|
||||
|
||||
This policy owns what documentation and examples may contain. Architecture owns
|
||||
library security boundaries and invariants. Public declarations and integration
|
||||
documents own consumer-visible security contracts. Internal documents own
|
||||
implementation mechanisms only.
|
||||
library security boundaries. Public declarations and integration documents own
|
||||
consumer-visible contracts; internal documents own implementation mechanisms.
|
||||
|
||||
## Architecture Decision Records
|
||||
|
||||
@@ -147,48 +148,47 @@ Use sequentially numbered ADR filenames such as
|
||||
6. alternatives considered;
|
||||
7. consequences.
|
||||
|
||||
Use one of these statuses:
|
||||
Treat the decision content of an accepted ADR as immutable. When a decision
|
||||
changes, create a new ADR and update the earlier ADR's status to superseded.
|
||||
Rejected architectural alternatives belong in the ADR; rejected product ideas
|
||||
belong in the roadmap.
|
||||
|
||||
- **Proposed:** the decision is under consideration and may change;
|
||||
- **Accepted:** the decision is approved, whether or not implementation is
|
||||
complete;
|
||||
- **Rejected:** the proposed decision was considered and not adopted;
|
||||
- **Superseded:** a later accepted ADR replaces the accepted decision.
|
||||
## Maintenance
|
||||
|
||||
A proposed ADR transitions to Accepted or Rejected. An Accepted ADR transitions
|
||||
to Superseded only when a later Accepted ADR replaces it. An ADR may be created
|
||||
as Accepted when the decision has already been made.
|
||||
When behavior changes, update its canonical owner in the same change. If
|
||||
ownership moves, remove the old definition and replace it with a link where
|
||||
navigation remains useful.
|
||||
|
||||
Treat the decision content of an Accepted ADR as immutable. A changed decision
|
||||
requires a later ADR rather than a rewrite of the accepted record. A Superseded
|
||||
ADR must link to its replacement, and the replacement must link back. Rejected
|
||||
architectural alternatives belong in the ADR; rejected feature ideas belong in
|
||||
a roadmap when they need to be retained.
|
||||
Before completing documentation work:
|
||||
|
||||
- verify affected behavior and examples;
|
||||
- check commands, flags, fields, defaults, schemas, and paths against their
|
||||
implementation;
|
||||
- keep unimplemented behavior in the roadmap, subject to the ADR exception;
|
||||
- remove stale references and validate links;
|
||||
- confirm that non-owning documents summarize and link rather than redefine;
|
||||
- confirm that no secrets or sensitive private data were added.
|
||||
|
||||
## Promptkit Documentation Scope
|
||||
|
||||
Promptkit is a Go library, so exported GoDoc and framework formats replace
|
||||
application CLI, configuration, operations, and HTTP-service owners. Reading
|
||||
triggers belong in the development guide, not this topic map.
|
||||
|
||||
## Document Lifecycle
|
||||
|
||||
Create durable current-state documentation with the implementation it
|
||||
describes. Update its canonical owner in the same change when behavior changes.
|
||||
If ownership moves, remove the old definition and leave a link where navigation
|
||||
remains useful.
|
||||
|
||||
Roadmaps are temporary coordination documents. When their work is complete,
|
||||
record completion, move any still-useful decisions or contracts to their
|
||||
durable owners, update incoming links, and archive or remove the roadmap
|
||||
according to repository practice. Do not preserve completed roadmaps as a
|
||||
second current-state reference.
|
||||
|
||||
Supplemental release documents may likewise be removed without preserving a
|
||||
replacement. Before removal, update maintained incoming links so current
|
||||
documentation does not depend on an optional historical guide.
|
||||
Retain release notes as historical records under the release policy. Do not
|
||||
apply temporary-roadmap retirement rules to published release notes.
|
||||
|
||||
Before completing documentation work:
|
||||
|
||||
- verify affected behavior and examples;
|
||||
- check commands, imports, declarations, defaults, schemas, and paths against
|
||||
their implementation;
|
||||
- keep unimplemented behavior in a roadmap, subject to the ADR exception;
|
||||
- validate links and fenced examples;
|
||||
- confirm non-owning documents summarize and link rather than redefine;
|
||||
- remove stale or unsupported claims; and
|
||||
- confirm that no secrets or sensitive private data were added.
|
||||
For Promptkit ADRs, retain the existing Proposed, Accepted, Rejected, and
|
||||
Superseded statuses. Proposed records may become Accepted or Rejected;
|
||||
Accepted records may become Superseded, with links between the earlier record
|
||||
and its replacement. Records may be created as Accepted when the decision has
|
||||
already been made. These lifecycle rules preserve existing local document
|
||||
semantics.
|
||||
|
||||
27
docs/policy/meta.md
Normal file
27
docs/policy/meta.md
Normal file
@@ -0,0 +1,27 @@
|
||||
# Policy Set Provenance
|
||||
|
||||
**Base Policy Version:** 1.0
|
||||
|
||||
**Last Updated**: 2026-09-27
|
||||
|
||||
**Upstream Source:** [meta.md](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/docs/policy/meta.md)
|
||||
|
||||
## Origin And Local Authority
|
||||
|
||||
This repository's policy set is adapted from
|
||||
[Standard Repository Policies](https://gitea.maximumdirect.net/eric/policy).
|
||||
The adopted local policies govern work in this repository. Upstream revisions
|
||||
become local policy through deliberate adoption, preserving local decisions.
|
||||
Each policy's `Upstream Source` identifies its adopted baseline.
|
||||
|
||||
## Adoption And Updates
|
||||
|
||||
For policy adoption, upstream revision reviews, and updates, consult the
|
||||
[upstream adoption and update guide](https://gitea.maximumdirect.net/eric/policy/src/branch/main/docs/adoption.md).
|
||||
That guide and the reference example collection are maintained upstream and
|
||||
accessed through links. Keep discovery links pointed at the current upstream
|
||||
project and guide; keep adopted-source and example links pinned to commits.
|
||||
|
||||
The local [developer guide](../development.md) routes tasks to applicable
|
||||
policies. Formal credit and licensing notices for the copied policy material
|
||||
belong in the local [attribution document](../oss/attribution.md).
|
||||
227
docs/policy/oss.md
Normal file
227
docs/policy/oss.md
Normal file
@@ -0,0 +1,227 @@
|
||||
# Open Source Licensing And Attribution Policy
|
||||
|
||||
**Base Policy Version:** 1.8
|
||||
|
||||
**Last Updated**: 2026-09-27
|
||||
|
||||
**Upstream Source:** [oss.md](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/docs/policy/oss.md)
|
||||
|
||||
## Purpose And Scope
|
||||
|
||||
This policy defines project licensing conventions and a repeatable House Format
|
||||
for upstream attribution and license notices. It covers the project's own code
|
||||
and upstream components incorporated into or distributed with the application
|
||||
or library, including relevant transitive dependencies and copied code,
|
||||
documentation, or assets.
|
||||
Determine treatment from the component's terms and actual use.
|
||||
|
||||
Consulting development guidance or using tooling alone does not require an
|
||||
attribution entry. Copying or adapting documentation for inclusion in the
|
||||
repository is different: preserve its applicable attribution and license terms.
|
||||
A material's role as development guidance does not exempt copied documents.
|
||||
Apply any express permissions for attribution-free reuse of example formatting.
|
||||
|
||||
The baseline is a publicly available source repository with releases published
|
||||
alongside it. The format is independent of language, forge, and build system.
|
||||
It does not require attribution for every development tool, establish license
|
||||
compatibility, or resolve every obligation arising from use or distribution.
|
||||
|
||||
## Canonical Locations
|
||||
|
||||
The [documentation policy](documentation.md) assigns ownership:
|
||||
|
||||
- `COPYRIGHT`: project name, copyright statements, SPDX expression, grant,
|
||||
warranty disclaimer, and third-party statement.
|
||||
- `LICENSE` contains the complete, unmodified project license text, including
|
||||
incorporated companion text. Dependency attribution does not relicense the project.
|
||||
- `docs/oss/attribution.md`: canonical component entries.
|
||||
- `docs/oss/licenses/`: complete, verbatim standard and applicable companion texts.
|
||||
|
||||
Use recognizable license filenames, such as `Apache-2.0.txt` or `LGPL-3.0.txt`.
|
||||
A shared standard text may serve multiple entries. Keep explanations outside
|
||||
license texts and preserve distinct versions and exceptions.
|
||||
|
||||
## Upstream Reference Examples
|
||||
|
||||
Use the commit-pinned upstream examples to create actual project notices.
|
||||
Retain their links and keep the example collection upstream; keep applicable
|
||||
license texts locally as specified above.
|
||||
|
||||
## Project Licensing
|
||||
|
||||
Unless the author specifies otherwise, use **GPL version 3 or later**
|
||||
(`GPL-3.0-or-later`) for applications and **LGPL version 3 or later**
|
||||
(`LGPL-3.0-or-later`) for libraries. An author's explicit choice takes precedence:
|
||||
for example, a one-off script may use MIT. Such a choice requires no exception
|
||||
rationale, additional approval, or compensating controls under this policy.
|
||||
|
||||
These defaults guide new license selection. Adopting this policy does not
|
||||
relicense existing code or override third-party terms. Preserve an existing
|
||||
license unless the author deliberately changes it. For mixed-purpose projects,
|
||||
follow the author's selected scope rather than inferring different licenses for
|
||||
individual packages.
|
||||
|
||||
Use the upstream [application COPYRIGHT example](https://gitea.maximumdirect.net/eric/policy/src/commit/a469273adda336625f96b9d769ea27b33999f146/examples/licensing/application/COPYRIGHT)
|
||||
or [library COPYRIGHT example](https://gitea.maximumdirect.net/eric/policy/src/commit/a469273adda336625f96b9d769ea27b33999f146/examples/licensing/library/COPYRIGHT)
|
||||
to create the project’s root `COPYRIGHT` file.
|
||||
Use verified project names, contribution years, and copyright holders.
|
||||
For another selected license, adapt the grant, SPDX expression, disclaimer, and
|
||||
text references together to reflect the author's selected grant.
|
||||
|
||||
For GPLv3 projects, copy the complete GPLv3 text into `LICENSE`. For LGPLv3
|
||||
projects, place the complete LGPLv3 text followed by the complete GPLv3 text in
|
||||
`LICENSE`; the LGPL incorporates the GPL terms. Keep the texts unmodified and
|
||||
place project-specific statements in `COPYRIGHT`. For the default grants, the
|
||||
grant and SPDX expression in `COPYRIGHT` must expressly identify `or-later`;
|
||||
follow the author’s choice if a different grant is selected.
|
||||
|
||||
For other licenses, provide their complete text and any required project-specific
|
||||
fields; the standard reference collection remains unchanged. Keep `COPYRIGHT`,
|
||||
`LICENSE`, SPDX expressions, and README descriptions consistent with the actual
|
||||
grant. This convention introduces no blanket requirement for per-file headers;
|
||||
preserve notices or add them where the applicable terms require them.
|
||||
|
||||
Identify separately licensed documentation or assets in the project grant, with
|
||||
links to their attribution and license information. An application's default
|
||||
code license does not override the terms of incorporated documentation.
|
||||
|
||||
## Upstream Attribution House Format
|
||||
|
||||
Begin `docs/oss/attribution.md` with `# Open Source Attribution Notices`,
|
||||
followed directly by component entries. These comprise the complete document.
|
||||
|
||||
Each entry presents these five fields in order, with blank lines between them:
|
||||
|
||||
1. **Project name** as a level-two heading. An optional identification paragraph
|
||||
may follow, such as `Selected module: MODULE at VERSION.` with the module and
|
||||
version in backticks. For copied documentation, a concise `Material:` sentence
|
||||
may identify the affected files and whether they were adapted or copied
|
||||
without modification. A factual qualifier such as `(transitive dependency)`
|
||||
is permitted. Limit optional identification prose to factual component and
|
||||
material scope.
|
||||
2. **Upstream copyright statement or statements**, preserving their wording.
|
||||
Put multiple statements in separate paragraphs. Include this field when the
|
||||
selected upstream source supplies an applicable statement; otherwise, omit it.
|
||||
3. **SPDX license identifier or expression**, when applicable, using
|
||||
`SPDX-License-Identifier: EXPRESSION`.
|
||||
4. **License and notice links**, beginning `License and notices:`, separated
|
||||
by ` | `, and ending with a period. Label local standard texts by license
|
||||
name, such as `MIT` or `Apache 2.0`. Label upstream files by their actual
|
||||
filenames, such as `COPYRIGHT`, `LICENSE`, `NOTICE`, or `PATENTS`.
|
||||
5. **Source repository link**, rendered exactly as:
|
||||
|
||||
```markdown
|
||||
[Source](COMMIT_URL).
|
||||
```
|
||||
|
||||
Use the upstream [attribution example](https://gitea.maximumdirect.net/eric/policy/src/commit/a469273adda336625f96b9d769ea27b33999f146/examples/licensing/attribution.md)
|
||||
to create the project’s `docs/oss/attribution.md`.
|
||||
The example illustrates omitted fields and multiple copyrights and licenses.
|
||||
Replace the uppercase placeholders and illustrative URLs with actual component
|
||||
information. Local license links are relative to the destination
|
||||
`docs/oss/attribution.md`; copy the applicable texts into its `licenses/`
|
||||
directory. Do not copy example entries for components the project does not use.
|
||||
|
||||
These are five fields, not a line limit. Omit the SPDX field when no applicable
|
||||
identifier exists; identify nonstandard terms through the linked text. Preserve
|
||||
licensing alternatives and exceptions, and record a selected option when
|
||||
relevant through the factual license identification. Do not invent an identifier
|
||||
or silently resolve an unsettled licensing choice.
|
||||
|
||||
Keep entries factual. Do not add agent-authored explanations of license
|
||||
expressions, file-level license scope, research performed, retained source
|
||||
notices, or compliance conclusions. Avoid filler such as "none is invented
|
||||
here" and embellished link labels such as "upstream LICENSE", "additional
|
||||
patent grant", or "at the imported commit". Preserve actual required upstream
|
||||
content; this rule excludes editorial additions, not required notices. Policy
|
||||
interpretation and exception reasoning belong in the adopted OSS policy or an
|
||||
appropriate decision record, not the attribution document.
|
||||
|
||||
## License Texts And Upstream Links
|
||||
|
||||
Link standard license texts to their local copies. Link project-specific
|
||||
notices to their upstream copies. Every upstream repository or notice link must
|
||||
identify the exact commit corresponding to the
|
||||
imported version, including the selected source revision for copied material.
|
||||
|
||||
Classify material according to its contents. An upstream `LICENSE`,
|
||||
`COPYING`, or similar file may contain project-specific notices or exceptions
|
||||
in addition to standard wording. Link both the local standard text and the
|
||||
commit-specific upstream file when needed. Make companion license texts directly
|
||||
reachable from the entry.
|
||||
|
||||
For project-specific license and attribution material, this policy accepts
|
||||
direct links to the applicable upstream texts as satisfying reproduction,
|
||||
inclusion, or copy requirements. This is the adopted house interpretation.
|
||||
Agents must apply it without adding local-copy requirements or seeking repeated
|
||||
approval solely because a license uses those terms.
|
||||
|
||||
This interpretation concerns notice reproduction. Concrete obligations such as
|
||||
marking modified files or supplying corresponding source receive their own
|
||||
applicable treatment. Preserve required notices already present in incorporated
|
||||
source files; the central attribution entry does not authorize stripping them.
|
||||
|
||||
## Discoverability
|
||||
|
||||
End the project's README with two friendly statements linking to its actual
|
||||
grant in `COPYRIGHT` and the attribution document. For example, for a project
|
||||
with the stated license:
|
||||
|
||||
```markdown
|
||||
PROJECT_NAME is licensed under the [LGPL v3.0 or later](COPYRIGHT).
|
||||
|
||||
[Open Source attribution notices](docs/oss/attribution.md) are available.
|
||||
```
|
||||
|
||||
Each new checked-in release document and its hosted download page must end
|
||||
with links whose visible text is `COPYRIGHT | LICENSE | ATTRIBUTION`. Use absolute
|
||||
URLs to those files at the release's immutable tag or corresponding immutable source
|
||||
revision so the links work in both locations. This release navigation is distinct
|
||||
from upstream component links, which must use the imported commit.
|
||||
|
||||
Reuse the checked-in release document as the hosted release body where supported.
|
||||
The release procedure owns concrete URLs and publication mechanics for delivering
|
||||
the links through the selected channel. Do not maintain duplicate notice lists.
|
||||
These conventions do not by themselves require separate downloadable license
|
||||
assets, source bundles, or a different binary package format.
|
||||
|
||||
## Maintenance And Exceptions
|
||||
|
||||
Update project licensing statements together when the author changes the grant
|
||||
or its scope. Update upstream notices in the change that adds, removes, upgrades,
|
||||
copies, or changes the distribution of a relevant component. Check the selected source's license
|
||||
and notices rather than assuming they are unchanged. Update affected entries,
|
||||
commit links, and texts; retain historical versions through Git.
|
||||
|
||||
Check affected statements, faithful standard-text copies, and link destinations
|
||||
as part of ordinary change review. For a future release tag, check intended file
|
||||
paths against the candidate source; the tag need not resolve remotely before
|
||||
publication. Do not add publication-time network checks, CI polling, recurring
|
||||
legal sign-offs, mandatory scanners, or a full software bill of materials merely
|
||||
to implement this policy.
|
||||
|
||||
Use the House Format routinely. Raise an exception only for a concrete obligation
|
||||
or unresolved rights question it does not address, including any arising from a
|
||||
particular distribution channel. Record the issue and selected treatment in the
|
||||
adopted OSS policy or an appropriate decision record. Attribution entries
|
||||
contain the resulting factual identification and required material, without
|
||||
repeating that reasoning. Resolve the issue before the affected use or
|
||||
distribution without reopening settled choices.
|
||||
Common ownership of projects is neither required nor a blanket exception.
|
||||
|
||||
Apply the formatting convention prospectively. Do not rewrite published tags,
|
||||
notes, binaries, or checksums to retrofit it. Address actual omissions deliberately
|
||||
under the applicable terms and the [release correction policy](release.md#immutability-and-failure-recovery).
|
||||
Ordinary attribution maintenance and release links are preparation work, not new
|
||||
release approval gates.
|
||||
|
||||
## Promptkit Adoption
|
||||
|
||||
Promptkit retains its LGPL-3.0-or-later library grant in
|
||||
[COPYRIGHT](../../COPYRIGHT). Adapted policy documentation has the separate
|
||||
scope identified there and in [attribution](../oss/attribution.md).
|
||||
|
||||
Promptkit publishes source-only Go module tags on Gitea, with no binaries or
|
||||
separate hosted download page. The [release procedure](../release.md) owns
|
||||
publication and concrete footer URLs. The hosted-page guidance applies if such
|
||||
a channel is deliberately introduced; it does not require creating one.
|
||||
252
docs/policy/release.md
Normal file
252
docs/policy/release.md
Normal file
@@ -0,0 +1,252 @@
|
||||
# Release Policy
|
||||
|
||||
**Base Policy Version:** 1.4
|
||||
|
||||
**Last Updated**: 2026-09-27
|
||||
|
||||
**Upstream Source:** [release.md](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/docs/policy/release.md)
|
||||
|
||||
## Purpose
|
||||
|
||||
This policy defines a proportionate workflow for validating and publishing an
|
||||
intended source revision under a fresh version. Releases should require little
|
||||
routine intervention, avoid repeated work, and preserve published contents.
|
||||
|
||||
The repository-specific commands belong in `docs/release.md`. Historical release
|
||||
summaries belong in `docs/releases/`. The procedure must agree with the adopted
|
||||
policy and link to it rather than introduce additional release requirements.
|
||||
|
||||
## Release Model And Completion
|
||||
|
||||
A source release identifies one exact committed source revision through an
|
||||
immutable version tag and a version-matched, checked-in release note. The
|
||||
primary release branch is `main` unless the adopted policy specifies otherwise.
|
||||
The procedure must name the authoritative publication remote.
|
||||
|
||||
A successful push of the intended tag, with its source commit available on the
|
||||
authoritative primary branch, completes the release. The source commit may be
|
||||
pushed before the tag or published together with it through an atomic push.
|
||||
The procedure must choose and document the supported approach.
|
||||
|
||||
Release commands must finish at successful publication. They must not wait for
|
||||
or poll CI, require successful CI status, inspect downloadable artifacts, or
|
||||
check consumer installation or Go module availability as completion gates.
|
||||
A failed or ambiguous push requires resolving the authoritative remote state
|
||||
before declaring success or retrying; successful pushes do not require a
|
||||
separate remote read-back gate.
|
||||
|
||||
Repositories may publish source only. Additional binaries, images, packages,
|
||||
checksums, or other declared deliverables may be published asynchronously from
|
||||
the tagged source. The procedure must identify those deliverables and the
|
||||
responsibility of their automation. Published deliverables must be traceable
|
||||
to that source and identify the same version.
|
||||
|
||||
Track source-release completion and artifact delivery separately. Report delivery
|
||||
failures and optional investigations under artifact-delivery status.
|
||||
|
||||
## Versioning And Compatibility
|
||||
|
||||
Stable releases use Semantic Versioning in the form `vMAJOR.MINOR.PATCH`, with
|
||||
no leading zeroes. Prereleases require an explicit adopted policy defining
|
||||
their format and use.
|
||||
|
||||
The selected version must be unused locally and on the authoritative remote.
|
||||
Do not require queries to every configured remote or publication service.
|
||||
A version known to have already been published must not be reused, even if the
|
||||
normal guards do not inspect that channel. Adding a distribution channel
|
||||
requires a deliberate update to the policy and procedure where needed.
|
||||
|
||||
For releases at or after `v1.0.0`:
|
||||
|
||||
- a major release may make intentional incompatible changes;
|
||||
- a minor release may add backward-compatible behavior; and
|
||||
- a patch release may make backward-compatible fixes.
|
||||
|
||||
Before `v1.0.0`, a minor release may change a documented public, operational,
|
||||
configuration, data, integration, or compatibility contract when the release
|
||||
note explains the impact and required consumer action. A patch release must
|
||||
not intentionally break those contracts within its minor line.
|
||||
|
||||
Additional compatibility promises and deliberate exceptions belong in the
|
||||
adopted policy or canonical contract documentation. Release notes explain
|
||||
changes; they do not redefine current compatibility contracts.
|
||||
|
||||
## Release Documentation
|
||||
|
||||
Every new release must include `docs/releases/<version>.md` in the tagged
|
||||
source tree. Use the complete version tag as the filename and begin with a
|
||||
title containing the product name and that exact version.
|
||||
|
||||
Include a short summary and any relevant compatibility impact or upgrade
|
||||
instructions. Use the Promptkit section structure below and keep the note
|
||||
proportionate to the change.
|
||||
|
||||
Keep exact current contracts in their canonical documents and update affected
|
||||
documentation with the behavior change. Link to those documents from the note
|
||||
where useful. If annotated tags are used, derive their summary from the
|
||||
checked-in note.
|
||||
|
||||
Include the release footer defined by the [OSS policy](oss.md#discoverability)
|
||||
in each new release document and preserve it on the hosted download page. The
|
||||
release procedure defines the concrete URLs and publication mechanics. Prepare
|
||||
these links with the release note; they do not add a network check or approval
|
||||
gate.
|
||||
|
||||
Handle version selection, release-note content, and retirement of completed
|
||||
roadmaps during preparation. Preserve planned and deferred work and repair
|
||||
affected links when retiring roadmaps. Reuse ordinary change review for
|
||||
unintended files, credentials, and private data. Scope release-content review
|
||||
to the tagged source.
|
||||
|
||||
## Candidate Validation
|
||||
|
||||
Provide one publication command that validates the final committed candidate,
|
||||
checks publication state, and publishes the intended refs. A validation-only
|
||||
mode or command may be provided for diagnosis, but must not be a required step
|
||||
before publication repeats the same checks.
|
||||
|
||||
Run the complete deterministic, offline default test suite once for the final
|
||||
candidate, following the [testing policy](testing.md). For repositories that
|
||||
publish builds, verify the required build targets. Select those targets for
|
||||
the application's actual support commitments; native testing on each platform
|
||||
and access to particular hardware are not default release prerequisites.
|
||||
|
||||
Additional gates must address a concrete risk that the existing checks do not
|
||||
adequately cover. Do not automatically require linting, race testing, module
|
||||
hygiene audits, documentation audits, credential scans, example validation, or
|
||||
expanded platform matrices solely because they are possible. Retain such
|
||||
checks when their distinct protection justifies their cost. A one-time migration
|
||||
check must not silently become a permanent release requirement.
|
||||
|
||||
Release tests must not require real credentials, mutable application services,
|
||||
paid APIs, provider probes, or inference calls. The procedure may describe dependency
|
||||
preparation separately from offline validation.
|
||||
|
||||
Avoid duplicate validation across release phases. Successful checks for the
|
||||
same final candidate need not be repeated solely to satisfy another phase;
|
||||
any reuse must clearly apply to that exact candidate. Persistent validation
|
||||
caches are not required. If the candidate changes, validate the new candidate.
|
||||
Cheap publication-state checks may be repeated immediately before pushing
|
||||
because local or remote state may have changed during validation.
|
||||
|
||||
Automate mechanical guards. Human review gates are strongly disfavored: add one
|
||||
only through an explicit project-specific policy decision explaining the
|
||||
concrete risk and why ordinary change review or automation is insufficient.
|
||||
Do not add routine confirmations for checks the command can perform itself.
|
||||
|
||||
## Publication Guards
|
||||
|
||||
Automate the checks needed to publish the intended source under a fresh version:
|
||||
|
||||
- the version has the adopted format and is unused locally and on the
|
||||
authoritative remote;
|
||||
- the worktree and index are clean on the primary release branch;
|
||||
- the source being published is the exact candidate that passed validation,
|
||||
including its matching release note;
|
||||
- the remote primary branch already identifies that candidate, or an atomic
|
||||
source/tag push will advance it to that candidate without discarding remote
|
||||
history; and
|
||||
- the tag identifies the validated candidate and only the intended refs are
|
||||
published.
|
||||
|
||||
Fetch the authoritative branch and tag state as needed for these checks.
|
||||
Do not force branch or tag updates to bypass a failed guard. Publish explicit
|
||||
refs rather than all local tags. Do not require CI success, publication-service
|
||||
availability checks, or a separate manual confirmation for each guard.
|
||||
|
||||
## Immutability And Failure Recovery
|
||||
|
||||
Published version tags and published contents are immutable. Do not move,
|
||||
overwrite, recreate, or delete a published tag, or replace a published artifact,
|
||||
checksum, signature, provenance statement, or release note under that version.
|
||||
The checked-in note at the immutable tag is that release's historical record.
|
||||
|
||||
If validation fails before publication, correct and commit the candidate and
|
||||
rerun the publication command. If a push fails or its outcome is ambiguous,
|
||||
inspect the authoritative remote before retrying. Remove an unpublished local
|
||||
tag only after confirming it was never published or used by a release channel.
|
||||
An unavailable remote leaves publication status unknown; do not overwrite a tag
|
||||
to resolve uncertainty.
|
||||
|
||||
Transient network, authentication, CI, or consumer-availability failures do not
|
||||
by themselves require a new version. Asynchronous delivery may be retried from
|
||||
the same immutable tag. Missing artifacts may be published later, provided
|
||||
existing published contents are not replaced and the additions remain consistent
|
||||
with the tagged source and any already-published checksum manifest. If that
|
||||
cannot be achieved, publish a new version.
|
||||
|
||||
A defect requiring changes to published source or contents requires a corrected
|
||||
commit, a new semantically appropriate version, a matching release note, and
|
||||
the release procedure again. Leave the original release intact. A channel's
|
||||
yank, deprecation, or advisory mechanism may warn consumers away from a defective
|
||||
release without replacing its contents or reusing its version.
|
||||
|
||||
## Repository Release Procedure
|
||||
|
||||
Maintain `docs/release.md` as a short, executable maintainer runbook. It must
|
||||
identify:
|
||||
|
||||
1. prerequisites, primary branch, and authoritative remote;
|
||||
2. version and release-note preparation;
|
||||
3. one publication command with its selected local checks and automated guards;
|
||||
4. declared deliverables, required build targets, and automation responsibilities;
|
||||
5. the successful-push completion point and separately reported delivery status;
|
||||
and
|
||||
6. failure handling, ambiguous-push recovery, and asynchronous delivery retries.
|
||||
|
||||
Use copyable commands where practical. Keep optional diagnostics and consumer
|
||||
verification clearly optional. The procedure must not turn preparation work,
|
||||
ordinary change review, or post-publication investigation into additional
|
||||
release gates.
|
||||
|
||||
## Promptkit Release Configuration
|
||||
|
||||
The release-note requirements were adopted on 2026-09-23 and licensing
|
||||
navigation on 2026-09-26. Apply them prospectively; previously published notes
|
||||
and tags retain their original records.
|
||||
|
||||
Promptkit publishes source-only Go modules from main to origin on Gitea, using
|
||||
annotated stable SemVer tags. It has no binaries, hosted download pages, or
|
||||
asynchronous deliverables. The source commit is pushed through main before the
|
||||
tag. The [release procedure](../release.md) owns commands and recovery.
|
||||
|
||||
The existing publication script requires Summary, Compatibility, Upgrade, and
|
||||
Changes headings beneath a title of `# Promptkit vMAJOR.MINOR.PATCH`. State
|
||||
“No action required” or “None” where appropriate. This retained local structure
|
||||
matches the automated release-note guard; changing it is separate work.
|
||||
|
||||
The [publication script](../../scripts/release.sh) runs the
|
||||
[local checks](../../scripts/release-check.sh) once for the final committed
|
||||
candidate: race-enabled tests, vet, three offline consumer examples, tracked
|
||||
Go formatting, Markdown links, module/workspace hygiene, whitespace, and a
|
||||
credential-pattern scan. These protect concurrency, assembled library use,
|
||||
source distribution, and documentation integrity. A separate ordinary test run
|
||||
or standalone build is not an additional release gate. Candidate, tag, and
|
||||
synchronized-main checks remain automated. No CI or module-availability wait
|
||||
is required. Licensing footer URLs are preparation work, not a network gate.
|
||||
|
||||
## Catalog Profile Compatibility
|
||||
|
||||
External catalog profile data may be added, removed, renamed, or modified
|
||||
without deprecation or advance notice. This exception is limited to profile
|
||||
data imported from the maintained catalog modules. It does not weaken ordinary
|
||||
compatibility for Promptkit's public Go API, configuration and format semantics,
|
||||
runtime behavior, catalog-module APIs, manifest schema, backend definitions, or
|
||||
consumer-owned definitions.
|
||||
|
||||
Within the catalog versions selected and tested by a Promptkit release, and
|
||||
unless a consumer overrides the name or its base, Promptkit keeps the literal
|
||||
profile IDs `promptkit-openrouter-standard` and
|
||||
`promptkit-openrouter-auto` available. The names do not promise a permanent
|
||||
model, backend, routing behavior, settings, cost, output, or upstream
|
||||
availability. The [framework format reference](../formats.md#built-in-profile-catalog)
|
||||
owns their current definitions.
|
||||
|
||||
`v0.10.0` is the initial minor release for this compatibility transition.
|
||||
Afterward, catalog-only updates and adjustments to the current selections may
|
||||
ship in a patch release when all other compatibility obligations remain intact.
|
||||
External profile removals can require consumer action; they are not automatic
|
||||
migrations.
|
||||
|
||||
Catalog adoption and release validation remain local and offline. They do not
|
||||
probe provider endpoints, request model lists, or make inference calls.
|
||||
@@ -1,39 +1,32 @@
|
||||
# Testing Policy
|
||||
|
||||
**Base Policy Version:** 1.3
|
||||
|
||||
**Last Updated**: 2026-09-27
|
||||
|
||||
**Upstream Source:** [testing.md](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/docs/policy/testing.md)
|
||||
|
||||
## Purpose
|
||||
|
||||
Our tests exist to make **incorrect changes expensive and correct changes cheap**.
|
||||
|
||||
We do not optimize for test count, line coverage, exhaustive isolation, or the fewest possible tests. We optimize for sufficient confidence in important behavior while imposing as little unnecessary friction as possible on future development.
|
||||
We optimize for sufficient confidence in important behavior while minimizing friction on future development.
|
||||
|
||||
## Every test has a cost
|
||||
|
||||
Testing is not an unqualified good. Every test imposes both an immediate cost and a continuing lifetime cost.
|
||||
|
||||
A test must be:
|
||||
|
||||
- written and reviewed;
|
||||
- understood by future maintainers and coding agents;
|
||||
- executed in maintainer-run validation;
|
||||
- diagnosed when it fails;
|
||||
- updated when legitimate behavior changes;
|
||||
- maintained as fixtures, APIs, and dependencies evolve; and
|
||||
- removed or rewritten when it becomes redundant, brittle, misleading, or obsolete.
|
||||
|
||||
Tests also create cognitive and architectural friction. They can constrain refactoring, duplicate policy, slow feedback loops, add noise to failures, and cause harmless implementation changes to require unrelated edits across the suite.
|
||||
Every test has an immediate and continuing cost: writing, review, execution,
|
||||
diagnosis, fixture maintenance, and updates when legitimate behavior changes.
|
||||
Tests can also constrain refactoring, duplicate policy, and add failure noise.
|
||||
|
||||
A test is warranted only when the confidence it provides justifies these costs.
|
||||
|
||||
Apply this cost-benefit analysis at two levels:
|
||||
Evaluate that tradeoff at two levels:
|
||||
|
||||
1. **Per test:** What realistic defect does this test detect, how consequential would that defect be, and is that protection worth the test's lifetime cost?
|
||||
2. **Across the suite:** Does this collection provide materially more confidence than a smaller, simpler suite would?
|
||||
|
||||
The preferred test suite is a **lean suite that provides sufficient confidence in the risks that matter, without redundant or low-value tests**. We seek sufficient confidence with the least unnecessary testing friction, not the fewest possible tests.
|
||||
Some friction is intentional. Tests should make dangerous changes—such as breaking compatibility, corrupting data, violating security boundaries, or reintroducing subtle bugs—require deliberate review. Keep ordinary internal changes inexpensive.
|
||||
|
||||
Some friction is intentional. Tests should make dangerous changes—such as breaking compatibility, corrupting data, violating security boundaries, or reintroducing subtle bugs—require deliberate review. They should not make ordinary internal changes needlessly expensive.
|
||||
|
||||
The cost of a test is not a reason to omit testing by default. Do not cite maintenance cost abstractly. When omitting a plausible test, be able to state why the protected failure is low-risk, already covered, obvious, reversible, or cheaper to detect elsewhere. For consequential, subtle, or difficult-to-observe behavior, the presumption should favor testing.
|
||||
Evaluate maintenance cost against the specific failure being protected. When omitting a plausible test, be able to state why the protected failure is low-risk, already covered, obvious, reversible, or cheaper to detect elsewhere. For consequential, subtle, or difficult-to-observe behavior, the presumption should favor testing.
|
||||
|
||||
## Default testing style
|
||||
|
||||
@@ -47,21 +40,34 @@ Use a **classical/Detroit-style** approach:
|
||||
|
||||
Examples of appropriate seams include clocks, randomness, subprocesses, remote APIs, object storage, email, and paid LLM calls.
|
||||
|
||||
## Promptkit Test Boundaries
|
||||
|
||||
Promptkit is a library without a CLI or hosted CI. The execution requirements,
|
||||
facade test ownership, public error guidance, and offline consumer examples
|
||||
below tailor the standard to that boundary.
|
||||
|
||||
## Test execution requirements
|
||||
|
||||
Promptkit currently uses maintainer-run validation rather than hosted CI.
|
||||
Maintainers run the complete local workflow in the
|
||||
[development guide](../development.md#maintainer-validation) before accepting
|
||||
changes. That guide is the canonical owner of exact commands, formatting,
|
||||
documentation-link validation, and repository-hygiene checks.
|
||||
Ordinary work requires validation proportionate to its scope and risk. Do not
|
||||
repeat successful checks for unchanged content merely to create a commit.
|
||||
Release candidates require the complete local workflow in the
|
||||
[development guide](../development.md#maintainer-validation). That guide routes
|
||||
contributors to the executable checks for formatting, documentation links,
|
||||
and repository hygiene.
|
||||
Introducing hosted CI later would supplement, not silently redefine, this
|
||||
documented validation model.
|
||||
|
||||
Maintainer validation must include ordinary and race-enabled package tests,
|
||||
static analysis, a complete build, and execution of both maintained offline
|
||||
consumer examples. The preparation example protects assembled preparation and
|
||||
Full release-candidate validation includes race-enabled package tests, explicit
|
||||
static analysis, and execution with automated semantic checks of the three maintained
|
||||
offline consumer examples. The test run builds the packages; the examples
|
||||
also exercise ordinary compilation. Separate ordinary tests and a standalone
|
||||
build are not additional release gates. Ordinary tests remain useful during
|
||||
development. The preparation example protects assembled preparation and
|
||||
inspection behavior. The execution example separately protects assembled
|
||||
`Run`, injected-client, validation, usage, and result behavior.
|
||||
`Run`, injected-client, validation, usage, and result behavior. The observation
|
||||
example protects assembled built-in retries, structured proposal and assessment
|
||||
runs, and application-owned interpretation of metadata.
|
||||
|
||||
Tests in the default suite must be deterministic, offline, and independent of
|
||||
real credentials. They must not invoke paid APIs, use live network
|
||||
@@ -87,7 +93,7 @@ Use each test type where it protects a distinct risk:
|
||||
interaction, while replacing live or nondeterministic external boundaries.
|
||||
- External-package root tests exercise the public facade as a Go consumer,
|
||||
while internal package tests own focused implementation behavior.
|
||||
- The maintained offline preparation and execution examples protect distinct
|
||||
- The maintained offline preparation, execution, and observation examples protect distinct
|
||||
representative assembled consumer workflows without contacting a model
|
||||
provider.
|
||||
- Fixtures should be minimal, synthetic, versioned with the behavior they
|
||||
@@ -99,7 +105,7 @@ Use each test type where it protects a distinct risk:
|
||||
|
||||
## What deserves tests
|
||||
|
||||
Prioritize tests for:
|
||||
To the extent the following features are present in the application, prioritize tests for:
|
||||
|
||||
1. Public and package-level contracts.
|
||||
2. Domain rules and important invariants.
|
||||
@@ -107,17 +113,29 @@ Prioritize tests for:
|
||||
4. Failure handling, cancellation, retries, recovery, and partial success.
|
||||
5. Serialization, schemas, compatibility, and round trips.
|
||||
6. Previously observed or plausible regressions.
|
||||
7. Representative integration and consumer workflows.
|
||||
7. Representative integration and end-to-end workflows.
|
||||
|
||||
A package-level contract is behavior relied upon by another package or major collaborator, not every observable detail of a package implementation.
|
||||
A package-level contract comprises behavior relied upon by another package or major collaborator.
|
||||
|
||||
For behavior involving **data integrity, destructive operations, compatibility, security, concurrency, idempotency, or recovery**, presume that durable tests are required unless the behavior is already credibly protected at another layer.
|
||||
Choose tests for requested behavior and established contracts. Tests must not
|
||||
introduce new restrictions, guarantees, or threat scenarios merely because they
|
||||
could be asserted. Follow the [agent policy](agents.md) when deciding whether a
|
||||
behavior belongs to the contract; current implementation behavior alone is not
|
||||
sufficient authority.
|
||||
|
||||
For established behavior involving **data integrity, destructive operations,
|
||||
compatibility, security, concurrency, idempotency, or recovery**, favor durable
|
||||
tests for realistic failures whose consequences warrant protection. Identify
|
||||
the failure and the existing contract it would violate. These categories call
|
||||
for attention, not an automatic test inventory or permission to strengthen the
|
||||
application's guarantees. Account for protection already provided at other layers.
|
||||
|
||||
Do not add tests merely because a function, branch, or line exists. Do not add a test when the same meaningful risk is already adequately protected elsewhere.
|
||||
|
||||
## Choose the right test boundary
|
||||
|
||||
Test through the narrowest stable boundary that expresses the behavior clearly.
|
||||
Choose the narrowest stable boundary that expresses the behavior clearly and
|
||||
provides durable confidence with minimal incidental coupling.
|
||||
|
||||
This is often the package API, but it may instead be:
|
||||
|
||||
@@ -125,30 +143,15 @@ This is often the package API, but it may instead be:
|
||||
- a package-level operation when several internal collaborators jointly produce the behavior; or
|
||||
- a larger integration boundary when correctness emerges from interaction with a real dependency.
|
||||
|
||||
Do not force all behavior through oversized consumer-workflow tests. Do not
|
||||
test every private helper merely because it exists. Choose the boundary that
|
||||
gives durable confidence with the least incidental coupling.
|
||||
|
||||
## Test behavior, not implementation
|
||||
|
||||
A test should protect a decision, contract, or invariant—not memorialize the current implementation.
|
||||
A test should protect a selected decision, contract, or invariant.
|
||||
|
||||
Before adding or retaining a test, ask:
|
||||
A test is suspect when it primarily detects changes to private constants or
|
||||
helpers, equivalent operation ordering, incidental formatting, equivalent
|
||||
algorithms, or internal structure that preserves behavior.
|
||||
|
||||
> What realistic defect would this test catch?
|
||||
|
||||
A test is suspect when its main purpose is to detect that someone:
|
||||
|
||||
- changed an internal constant;
|
||||
- renamed or split a private helper;
|
||||
- reordered equivalent internal operations;
|
||||
- changed incidental formatting;
|
||||
- replaced one correct algorithm with another; or
|
||||
- refactored internal object structure without changing behavior.
|
||||
|
||||
Refactoring should normally require no test edits unless the refactored structure is itself part of the contract.
|
||||
|
||||
A test can be factually correct and still have negative value. Accurately describing current behavior is not enough; the protected behavior must be important enough to justify the future friction.
|
||||
Behavioral tests should normally remain stable through internal refactoring that preserves the contract.
|
||||
|
||||
## Expected effects of different changes
|
||||
|
||||
@@ -165,13 +168,13 @@ A test failing is not the same as a test needing to be edited. Many tests may co
|
||||
|
||||
## Separate mechanism from policy
|
||||
|
||||
Configurable thresholds and defaults must not be duplicated throughout the test suite.
|
||||
Derive expectations from test-controlled configuration and relationships.
|
||||
|
||||
The following fragments are illustrative rather than standalone Go programs.
|
||||
Do not encode an internal concurrency limit indirectly:
|
||||
For this example, assume the selected contract rejects a request to start more
|
||||
processes than a configured limit. Do not encode its production default indirectly:
|
||||
|
||||
```go
|
||||
// Production policy:
|
||||
// Production default:
|
||||
const maxConcurrency = 4
|
||||
|
||||
// Brittle test:
|
||||
@@ -189,37 +192,40 @@ require.NoError(t, runner.Start(limit))
|
||||
require.ErrorIs(t, runner.Start(limit+1), ErrTooMuchConcurrency)
|
||||
```
|
||||
|
||||
The test should prove:
|
||||
For this selected contract, the test should prove:
|
||||
|
||||
- the configured limit is accepted; and
|
||||
- one beyond the configured limit is rejected.
|
||||
|
||||
Rejection is not a general requirement for concurrency control. If the selected
|
||||
contract queues excess work or limits active workers while accepting larger
|
||||
batches, test that behavior instead. Do not add rejection or a configurable
|
||||
limit solely to reproduce this example.
|
||||
|
||||
The production default should be tested exactly only when its literal value is itself a public, operational, safety, protocol, or compatibility requirement.
|
||||
|
||||
Apply the same rule to limits, timeouts, capacities, retry counts, and ranges: test relationships and behavior, not duplicated literals.
|
||||
Apply the same rule to limits, timeouts, capacities, retry counts, and ranges: test relationships and behavior.
|
||||
|
||||
For concurrency limits, test both kinds of behavior when relevant:
|
||||
For established concurrency limits, distinguish these behaviors where the contract requires them:
|
||||
|
||||
1. **Configuration enforcement:** invalid or excessive requested values are handled correctly.
|
||||
1. **Configuration handling:** values are accepted or rejected according to the selected contract.
|
||||
2. **Runtime enforcement:** observed peak concurrency never exceeds the configured limit.
|
||||
|
||||
Use a test-controlled limit and measure the behavior relative to that limit. Do not merely assert today's default value.
|
||||
Use a test-controlled limit and measure the behavior relative to that limit.
|
||||
|
||||
## Avoid semantic duplication across layers
|
||||
## Assign each behavior a test owner
|
||||
|
||||
Each behavior should have a clear test owner.
|
||||
Where these layers exist, assign ownership as follows:
|
||||
|
||||
- Parser tests own parsing cases.
|
||||
- Validator tests own validation rules.
|
||||
- Domain tests own transformations and invariants.
|
||||
- Boundary tests own external integration behavior.
|
||||
- Adapter tests own external integration behavior.
|
||||
- Orchestrator tests own coordination and failure propagation.
|
||||
- Consumer-workflow tests prove that representative assembled library use
|
||||
works.
|
||||
- External-package root tests own the public Go facade contract.
|
||||
- Consumer-workflow tests prove that representative assembled library use works.
|
||||
|
||||
Higher-level tests should not repeat every lower-level case. A single intentional policy change should not require unrelated edits across many test files.
|
||||
|
||||
Tests that are individually reasonable may still be collectively redundant. Evaluate the marginal value of each additional test in light of the protection already provided by the rest of the suite.
|
||||
Give each layer its distinct responsibility and keep expectation updates localized to the contract being changed.
|
||||
|
||||
## Use test doubles deliberately
|
||||
|
||||
@@ -240,111 +246,78 @@ Mocks are appropriate when the contract includes facts such as:
|
||||
- an expensive API is called no more than once; or
|
||||
- a security audit event is emitted.
|
||||
|
||||
Do not use mocks merely to isolate every object or reproduce the implementation's call graph.
|
||||
|
||||
## Go-specific guidance
|
||||
|
||||
Use:
|
||||
Select techniques by behavior and risk:
|
||||
|
||||
- table-driven tests for meaningful behavioral categories and boundaries;
|
||||
- `t.TempDir()` for real filesystem behavior;
|
||||
- `httptest.Server` for realistic HTTP interactions;
|
||||
- fuzz tests for parsers, normalization, path handling, and broad input spaces;
|
||||
- fuzz tests when exploring broad input spaces, such as parsers or path handling, adds useful protection beyond representative cases;
|
||||
- golden files only when the complete output is intentionally stable;
|
||||
- integration tests where correctness depends on component interaction; and
|
||||
- a small number of representative consumer-workflow tests.
|
||||
- a small number of representative end-to-end tests when assembled workflows expose meaningful risks not adequately covered at narrower boundaries.
|
||||
|
||||
Avoid exact error-string assertions unless the wording is itself contractual. Prefer `errors.Is`, `errors.As`, typed errors, or structured error fields.
|
||||
Prefer `errors.Is`, `errors.As`, typed errors, or structured error fields. Use exact error-wording assertions only when the wording is contractual.
|
||||
|
||||
At public API boundaries, prefer stable error identity, structured values, and
|
||||
the smallest semantic fragment needed to identify the failure. Do not snapshot
|
||||
complete diagnostic wording unless it is contractual.
|
||||
the smallest semantic fragment needed to identify the failure. Use complete
|
||||
diagnostic snapshots only when the wording is contractual.
|
||||
|
||||
Golden-file updates must require an explicit local flag. Ordinary validation
|
||||
runs must never update golden files automatically, and maintainers must inspect
|
||||
the semantic diff before accepting an update.
|
||||
Golden-file updates must require an explicit local flag. Ordinary validation runs must never update golden files automatically, and reviewers must inspect the semantic diff before accepting an update.
|
||||
|
||||
Keep tests readable and direct. Test helpers and fixture frameworks must earn their own maintenance cost; do not build elaborate test infrastructure for small or isolated needs.
|
||||
Keep tests readable and direct. Size test helpers and fixture frameworks to the maintenance benefit they provide.
|
||||
|
||||
## Coverage
|
||||
|
||||
Coverage is a diagnostic, not a target.
|
||||
Use coverage to locate untested critical branches and unexpectedly weak packages. Evaluate test quality through meaningful behavioral protection and choose coverage according to risk.
|
||||
|
||||
Use it to find untested critical branches and unexpectedly weak packages. Do not write low-value tests solely to increase a percentage, and do not infer test quality from coverage alone.
|
||||
|
||||
Pure domain logic will often warrant higher coverage than facade wiring or
|
||||
external adapters. Uneven coverage is acceptable when it reflects risk.
|
||||
|
||||
Increasing coverage is valuable only when the newly covered behavior protects a meaningful risk at an acceptable cost.
|
||||
Pure domain logic will often warrant higher coverage than facade wiring or external adapters. Uneven coverage is acceptable when it reflects risk.
|
||||
|
||||
## Regression tests
|
||||
|
||||
A bug fix should normally include a regression test that fails before the fix and passes afterward.
|
||||
|
||||
Retain the test when the defect could realistically recur and its consequences justify the ongoing cost. Prefer the narrowest durable test of the violated contract or invariant; do not preserve accidental implementation details from the original bug.
|
||||
Retain the test when the defect could realistically recur and its consequences justify the ongoing cost. Prefer the narrowest durable test of the violated contract or invariant.
|
||||
|
||||
Not every historical bug requires a permanent test. If the underlying design has made recurrence impossible, the test has become redundant, or a stronger invariant test now subsumes it, remove or consolidate it.
|
||||
Remove or consolidate a test when the underlying design has made recurrence impossible, its protection is redundant, or a stronger invariant test subsumes it.
|
||||
|
||||
## Deleting or rewriting tests
|
||||
|
||||
Tests are maintained code, not permanent historical artifacts.
|
||||
|
||||
Delete or rewrite a test when its maintenance cost exceeds the confidence it provides.
|
||||
Delete or rewrite tests when maintenance cost exceeds their continuing
|
||||
protective value.
|
||||
|
||||
Strong candidates include tests that:
|
||||
|
||||
- require updates after harmless internal changes;
|
||||
- directly assert private constants without protecting a real contract;
|
||||
- duplicate the same policy across several layers;
|
||||
- assert incidental private constants or snapshot large amounts of incidental output;
|
||||
- verify mock choreography rather than outcomes;
|
||||
- snapshot large amounts of incidental output;
|
||||
- test trivial private helpers already exercised through stable package behavior;
|
||||
- protect risks already covered more effectively elsewhere;
|
||||
- duplicate policy across layers, retest trivial helpers covered by package
|
||||
behavior, or protect risks covered more effectively elsewhere;
|
||||
- are flaky, misleading, obsolete, or disproportionately expensive to diagnose; or
|
||||
- no longer correspond to a plausible failure mode.
|
||||
|
||||
Several brittle tests may encode one genuine requirement. Replace them with one durable behavior-level or invariant test rather than preserving all of them.
|
||||
|
||||
Deleting a low-value test can improve the quality of the suite by reducing noise, maintenance burden, and friction around legitimate change.
|
||||
Several brittle tests may encode one genuine requirement. Consolidate them into one durable behavior-level or invariant test.
|
||||
|
||||
## Reviewing a proposed test
|
||||
|
||||
Use the following questions when the value, boundary, or durability of a proposed test is not self-evident. Significant test additions should be reviewable against them, but written answers are not required for every routine test.
|
||||
Use the following questions when a proposed test needs closer evaluation. Significant test additions should be reviewable against them; written answers are optional for routine tests.
|
||||
|
||||
1. What realistic defect would it catch?
|
||||
2. How likely is that defect?
|
||||
3. How consequential would it be?
|
||||
4. Is the behavior already protected elsewhere?
|
||||
5. At which layer should this behavior be owned?
|
||||
6. Does the test assert a durable contract or an incidental implementation detail?
|
||||
7. Could the implementation be refactored without changing the behavior and without editing this test?
|
||||
8. What should cause this test to fail?
|
||||
9. What legitimate changes should not cause this test to fail?
|
||||
10. What ongoing maintenance, execution, and diagnostic cost will the test impose?
|
||||
11. Is there a smaller or more direct test that protects the same risk?
|
||||
|
||||
Do not add the test when its expected lifetime cost exceeds its expected protective value.
|
||||
|
||||
When deciding not to test plausible behavior, record or be able to explain why the risk is low, already protected, obvious, reversible, or cheaper to detect elsewhere.
|
||||
1. Which requested behavior or established contract does it protect, and what realistic defect would violate it?
|
||||
2. How likely and consequential is that defect, and is it already protected elsewhere?
|
||||
3. What is the narrowest stable boundary that can test it clearly?
|
||||
4. Would a correct internal refactor or a change to an incidental default require test edits?
|
||||
5. Does its confidence justify its maintenance, execution, and diagnostic cost, or would a smaller test suffice?
|
||||
|
||||
## Definition of sufficient
|
||||
|
||||
A test suite is sufficient when:
|
||||
|
||||
- important contracts and invariants are protected;
|
||||
- meaningful boundaries and failure modes are exercised;
|
||||
- realistic and consequential regressions are credibly protected against silent recurrence;
|
||||
- behavior involving data integrity, destructive operations, compatibility, security, concurrency, idempotency, and recovery is credibly protected;
|
||||
- important external boundaries have realistic integration coverage;
|
||||
- representative complete workflows are tested;
|
||||
- failures provide useful signal rather than redundant noise;
|
||||
- legitimate internal changes usually do not require test edits; and
|
||||
- important requested behavior and established contracts are protected against realistic, consequential failures;
|
||||
- boundaries and component interactions have coverage where they introduce meaningful risks not protected elsewhere;
|
||||
- failures provide useful, distinct signals;
|
||||
- tests usually remain stable through legitimate internal changes; and
|
||||
- additional tests would mostly repeat existing protection or preserve inconsequential implementation details.
|
||||
|
||||
Sufficiency is a risk judgment, not a coverage percentage or test count.
|
||||
Reassess it as the library, its consumers, and the consequences of failure
|
||||
evolve.
|
||||
|
||||
The governing rule is:
|
||||
|
||||
> Test heavily where failure is consequential, subtle, or difficult to detect after the fact. Test lightly where failure is obvious, reversible, and inexpensive—and retain no test whose lifetime cost exceeds the confidence it provides.
|
||||
Reassess sufficiency as the application, its users, and the consequences of
|
||||
failure evolve.
|
||||
|
||||
295
docs/release.md
295
docs/release.md
@@ -1,242 +1,97 @@
|
||||
# Release Procedure
|
||||
|
||||
## Release Model
|
||||
This runbook implements the [release policy](policy/release.md). A successful
|
||||
push of the selected annotated tag to origin completes the release. Never wait
|
||||
for CI to finish or succeed.
|
||||
|
||||
Promptkit publishes a Go library through source commits and semantic Go module
|
||||
tags. It does not publish runnable binaries or binary packages and does not
|
||||
currently use hosted CI. The release maintainer performs and records the
|
||||
required validation.
|
||||
## Prepare The Candidate
|
||||
|
||||
`v0.1.0` is the initial published release. Later releases use semantic
|
||||
`vMAJOR.MINOR.PATCH` tags. Before `v1`, minor releases may change the public
|
||||
API and patch releases preserve compatibility within their minor line. Every
|
||||
pre-`v1` release note must summarize compatibility, identify public API
|
||||
changes, and state any action required of consumers.
|
||||
Required tools are Git, Go with race-detector support, POSIX shell, and Python
|
||||
3. Publication requires push access to origin. Work from the repository root.
|
||||
|
||||
Promptkit releases are source-only. The annotated tag message is the release
|
||||
note; there is no separate hosted release or binary packaging step.
|
||||
Select an unused version and create `docs/releases/<version>.md` using the
|
||||
[required structure](policy/release.md#release-documentation). Describe
|
||||
compatibility impact and consumer actions and update affected canonical docs.
|
||||
For catalog updates, record selected catalog versions and any consumer action,
|
||||
linking to the catalog policy rather than repeating its contracts.
|
||||
|
||||
## Establish The Candidate
|
||||
Follow the [OSS policy](policy/oss.md) when preparing licensing and attribution
|
||||
changes. Each new note must end with the following navigation, replacing
|
||||
`vMAJOR.MINOR.PATCH` with the selected release tag:
|
||||
|
||||
Choose a version that has not been published and export it as
|
||||
`RELEASE_VERSION`. Run every command in this procedure from the Promptkit
|
||||
repository root in the same POSIX shell. Do not reuse `v0.1.0` or another
|
||||
existing version.
|
||||
```markdown
|
||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit/src/tag/vMAJOR.MINOR.PATCH/COPYRIGHT) | [LICENSE](https://gitea.maximumdirect.net/eric/promptkit/src/tag/vMAJOR.MINOR.PATCH/LICENSE) | [ATTRIBUTION](https://gitea.maximumdirect.net/eric/promptkit/src/tag/vMAJOR.MINOR.PATCH/docs/oss/attribution.md)
|
||||
```
|
||||
|
||||
The following guard derives the release commit from `HEAD` and stops on a
|
||||
missing or malformed version, a checkout other than synchronized `main`,
|
||||
uncommitted changes, an active Go workspace, a module replacement, a vendor
|
||||
tree, or an existing local or remote tag:
|
||||
Check these file paths against the candidate; a future tag need not resolve
|
||||
remotely during preparation. The annotated tag derives from this same note,
|
||||
including its footer. Promptkit publishes no separate hosted release body or
|
||||
download page. If such a channel is deliberately introduced, reuse the note
|
||||
and preserve its footer. No extra network checks, artifacts, or approval gates
|
||||
are introduced, and published notes are not retrofitted.
|
||||
|
||||
Retire completed roadmaps, preserving planned or deferred work and repairing
|
||||
links. This is housekeeping, not a separate approval gate. Review changes
|
||||
through the ordinary development workflow, commit them, and push main to
|
||||
origin. No repeated release-specific diff review is required.
|
||||
|
||||
## Validate And Publish
|
||||
|
||||
Run the following with the chosen unused version (replace the placeholder):
|
||||
|
||||
```sh
|
||||
set -eu
|
||||
|
||||
: "${RELEASE_VERSION:?export an unpublished vMAJOR.MINOR.PATCH version}"
|
||||
if ! printf '%s\n' "$RELEASE_VERSION" |
|
||||
grep -Eq '^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$'
|
||||
then
|
||||
printf '%s\n' "invalid release version: $RELEASE_VERSION" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
RELEASE_COMMIT=$(git rev-parse --verify 'HEAD^{commit}')
|
||||
export RELEASE_COMMIT
|
||||
|
||||
check_release_candidate() {
|
||||
test "$(git branch --show-current)" = main
|
||||
test -z "$(git status --porcelain)"
|
||||
|
||||
gowork_value=$(go env GOWORK)
|
||||
case "$gowork_value" in
|
||||
''|off) ;;
|
||||
*)
|
||||
printf '%s\n' "active Go workspace: $gowork_value" >&2
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
|
||||
test -z "$(git ls-files go.work go.work.sum)"
|
||||
test ! -e vendor
|
||||
if grep -Eq '^[[:space:]]*replace([[:space:]]|\()' go.mod
|
||||
then
|
||||
printf '%s\n' 'go.mod contains a replacement' >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
git fetch origin main --tags
|
||||
test "$RELEASE_COMMIT" = \
|
||||
"$(git rev-parse --verify 'refs/remotes/origin/main^{commit}')"
|
||||
|
||||
if git show-ref --verify --quiet "refs/tags/$RELEASE_VERSION"
|
||||
then
|
||||
printf '%s\n' "local tag already exists: $RELEASE_VERSION" >&2
|
||||
return 1
|
||||
fi
|
||||
if test -n "$(
|
||||
git ls-remote --tags origin \
|
||||
"refs/tags/$RELEASE_VERSION" \
|
||||
"refs/tags/$RELEASE_VERSION^{}"
|
||||
)"
|
||||
then
|
||||
printf '%s\n' "remote tag already exists: $RELEASE_VERSION" >&2
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
check_release_candidate
|
||||
./scripts/release.sh vMAJOR.MINOR.PATCH
|
||||
```
|
||||
|
||||
Do not continue unless the guard completes successfully. In particular, push
|
||||
the intended commit through the normal `main` branch workflow before release;
|
||||
the tag procedure is not a substitute for publishing the source commit.
|
||||
The [publication script](../scripts/release.sh) checks a clean, synchronized
|
||||
main candidate, the version and unused tag, and the release note's title and
|
||||
required headings. It runs the [local checks](../scripts/release-check.sh)
|
||||
once, rechecks candidate identity and publication state, generates the
|
||||
annotation from the committed note plus the validated commit, and pushes only
|
||||
the selected annotated tag.
|
||||
|
||||
## Validate The Candidate
|
||||
There is no separate tag-message authoring, example-output inspection,
|
||||
metadata inspection, or human sign-off. Compatibility and release-note content
|
||||
remain ordinary editorial responsibilities. Development validation is useful
|
||||
but does not require a second full run before invoking the publication script.
|
||||
|
||||
Confirm the module and root package metadata:
|
||||
The script stops successfully after the tag push. CI completion, CI success,
|
||||
remote read-back, and Go module availability are not release completion gates.
|
||||
Do not wait or poll for them. Provider probes and inference calls are never
|
||||
part of release validation.
|
||||
|
||||
## Catalog Profile Compatibility
|
||||
|
||||
The [release policy](policy/release.md#catalog-profile-compatibility) owns the
|
||||
catalog compatibility boundary. This heading preserves older incoming links.
|
||||
|
||||
## Failure Recovery
|
||||
|
||||
If local checks fail, fix the candidate, commit and push changes, then rerun
|
||||
the publication command. It never automatically removes a tag.
|
||||
|
||||
If a push fails or its result is ambiguous, set `RELEASE_VERSION` to the
|
||||
attempted version and inspect origin before retrying:
|
||||
|
||||
```sh
|
||||
go list -m -f '{{.Path}} {{.GoVersion}}'
|
||||
go list -f '{{.Name}} {{.ImportPath}}' .
|
||||
git ls-remote --tags origin "refs/tags/$RELEASE_VERSION" "refs/tags/$RELEASE_VERSION^{}"
|
||||
git rev-parse "refs/tags/$RELEASE_VERSION" "refs/tags/$RELEASE_VERSION^{}"
|
||||
```
|
||||
|
||||
The output must be:
|
||||
Matching remote tag and peeled commit values confirm publication, even if the
|
||||
push command reported an error. If the remote query fails, publication status
|
||||
is still unknown. Do not delete or recreate the local tag until absence on
|
||||
origin is confirmed. Once confirmed unpublished, an unsuccessful attempt's
|
||||
local tag can be removed and the publication command rerun. Never overwrite
|
||||
a published tag.
|
||||
|
||||
```text
|
||||
gitea.maximumdirect.net/eric/promptkit 1.25.5
|
||||
promptkit gitea.maximumdirect.net/eric/promptkit
|
||||
```
|
||||
|
||||
As a release prerequisite, run the complete
|
||||
[maintainer validation workflow](development.md#maintainer-validation) against
|
||||
the clean candidate. Do not substitute a partial command list: the development
|
||||
guide owns the tests, race checks, analysis, build, both offline examples,
|
||||
formatting, Markdown links, generated-output and credential review, and
|
||||
repository hygiene. Record the successful workflow result with the candidate.
|
||||
|
||||
## Write The Release Note
|
||||
|
||||
Prepare a plain-text annotated-tag message outside the repository and export
|
||||
its path as `RELEASE_NOTES_FILE`. Use this form, replacing each summary with
|
||||
release-specific text; write `None.` when there are no public API changes or
|
||||
consumer actions:
|
||||
|
||||
```text
|
||||
Promptkit vMAJOR.MINOR.PATCH
|
||||
|
||||
Validated commit: full commit ID
|
||||
Compatibility: compatibility summary
|
||||
Public API changes: changes or None.
|
||||
Consumer action: required action or None.
|
||||
```
|
||||
|
||||
After writing it, require all release-note fields, the selected version, and
|
||||
the validated commit to be present:
|
||||
|
||||
```sh
|
||||
: "${RELEASE_NOTES_FILE:?export the path to the release-note file}"
|
||||
test -f "$RELEASE_NOTES_FILE"
|
||||
test -s "$RELEASE_NOTES_FILE"
|
||||
grep -F "Promptkit $RELEASE_VERSION" "$RELEASE_NOTES_FILE"
|
||||
grep -F "Validated commit: $RELEASE_COMMIT" "$RELEASE_NOTES_FILE"
|
||||
grep -F 'Compatibility:' "$RELEASE_NOTES_FILE"
|
||||
grep -F 'Public API changes:' "$RELEASE_NOTES_FILE"
|
||||
grep -F 'Consumer action:' "$RELEASE_NOTES_FILE"
|
||||
```
|
||||
|
||||
Inspect the complete message and confirm that it accurately records the
|
||||
compatibility impact, public API changes, and required consumer action.
|
||||
|
||||
## Create And Inspect The Tag
|
||||
|
||||
Run the candidate guard again immediately before tag creation. This ensures
|
||||
that validation or release-note preparation did not change the checkout and
|
||||
that the commit is still published and untagged:
|
||||
|
||||
```sh
|
||||
check_release_candidate
|
||||
```
|
||||
|
||||
Create the annotated tag from the prepared release note and bind it explicitly
|
||||
to the validated commit:
|
||||
|
||||
```sh
|
||||
git tag --annotate "$RELEASE_VERSION" \
|
||||
--file "$RELEASE_NOTES_FILE" \
|
||||
"$RELEASE_COMMIT"
|
||||
```
|
||||
|
||||
Inspect both the tag message and its source commit before publication:
|
||||
|
||||
```sh
|
||||
test "$(git cat-file -t "refs/tags/$RELEASE_VERSION")" = tag
|
||||
git show --no-patch --decorate "refs/tags/$RELEASE_VERSION"
|
||||
test "$(
|
||||
git rev-parse --verify "refs/tags/$RELEASE_VERSION^{commit}"
|
||||
)" = "$RELEASE_COMMIT"
|
||||
```
|
||||
|
||||
If inspection finds an error, delete the unpublished local tag, correct the
|
||||
release note or candidate, and repeat the guards. Never move or recreate a tag
|
||||
that has been published.
|
||||
|
||||
## Publish The Selected Tag
|
||||
|
||||
Push only the selected tag ref. Do not use `git push --tags`:
|
||||
|
||||
```sh
|
||||
git push origin \
|
||||
"refs/tags/$RELEASE_VERSION:refs/tags/$RELEASE_VERSION"
|
||||
```
|
||||
|
||||
## Verify Publication
|
||||
|
||||
Compare the remote annotated-tag object with the local object, then compare the
|
||||
remote peeled source commit with the validated commit:
|
||||
|
||||
```sh
|
||||
remote_tag=$(
|
||||
git ls-remote --tags origin "refs/tags/$RELEASE_VERSION" |
|
||||
awk 'NR == 1 { print $1 }'
|
||||
)
|
||||
remote_commit=$(
|
||||
git ls-remote --tags origin "refs/tags/$RELEASE_VERSION^{}" |
|
||||
awk 'NR == 1 { print $1 }'
|
||||
)
|
||||
test -n "$remote_tag"
|
||||
test "$remote_tag" = \
|
||||
"$(git rev-parse --verify "refs/tags/$RELEASE_VERSION")"
|
||||
test "$remote_commit" = "$RELEASE_COMMIT"
|
||||
```
|
||||
|
||||
Finally, resolve the version as an ordinary Go module in a temporary module
|
||||
outside this repository and without a workspace or replacement:
|
||||
|
||||
```sh
|
||||
resolution_dir=$(mktemp -d)
|
||||
(
|
||||
trap 'rm -rf "$resolution_dir"' 0 1 2 15
|
||||
cd "$resolution_dir"
|
||||
GOWORK=off go mod init example.com/promptkit-release-check
|
||||
GOWORK=off go mod download \
|
||||
"gitea.maximumdirect.net/eric/promptkit@$RELEASE_VERSION"
|
||||
resolved_version=$(
|
||||
GOWORK=off go list -m -f '{{.Version}}' \
|
||||
"gitea.maximumdirect.net/eric/promptkit@$RELEASE_VERSION"
|
||||
)
|
||||
test "$resolved_version" = "$RELEASE_VERSION"
|
||||
)
|
||||
```
|
||||
|
||||
Promptkit must publish and verify the required version before Scriptorium or
|
||||
another consumer publishes a release that depends on it. This ordering does
|
||||
not replace the consumer project's own release procedure. Released consumers
|
||||
must select the published Promptkit tag through ordinary module resolution,
|
||||
without a workspace, replacement, vendored Promptkit source, or unpublished
|
||||
revision.
|
||||
CI or module-resolution problems may be investigated separately; temporary
|
||||
service, network, or authentication failures do not require another release.
|
||||
Only an actual defect requiring changes to published contents warrants a new
|
||||
appropriate version. See the [recovery policy](policy/release.md#immutability-and-failure-recovery).
|
||||
|
||||
## Policy Changes
|
||||
|
||||
Document and approve a durable policy change before introducing hosted
|
||||
automation, binary artifacts, or different release governance. Update this
|
||||
procedure in the same change so maintainers do not rely on hidden release
|
||||
requirements.
|
||||
Update the policy and this procedure together when changing release governance.
|
||||
Consumers own their dependency-availability and release checks; their readiness
|
||||
does not delay completion of a Promptkit release.
|
||||
|
||||
44
docs/releases/v0.10.0.md
Normal file
44
docs/releases/v0.10.0.md
Normal file
@@ -0,0 +1,44 @@
|
||||
# Promptkit v0.10.0
|
||||
|
||||
This supplemental guide summarizes the `v0.10.0` changes and migration needs.
|
||||
The annotated tag remains the authoritative release record.
|
||||
|
||||
## Transition Summary
|
||||
|
||||
`v0.10.0` adopts independently versioned OpenRouter and Rakestrawhome catalog
|
||||
modules whose external profile data can change between catalog releases. It
|
||||
also introduces the maintained literal names
|
||||
`promptkit-openrouter-standard` and `promptkit-openrouter-auto`.
|
||||
|
||||
The public package now delegates assembly to an internal engine while retaining
|
||||
the existing Go API. This refactor requires no consumer code changes.
|
||||
|
||||
This release selects OpenRouter `v2.20260922.1` and Rakestrawhome
|
||||
`v2.20260922.0`; their exact inventories are linked from the
|
||||
[format reference](../formats.md#external-profile-inventories).
|
||||
|
||||
The maintained names provide continuity only within the catalog versions
|
||||
selected and tested by a Promptkit release and when consumers do not override
|
||||
the name or its base. Their current selections may change. External catalog
|
||||
profile IDs can be removed, renamed, or modified, so an application that
|
||||
selects one directly may need to update its configuration when upgrading.
|
||||
|
||||
## Consumer Migration
|
||||
|
||||
Before upgrading, review every profile ID used by prompts, requests, and
|
||||
application-owned inheritance. Choose a maintained name, accept an external
|
||||
catalog profile's volatility, or define a complete application-owned profile
|
||||
when the application must control its target and settings. An alias that
|
||||
inherits from an external base is not a frozen copy.
|
||||
|
||||
In particular, the selected OpenRouter catalog removes `deepseek-3-2`,
|
||||
`deepseek-4-flash`, and `deepseek-4-pro`. Consumers using those IDs must choose
|
||||
an available profile or supply their own complete definition; the new
|
||||
`deepseek-flash-latest` and `deepseek-pro-latest` profiles are not automatic
|
||||
aliases and need not preserve the previous settings.
|
||||
|
||||
See the [release policy](../release.md#catalog-profile-compatibility) for the
|
||||
compatibility boundary, the [format reference](../formats.md#built-in-profile-catalog)
|
||||
for current Promptkit-owned definitions and selected external inventories, and
|
||||
the [consumer guide](../consumers/pkg-promptkit.md#choose-a-profile-ownership-model)
|
||||
for configuration choices.
|
||||
76
docs/releases/v0.11.0.md
Normal file
76
docs/releases/v0.11.0.md
Normal file
@@ -0,0 +1,76 @@
|
||||
# Promptkit v0.11.0
|
||||
|
||||
This release adds reliable, observable model execution and changes Promptkit's
|
||||
license to LGPL-3.0-or-later.
|
||||
|
||||
## Summary
|
||||
|
||||
- Built-in model calls can recover from selected transient HTTP failures.
|
||||
- Optional execution observations report attempts, timing, outcomes, and known
|
||||
token usage, including work completed before a run fails.
|
||||
- Public errors distinguish transport failures, malformed responses, and retry
|
||||
exhaustion without parsing diagnostic text.
|
||||
|
||||
## Compatibility
|
||||
|
||||
The built-in client now defaults to two additional transient attempts. This can
|
||||
increase latency, request counts, and cost; retries do not guarantee exactly-once
|
||||
generation. Injected `LLMClient` implementations retain their own retry policy
|
||||
and are not automatically replayed. Existing generation timeouts apply per
|
||||
attempt, while the caller context governs the complete operation. See the
|
||||
[provider integration contract](../integrations/openai-compatible-chat.md).
|
||||
|
||||
Public retry and observation controls are additive. Existing successful-result
|
||||
usage semantics remain unchanged, and operational failures still return no
|
||||
partial `RunResult`. Observations distinguish unknown usage from reported zero;
|
||||
they are not a complete billing ledger. See the
|
||||
[consumer guide](../consumers/pkg-promptkit.md) and
|
||||
[observation GoDoc](../../observations.go).
|
||||
|
||||
Promptkit is now licensed under LGPL-3.0-or-later. Review the
|
||||
[license summary](../../README.md#license), [LGPL text](../../LICENSE), and
|
||||
[accompanying GPL text](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.11.0/COPYING) before adoption or distribution.
|
||||
Previously published releases and dependency licenses are unchanged.
|
||||
|
||||
## Upgrade
|
||||
|
||||
Update the dependency:
|
||||
|
||||
```sh
|
||||
go get gitea.maximumdirect.net/eric/promptkit@v0.11.0
|
||||
```
|
||||
|
||||
To retain single-attempt behavior for built-in generation, set
|
||||
`Config.TransientRetries` to a pointer to zero. `RunRequest.TransientRetries`
|
||||
can override the engine choice per request. Review any existing application or
|
||||
HTTP-transport retry loops to avoid unintentionally multiplying attempts.
|
||||
Transient retries and structural repairs have separate budgets; provide a
|
||||
caller deadline when a total execution time limit is needed.
|
||||
|
||||
Observation is opt-in through `RunRequest.Observer`, including for prepared
|
||||
execution. Callbacks are synchronous; shared observers must support concurrent
|
||||
runs. Injected clients can supply `GenerateResponse.UsagePresence` to explicitly
|
||||
report known zero counts. No observer or usage-presence changes are required
|
||||
for existing consumers. See the [consumer guide](../consumers/pkg-promptkit.md)
|
||||
for adoption and the [offline example](../../examples/go-library/observe/main.go)
|
||||
for a structured proposal-and-assessment workflow.
|
||||
|
||||
## Changes
|
||||
|
||||
- Centralized built-in retry coordination with cancellable backoff and provider
|
||||
retry timing. Retries and structural repair share existing backend capacity;
|
||||
backoff releases active permits without releasing logical run admission.
|
||||
- Added `ErrTransport`, `ErrMalformedResponse`, `ErrRetriesExhausted`, and
|
||||
`RetryExhaustedError`, retaining applicable underlying error identities.
|
||||
- Added request-scoped lifecycle observations and presence-aware token accounting
|
||||
for ordinary and prepared execution, without recording prompt/response bodies.
|
||||
- Isolated delegated operations' observation state and made injected-client
|
||||
attempt classifications consistent with terminal classifications.
|
||||
- Added an offline observation example and regression coverage for retry,
|
||||
repair, capacity, error, and observation interactions.
|
||||
- Adopted LGPL-3.0-or-later and streamlined the documented local release process.
|
||||
|
||||
Backend catalog selections are unchanged from v0.10.0: OpenRouter
|
||||
`v2.20260922.1` and Rakestrawhome `v2.20260922.0`. See the
|
||||
[catalog compatibility policy](../policy/release.md#catalog-profile-compatibility)
|
||||
and [profile inventories](../formats.md#external-profile-inventories).
|
||||
43
docs/releases/v0.11.1.md
Normal file
43
docs/releases/v0.11.1.md
Normal file
@@ -0,0 +1,43 @@
|
||||
# Promptkit v0.11.1
|
||||
|
||||
This patch updates the maintained catalogs to MIT-licensed releases and
|
||||
improves licensing documentation.
|
||||
|
||||
## Summary
|
||||
|
||||
Both OpenRouter and Rakestrawhome catalogs now use `v2.20260926.0`. These
|
||||
catalog releases change licensing and documentation only; their profiles,
|
||||
backend definitions, and Go code are unchanged.
|
||||
|
||||
## Compatibility
|
||||
|
||||
No public API, runtime behavior, or profile changes are introduced.
|
||||
Promptkit remains LGPL-3.0-or-later; the catalogs are now MIT-licensed.
|
||||
See the [project grant](../../COPYRIGHT),
|
||||
[component attribution](../oss/attribution.md), and
|
||||
[catalog compatibility policy](../policy/release.md#catalog-profile-compatibility).
|
||||
|
||||
## Upgrade
|
||||
|
||||
Update the module dependency:
|
||||
|
||||
```sh
|
||||
go get gitea.maximumdirect.net/eric/promptkit@v0.11.1
|
||||
```
|
||||
|
||||
No code or configuration migration is required from v0.11.0.
|
||||
Consumers upgrading from earlier versions should also read the
|
||||
[v0.11.0 migration guide](v0.11.0.md).
|
||||
|
||||
## Changes
|
||||
|
||||
- Adopt the MIT-licensed catalog releases and update inventory links.
|
||||
- Add centralized attribution and dependency license texts.
|
||||
- Separate the project grant into COPYRIGHT and consolidate the LGPLv3 and
|
||||
incorporated GPLv3 texts in LICENSE, replacing the separate COPYING file.
|
||||
- Align documentation ownership and future release navigation with the
|
||||
[OSS policy](../policy/oss.md), without adding release gates.
|
||||
- Repair the historical v0.11.0 GPL-text link and retire the completed
|
||||
licensing roadmap.
|
||||
|
||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.11.1/COPYRIGHT) | [LICENSE](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.11.1/LICENSE) | [ATTRIBUTION](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.11.1/docs/oss/attribution.md)
|
||||
38
docs/releases/v0.11.2.md
Normal file
38
docs/releases/v0.11.2.md
Normal file
@@ -0,0 +1,38 @@
|
||||
# Promptkit v0.11.2
|
||||
|
||||
This patch aligns repository policies with the current standard and clarifies
|
||||
licensing and attribution for the copied policy documents.
|
||||
|
||||
## Summary
|
||||
|
||||
The update adds policy provenance, selective contributor reading guidance, and
|
||||
consistent upstream policy language while preserving Promptkit-specific rules.
|
||||
|
||||
## Compatibility
|
||||
|
||||
No public API, runtime behavior, dependency, or catalog changes are introduced.
|
||||
Promptkit remains LGPL-3.0-or-later. The [project grant](../../COPYRIGHT)
|
||||
identifies the separate CC-BY-SA-4.0 scope for copied or adapted policy material;
|
||||
the [attribution notices](../oss/attribution.md) identify the affected documents.
|
||||
The existing release checks and successful-tag-push completion point are unchanged.
|
||||
|
||||
## Upgrade
|
||||
|
||||
```sh
|
||||
go get gitea.maximumdirect.net/eric/promptkit@v0.11.2
|
||||
```
|
||||
|
||||
No code or configuration migration is required from v0.11.1.
|
||||
|
||||
## Changes
|
||||
|
||||
- Align adopted policies with Standard Repository Policies while retaining
|
||||
library-specific requirements and deliberate local decisions.
|
||||
- Add per-document upstream source links and the
|
||||
[policy-set provenance document](../policy/meta.md).
|
||||
- Add selective policy reading to the [contributor guide](../development.md)
|
||||
and update agent orientation.
|
||||
- Standardize factual attribution entries and identify copied-policy licensing.
|
||||
- Retire the completed policy-alignment roadmap.
|
||||
|
||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.11.2/COPYRIGHT) | [LICENSE](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.11.2/LICENSE) | [ATTRIBUTION](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.11.2/docs/oss/attribution.md)
|
||||
155
docs/releases/v0.7.0.md
Normal file
155
docs/releases/v0.7.0.md
Normal file
@@ -0,0 +1,155 @@
|
||||
# Promptkit v0.7.0
|
||||
|
||||
This supplemental changelog and migration guide summarizes the consumer-facing
|
||||
changes from `v0.6.0` to `v0.7.0`. The annotated `v0.7.0` tag is the
|
||||
authoritative release record. Exact current contracts belong to the linked
|
||||
GoDoc and durable documentation.
|
||||
|
||||
## Summary
|
||||
|
||||
`v0.7.0` expands provider integration and profile composition while making
|
||||
credential and generation-failure handling more flexible:
|
||||
|
||||
- Promptkit now includes the `rakestrawhome` backend and its Gemma profile;
|
||||
- built-in generation failures expose bounded structured provider details;
|
||||
- an unavailable optional API-key environment source no longer prevents a
|
||||
request from reaching an upstream that permits unauthenticated access; and
|
||||
- profiles can inherit from and selectively refine another profile.
|
||||
|
||||
## Compatibility
|
||||
|
||||
This release adds public declarations and fields but removes none. Existing
|
||||
keyed configuration literals and ordinary `errors.Is` handling continue to
|
||||
work.
|
||||
|
||||
Adding `BaseProfileID` to `Profile` and `OpenAICompatibleProfileConfig` changes
|
||||
their struct shape. Consumers using positional composite literals for either
|
||||
type must convert them to keyed literals. Existing keyed literals require no
|
||||
change.
|
||||
|
||||
The `rakestrawhome` backend ID is now built in and reserved. A consumer that
|
||||
previously registered that exact ID with `WithBackend` must remove its manual
|
||||
registration before upgrading. Other custom backend registrations are
|
||||
unchanged.
|
||||
|
||||
When an optional backend, profile, or request `APIKeyEnv` is unset, empty, or
|
||||
whitespace-only, the built-in client now omits `Authorization` and sends the
|
||||
request. Previously this condition could fail before transport. Set
|
||||
`Profile.APIKeyRequired` when missing credentials must remain a local
|
||||
preflight error.
|
||||
|
||||
Provider non-success responses continue to match `ErrLLMGenerate`. Their
|
||||
rendered wording is not a compatibility contract; consumers can now use
|
||||
`errors.As` with `*GenerationError` when structured status information is
|
||||
needed.
|
||||
|
||||
## Upgrade
|
||||
|
||||
Update the module dependency with:
|
||||
|
||||
```sh
|
||||
go get gitea.maximumdirect.net/eric/promptkit@v0.7.0
|
||||
go mod tidy
|
||||
```
|
||||
|
||||
Remove any manual `rakestrawhome` backend registration, convert positional
|
||||
profile literals to keyed literals, and run the consuming project's ordinary
|
||||
and race-enabled tests.
|
||||
|
||||
## Rakestrawhome Built-In Backend And Profile
|
||||
|
||||
Every engine now includes the reserved `rakestrawhome` backend, identified by
|
||||
`BackendRakestrawHome`. The built-in `rakestrawhome-gemma-4-31b` profile
|
||||
selects that backend. Consumers can use the maintained endpoint, credential,
|
||||
capacity, and model defaults without registering either definition themselves.
|
||||
|
||||
See the [built-in backend and profile catalogs](../formats.md#built-in-backends)
|
||||
and the [consumer adoption example](../consumers/pkg-promptkit.md#use-the-rakestrawhome-built-in-profile)
|
||||
for the current contracts.
|
||||
|
||||
## Structured Generation Errors
|
||||
|
||||
Non-2xx responses from the built-in OpenAI-compatible client now return an
|
||||
immutable `*GenerationError`. Consumers can inspect the HTTP status and any
|
||||
safely extracted provider code, type, or message while retaining the ordinary
|
||||
generation-error category:
|
||||
|
||||
```go
|
||||
var generationErr *promptkit.GenerationError
|
||||
if errors.As(err, &generationErr) {
|
||||
status := generationErr.StatusCode()
|
||||
_ = status
|
||||
}
|
||||
```
|
||||
|
||||
Provider fields are bounded and normalized but remain untrusted and may
|
||||
contain sensitive request or schema details. Default and Go-syntax formatting
|
||||
omit those fields. Applications must apply their own disclosure policy before
|
||||
logging or presenting accessor values.
|
||||
|
||||
See the [`GenerationError` GoDoc](../../generation_error.go), the
|
||||
[consumer error-handling guide](../consumers/pkg-promptkit.md#handle-errors),
|
||||
and the [OpenAI-compatible response contract](../integrations/openai-compatible-chat.md#response-handling).
|
||||
|
||||
## Optional Credential Sources
|
||||
|
||||
`APIKeyEnv` names an optional environment lookup source unless the selected
|
||||
profile explicitly sets `APIKeyRequired`. When neither a direct request key nor
|
||||
a usable environment value exists, the built-in client omits the bearer header
|
||||
and handles the upstream response normally. This supports local and other
|
||||
OpenAI-compatible providers that permit unauthenticated requests without
|
||||
hiding an authentication error returned by a provider that requires one.
|
||||
|
||||
The [credential format reference](../formats.md#credentials), the
|
||||
[`Backend` GoDoc](../../backends.go), the
|
||||
[`ExecutionTargetOverride` GoDoc](../../types.go), and the
|
||||
[authentication integration contract](../integrations/openai-compatible-chat.md#authentication)
|
||||
define the current precedence and availability rules.
|
||||
|
||||
## Profile Inheritance
|
||||
|
||||
YAML profiles can name one parent with `base_profile`; in-memory profiles use
|
||||
`Profile.BaseProfileID`, and `OpenAICompatibleProfileConfig` forwards the same
|
||||
field. A profile can act as an application-owned alias of a built-in or refine
|
||||
selected inherited settings:
|
||||
|
||||
```go
|
||||
promptkit.WithProfiles(promptkit.Profile{
|
||||
ID: "weather-light",
|
||||
BaseProfileID: "deepseek-4-flash",
|
||||
ReasoningEffort: "high",
|
||||
})
|
||||
```
|
||||
|
||||
Base lookup observes the existing source precedence. Chains are linear,
|
||||
cycle-safe, and resolved afresh for ordinary operations. Prepared execution
|
||||
freezes the fully resolved target. The selected leaf ID remains public while
|
||||
effective execution settings reflect the resolved chain.
|
||||
|
||||
See the [profile inheritance format reference](../formats.md#profile-inheritance),
|
||||
the [consumer alias example](../consumers/pkg-promptkit.md#alias-a-built-in-profile),
|
||||
and the [`Profile` GoDoc](../../types.go) for exact merge and validation
|
||||
behavior.
|
||||
|
||||
## Public API Changes
|
||||
|
||||
The release adds:
|
||||
|
||||
- `BackendRakestrawHome`;
|
||||
- `GenerationError`, including `StatusCode`, `ProviderCode`, `ProviderType`,
|
||||
`ProviderMessage`, `Error`, `GoString`, and `Unwrap`;
|
||||
- `Profile.BaseProfileID`; and
|
||||
- `OpenAICompatibleProfileConfig.BaseProfileID`.
|
||||
|
||||
No public declaration was removed.
|
||||
|
||||
## Consumer Action
|
||||
|
||||
- Remove a manual backend registration whose ID is exactly `rakestrawhome`.
|
||||
- Convert positional `Profile` or `OpenAICompatibleProfileConfig` literals to
|
||||
keyed literals.
|
||||
- Set `Profile.APIKeyRequired` where a missing credential must fail locally
|
||||
instead of reaching the provider unauthenticated.
|
||||
- Treat `GenerationError` provider fields as untrusted and potentially
|
||||
sensitive when adopting the new accessors.
|
||||
- Run consumer ordinary and race-enabled tests after updating the module.
|
||||
109
docs/releases/v0.8.0.md
Normal file
109
docs/releases/v0.8.0.md
Normal file
@@ -0,0 +1,109 @@
|
||||
# Promptkit v0.8.0
|
||||
|
||||
This supplemental changelog and migration guide summarizes the consumer-facing
|
||||
changes from `v0.7.0` to `v0.8.0`. The annotated `v0.8.0` tag is the
|
||||
authoritative release record. Exact current contracts belong to the linked
|
||||
GoDoc and durable documentation.
|
||||
|
||||
## Summary
|
||||
|
||||
`v0.8.0` activates Promptkit's bounded output-repair workflow:
|
||||
|
||||
- failed nonempty-text, JSON, and JSON Schema validation can make a limited
|
||||
number of corrective model calls;
|
||||
- corrective calls preserve the original rendered conversation, effective
|
||||
target, session, structured-output contract, and backend capacity policy;
|
||||
- results report cumulative usage and the number of corrective calls actually
|
||||
made; and
|
||||
- explicitly empty OpenAI-compatible response content now reaches output
|
||||
validation instead of being classified as a malformed provider envelope.
|
||||
|
||||
## Compatibility
|
||||
|
||||
This release adds no public declarations or fields and removes none. Existing
|
||||
source code remains source-compatible.
|
||||
|
||||
The behavior of the existing `OutputContract.RepairAttempts` field and prompt
|
||||
YAML `repair_attempts` field has changed. A positive value now authorizes real
|
||||
additional model calls after eligible validation failures; earlier releases
|
||||
accepted the field but the public engine remained single-pass. Consumers that
|
||||
set a positive value should expect additional latency, token usage, and
|
||||
provider cost when repair is needed.
|
||||
|
||||
Repair budgets must now be between zero and three. A positive budget requires
|
||||
`basic`, `json`, or `json_schema` validation. Values above three and a positive
|
||||
budget paired with `none` are invalid contracts rather than ignored settings.
|
||||
|
||||
An explicitly present empty or whitespace-only string returned by the built-in
|
||||
OpenAI-compatible client is now a completed generation candidate. `none`
|
||||
validation permits it, while `basic`, `json`, and `json_schema` classify it
|
||||
under their ordinary validation rules and may repair it when configured.
|
||||
Missing, `null`, or non-string content remains a malformed provider response.
|
||||
|
||||
## Upgrade
|
||||
|
||||
Update the module dependency with:
|
||||
|
||||
```sh
|
||||
go get gitea.maximumdirect.net/eric/promptkit@v0.8.0
|
||||
go mod tidy
|
||||
```
|
||||
|
||||
Review every prompt definition and request override that sets a positive repair
|
||||
budget. Use zero or omit the field to retain single-pass execution. Ensure each
|
||||
positive budget is no greater than three and uses an eligible validation mode,
|
||||
then run the consuming project's ordinary and race-enabled tests.
|
||||
|
||||
## Bounded Output Repair
|
||||
|
||||
`repair_attempts` counts corrective calls in addition to the initial model
|
||||
call. Promptkit validates each completed candidate, stops at the first valid
|
||||
one, and never exceeds the configured bound. If every candidate remains
|
||||
invalid, the run completes successfully with the final candidate and its
|
||||
failed validation result rather than returning an operational error.
|
||||
|
||||
Each correction starts from the original rendered messages and includes only
|
||||
the latest invalid candidate and latest validation diagnostics. JSON Schema
|
||||
mode retains the provider-native structured-output request as its first line of
|
||||
defense. Promptkit performs only deterministic structural validation; a valid
|
||||
response is not necessarily factual or correct for an application's domain.
|
||||
|
||||
Usage in the final result is cumulative across the initial response and every
|
||||
completed corrective response. `ValidationResult.RepairAttempts` reports the
|
||||
number of corrective calls actually made. Corrective generation failures use
|
||||
the same public generation-error categories and structured provider details as
|
||||
an initial generation failure.
|
||||
|
||||
See the [output-contract format reference](../formats.md#output-contract), the
|
||||
[consumer repair example](../consumers/pkg-promptkit.md#repair-a-structured-result),
|
||||
and the [`OutputContract` and `ValidationResult` GoDoc](../../types.go) for the
|
||||
current contracts.
|
||||
|
||||
## Explicit Empty Content
|
||||
|
||||
The built-in OpenAI-compatible client now distinguishes an explicitly present
|
||||
empty string from a missing or malformed `content` field. This aligns built-in
|
||||
and injected clients by letting the selected output contract decide whether an
|
||||
empty candidate is acceptable, invalid, or eligible for repair.
|
||||
|
||||
See the
|
||||
[OpenAI-compatible response contract](../integrations/openai-compatible-chat.md#response-handling)
|
||||
for the exact envelope behavior.
|
||||
|
||||
## Public API Changes
|
||||
|
||||
None. This release activates and tightens the documented behavior of existing
|
||||
fields.
|
||||
|
||||
## Consumer Action
|
||||
|
||||
- Remove or set `repair_attempts` to zero where execution must remain
|
||||
single-pass.
|
||||
- Keep every positive repair budget at three or fewer and pair it with
|
||||
`basic`, `json`, or `json_schema` validation.
|
||||
- Account for additional latency, usage, and provider cost when enabling
|
||||
repair.
|
||||
- Continue checking the returned validation status because bounded repair can
|
||||
exhaust without producing a valid candidate.
|
||||
- Review workflows that previously treated explicit empty provider content as
|
||||
a generation error.
|
||||
125
docs/releases/v0.9.0.md
Normal file
125
docs/releases/v0.9.0.md
Normal file
@@ -0,0 +1,125 @@
|
||||
# Promptkit v0.9.0
|
||||
|
||||
This supplemental changelog and migration guide summarizes the consumer-facing
|
||||
changes from `v0.8.0` to `v0.9.0`. The annotated `v0.9.0` tag is the
|
||||
authoritative release record. Exact current contracts belong to the linked
|
||||
GoDoc and durable documentation.
|
||||
|
||||
## Summary
|
||||
|
||||
`v0.9.0` adds stateless request-message composition and separates maintained
|
||||
provider data from Promptkit's core implementation:
|
||||
|
||||
- callers can append already-rendered messages to a configured prompt for
|
||||
application-owned conversations and semantic correction workflows;
|
||||
- the public package now publishes constants for the four supported text-chat
|
||||
roles, and prompt definitions use the same normalized role vocabulary; and
|
||||
- the OpenRouter and Rakestrawhome backend/profile catalogs now come from two
|
||||
independently versioned Go module dependencies.
|
||||
|
||||
## Compatibility
|
||||
|
||||
This release adds one field and four constants to the public API and removes no
|
||||
public declarations. Existing keyed `RunRequest` literals that omit
|
||||
`AppendedMessages` retain their behavior. Adding the field changes the struct
|
||||
shape, so consumers using positional `RunRequest` literals must convert them
|
||||
to keyed literals.
|
||||
|
||||
Message roles in prompt definitions are now trimmed, lowercased, and required
|
||||
to be `developer`, `system`, `user`, or `assistant`. Definitions using another
|
||||
role that earlier releases accepted as an arbitrary nonblank string now fail
|
||||
prompt loading. In particular, the text-only message contract does not support
|
||||
`tool` or the deprecated `function` role. Otherwise valid roles with different
|
||||
case or surrounding whitespace are normalized rather than rejected.
|
||||
|
||||
The catalog extraction preserves Promptkit's public API, built-in backend and
|
||||
profile IDs, configuration, precedence, credential handling, and capacity
|
||||
behavior. Consumers do not import or register either catalog themselves, and
|
||||
no configuration migration is required. Promptkit now selects two
|
||||
independently versioned data dependencies and guarantees only the catalog
|
||||
versions selected and tested by this Promptkit release.
|
||||
|
||||
## Upgrade
|
||||
|
||||
Update the module dependency with:
|
||||
|
||||
```sh
|
||||
go get gitea.maximumdirect.net/eric/promptkit@v0.9.0
|
||||
go mod tidy
|
||||
```
|
||||
|
||||
Convert any positional `RunRequest` literals to keyed literals. Review prompt
|
||||
definitions for unsupported roles, then run the consuming project's ordinary
|
||||
and race-enabled tests.
|
||||
|
||||
## Appended Request Messages
|
||||
|
||||
`RunRequest.AppendedMessages` accepts caller-owned `RenderedMessage` values
|
||||
that Promptkit validates, copies, and appends after every rendered
|
||||
prompt-definition message in caller order. Promptkit does not template this
|
||||
content, retain conversation state between calls, impose a retry policy, or
|
||||
apply a message-count, byte-size, token, or context-window limit. Upstream
|
||||
rejections continue through the ordinary generation-error boundary.
|
||||
|
||||
This primitive supports application-owned conversation continuations and
|
||||
domain-aware correction loops while preserving Promptkit's existing
|
||||
preparation, hashing, prepared-execution, structural repair, backend-capacity,
|
||||
credential, and cancellation behavior. Appended content can include sensitive
|
||||
model output or application feedback; prepared values expose the complete
|
||||
effective messages by design, while default request formatting reports only
|
||||
the appended-message count.
|
||||
|
||||
See the
|
||||
[consumer appended-message example](../consumers/pkg-promptkit.md#append-already-rendered-messages),
|
||||
the [`RunRequest`, `RenderedMessage`, and `CacheControl` GoDoc](../../types.go),
|
||||
and the [OpenAI-compatible request contract](../integrations/openai-compatible-chat.md#request-body)
|
||||
for current behavior.
|
||||
|
||||
## Supported Message Roles
|
||||
|
||||
The new `RoleDeveloper`, `RoleSystem`, `RoleUser`, and `RoleAssistant`
|
||||
constants identify the complete role vocabulary accepted by Promptkit's
|
||||
text-chat message model. The same validation and normalization now apply to
|
||||
prompt-definition messages and request-supplied appended messages. Promptkit
|
||||
does not translate between roles; provider- or model-specific rejection of an
|
||||
otherwise supported role remains an upstream generation error.
|
||||
|
||||
See the [message format reference](../formats.md#messages-and-templates) for
|
||||
the canonical prompt-definition contract.
|
||||
|
||||
## Independently Versioned Backend Catalogs
|
||||
|
||||
Promptkit imports immutable catalog data for its maintained OpenRouter and
|
||||
Rakestrawhome backends and profiles. Engine construction validates and
|
||||
assembles both catalogs behind the existing built-in registry and profile
|
||||
precedence rules. Promptkit no longer keeps duplicate embedded profile assets
|
||||
or hard-coded definitions for those maintained backends.
|
||||
|
||||
The module versions in Promptkit's `go.mod` identify the catalog releases
|
||||
tested with this release. The [built-in backend and profile format
|
||||
reference](../formats.md#built-in-backends) remains the canonical consumer
|
||||
contract, while the [internal source documentation](../internal/sources.md#profiles-and-built-ins)
|
||||
describes the dependency boundary.
|
||||
|
||||
## Public API Changes
|
||||
|
||||
The release adds:
|
||||
|
||||
- `RunRequest.AppendedMessages`;
|
||||
- `RoleDeveloper`;
|
||||
- `RoleSystem`;
|
||||
- `RoleUser`; and
|
||||
- `RoleAssistant`.
|
||||
|
||||
No public declaration was removed.
|
||||
|
||||
## Consumer Action
|
||||
|
||||
- Convert positional `RunRequest` literals to keyed literals.
|
||||
- Replace unsupported prompt-definition roles with an appropriate supported
|
||||
role, or keep richer tool-call protocols in an application-owned client.
|
||||
- Treat appended messages and prepared effective messages according to the
|
||||
application's sensitive-data policy.
|
||||
- Do not add direct catalog imports or registration calls; existing Promptkit
|
||||
construction and configuration remain correct.
|
||||
- Run consumer ordinary and race-enabled tests after updating the module.
|
||||
505
docs/roadmap/configkit-assessment.md
Normal file
505
docs/roadmap/configkit-assessment.md
Normal file
@@ -0,0 +1,505 @@
|
||||
# ConfigKit Adoption Assessment
|
||||
|
||||
Status: assessment aligned to released ConfigKit v0.5.0. Upstream capability
|
||||
prerequisites are available; PromptKit integration and migration decisions remain
|
||||
for the feature roadmap and implementation plan. No runtime changes are part of
|
||||
this assessment.
|
||||
|
||||
## Goal And Recommendation
|
||||
|
||||
Use ConfigKit to standardize PromptKit's configuration semantics and simplify
|
||||
its loading machinery, improving ConfigKit where a reusable capability is
|
||||
justified. Preserve substantive consumer capabilities, not historical loading
|
||||
sequences, file syntax, or Go signatures. Document intentional breaking changes
|
||||
and their migration paths.
|
||||
|
||||
Recommend adoption through an eager, immutable definition-loading boundary.
|
||||
ConfigKit should own YAML interpretation, source-tier combination, strict typed
|
||||
decoding, sibling inheritance, presence, and source diagnostics. PromptKit should
|
||||
own definition selection, LLM-specific rules, and execution. Do not retain the lazy loading
|
||||
architecture merely to minimize migration, or introduce a second generic
|
||||
configuration framework inside PromptKit.
|
||||
|
||||
The expected benefits are consistent configuration semantics for applications
|
||||
using both libraries and removal of repeated discovery, partial metadata parsing,
|
||||
lazy error arbitration, and repeated profile-chain resolution. Some of those
|
||||
savings come from the loading redesign rather than ConfigKit itself. ConfigKit
|
||||
adds ongoing value by owning the generic decoding behavior and its tests.
|
||||
|
||||
## Released ConfigKit Capabilities
|
||||
|
||||
ConfigKit v0.5.0 satisfies the two requested prerequisites and also implements
|
||||
sibling inheritance. Adopt the released public contracts rather than waiting for
|
||||
the earlier proposals to be implemented:
|
||||
|
||||
1. **Filesystem sources:** `NewFilesystem(label, fsys).File(name)` borrows an
|
||||
`fs.FS` with reusable namespace identity. Imports and resource contexts retain
|
||||
that namespace. `Result.SourceContext` and `Resolution.SourceContext` expose
|
||||
passive `ResourceContext` values with `FS` and `BaseDir`.
|
||||
2. **Source-backed underlays:** `LoadLayered` accepts a main `Input`
|
||||
and ordered `Underlay` groups. Each group composes strictly; groups overlay
|
||||
recursively from lowest to highest priority. `Input{Source: NoSource()}` omits
|
||||
the main input. `Mapping(label, map[string]any)` snapshots supported built-in
|
||||
scalar/list/map values with explicit presence; it does not ingest arbitrary
|
||||
typed structs. Generic programmatic input is not represented as CLI overrides.
|
||||
3. **Sibling inheritance:** the reserved `extends` key names one immediate sibling
|
||||
mapping. Relationships resolve after precedence assembly and before typed
|
||||
decoding, with inherited origins, history, and resource context preserved.
|
||||
This is built-in behavior, not the opt-in/registered-collection API originally
|
||||
suggested by PromptKit.
|
||||
|
||||
Adopt ConfigKit's recursive combination model rather than preserving PromptKit's
|
||||
whole-profile shadowing or atomic provider-map replacement across source tiers.
|
||||
No hidden replacement or inheritance pass should run after ConfigKit loading.
|
||||
Use released sibling inheritance for profiles, leaving domain validation in
|
||||
PromptKit. This assessment does not authorize code changes in either repository.
|
||||
|
||||
## Evidence And Scope
|
||||
|
||||
PromptKit's runtime baseline remains v0.11.2; this assessment has since received
|
||||
documentation-only revisions. Updated upstream review on 2026-09-30:
|
||||
|
||||
- PromptKit: `12d6d80024e80c552d122a076c22429bf6f2f6c2` (v0.11.2).
|
||||
- ConfigKit: `dc09959cca4e105f49ec316c6d4ba585bd2340bd` (v0.5.0), verified
|
||||
against the published tag. Its clean checkout passed `go test ./...` during
|
||||
the release review. This replaces the earlier v0.4.2 and uncommitted-roadmap
|
||||
comparison as the upstream basis for adoption.
|
||||
|
||||
PromptKit evidence: [consumer workflows](../consumers/pkg-promptkit.md),
|
||||
[file formats](../formats.md), [source implementation](../internal/sources.md),
|
||||
[runner](../internal/runner.md), and the implementation owners identified below.
|
||||
ConfigKit evidence at that release: `docs/config.md`, `docs/releases/v0.5.0.md`,
|
||||
`layered.go`, `source.go`, `resource_context.go`, and ADR 0009 in the sibling
|
||||
checkout. That checkout is review context, not a build dependency or required
|
||||
directory layout. The released configuration contract owns exact upstream behavior;
|
||||
the summaries here identify its consequences for the proposed PromptKit design.
|
||||
|
||||
This is a source-and-contract assessment, not a performance benchmark or full
|
||||
correctness audit. It does not claim measured allocation, latency, or line-count
|
||||
savings. Current code and canonical contracts remain authoritative until an
|
||||
approved implementation changes them.
|
||||
|
||||
## 1. Capability Inventory
|
||||
|
||||
| Preserve the ability to… | Replaceable mechanics and representation |
|
||||
| --- | --- |
|
||||
| Define named, versioned prompts, inspect their declared inputs, and select a version explicitly | Directory scans on each operation, YAML metadata prepasses, filename layout, constructor signatures |
|
||||
| Use inline and file-backed message templates, variables, input helpers, session templates, roles, and cache control | Timing of definition/content reads and private repository interfaces |
|
||||
| Configure named profiles, alias/inherit another profile, and override selected settings | Zero-as-absence fields, per-call chain traversal, raw YAML decoding types |
|
||||
| Supply embedded application defaults, operator definitions, maintained catalogs, and programmatic profiles | Repository-wrapper stacks and exact source-option API |
|
||||
| Register custom/local backends, use built-ins or endpoint-only profiles, and control backend capacity | Configuration DTO shapes and registry assembly order |
|
||||
| Prepare, inspect, execute, or freeze an execution for later single-claim execution | Source reloading during ordinary preparation; exact preparation error ordering |
|
||||
| Override per-run settings, credentials, reasoning, session, and output contract; append rendered messages | Public value signatures if presence or clearer configuration separation requires revision |
|
||||
| Supply inline/file artifacts and inject artifact readers, model clients, and HTTP clients | Assembly APIs; runtime collaborators need not be YAML-decodable |
|
||||
| Validate output, use provider-native structured output, bounded repair, transport retries, cancellation, and observation | Internal wiring, not the substantive execution behaviors |
|
||||
| Receive results, usage, identities, hashes, validation outcomes, and inspectable failures | Configuration error types and when definition failures appear |
|
||||
|
||||
Preserve independent profile and prompt inspection as useful operations. They
|
||||
need not retain the old guarantee that engine construction has never examined
|
||||
other definitions. Preserve custom backend identity and capacity behavior,
|
||||
optional upstream credentials, and the distinction between failed generated
|
||||
content and operational errors. This project does not authorize silently
|
||||
dropping execution features to simplify the configuration layer.
|
||||
|
||||
Add explicit configuration inspection for consumers implementing operator-facing
|
||||
CLI commands or debug output. This is a new capability motivated by this adoption,
|
||||
not an assertion that the current prompt/profile inspection methods already
|
||||
provide configuration provenance.
|
||||
|
||||
## 2. Responsibility Mapping
|
||||
|
||||
| Concern | Recommended owner and disposition |
|
||||
| --- | --- |
|
||||
| YAML grammar, scalar interpretation, one-document enforcement, unknown fields, typed ranges, null legality | Delegate to ConfigKit; remove corresponding PromptKit parser logic and redundant parser tests |
|
||||
| Field presence, source coordinates, and configuration provenance | Delegate to ConfigKit; retain its inspection state with the loaded snapshot for explicit operator queries, separate from execution-ready values |
|
||||
| Operator-facing configuration inspection | Provide a PromptKit consumer API reusing ConfigKit inspection facilities; applications own CLI commands, presentation, and disclosure policy |
|
||||
| File/directory/embedded source selection and definition ID indexing | Retain a small PromptKit collection loader; ConfigKit loads documents, not a prompt catalog |
|
||||
| Configuration filesystem sources, relative imports, and generic layer origins | Delegate to the prerequisite ConfigKit APIs; do not maintain parallel reader-based loading machinery |
|
||||
| Definition discovery and relative message-content references | Retain PromptKit's domain resource adapters, using ConfigKit source context and standard `io/fs` facilities where sufficient |
|
||||
| Profile-source priority and combination | PromptKit selects tiers; ConfigKit owns strict within-tier composition and recursive cross-tier overlay; eliminate repository fallback and whole-definition shadowing |
|
||||
| Named profile inheritance | Delegate sibling graph resolution, combination, and derivation to ConfigKit's `extends`; remove the custom profile resolver |
|
||||
| Backend selection and domain normalization | Retain PromptKit domain rules; use recorded application resolution for configuration transformations not already handled by structural inheritance |
|
||||
| LLM setting bounds, role/session normalization, output rules, reserved provider fields | Retain shared PromptKit domain owners; ConfigKit validation callbacks may invoke them |
|
||||
| Definition defaults | Use ConfigKit defaults where they do not erase inheritance presence; apply effective execution defaults after inheritance |
|
||||
| Programmatic configuration | Prefer labeled mapping input with explicit presence; evaluate whether typed PromptKit convenience constructors justify conversion code, without YAML round trips or zero-value inference |
|
||||
| Runtime clients, observers, artifact readers, request data | Retain normal Go injection and request APIs; do not load them through ConfigKit |
|
||||
| Credentials | Retain optional environment-name and direct request-key semantics; do not replace these with ConfigKit's required typed secret resolution |
|
||||
| Catalog manifest, ownership, reserved names, secret-safety checks | Retain the catalog adapter; reuse the common ConfigKit profile decoder |
|
||||
| JSON Schema resources and generated-output validation | Retain JSON Schema implementation; these are not application configuration documents |
|
||||
| Application imports, CLI overrides, deployment overlays, secrets-directory bootstrap | Leave with consuming applications unless a concrete PromptKit need is accepted |
|
||||
| Generic source/decoding diagnostics | Reuse ConfigKit; add only PromptKit identity/context and domain relationship diagnostics |
|
||||
|
||||
## 3. Recommended Target Loading Model
|
||||
|
||||
### Definition Snapshot
|
||||
|
||||
Create one immutable definition snapshot during engine construction. Discover
|
||||
the explicitly supplied sources, assemble their contributions through ConfigKit,
|
||||
decode the effective configuration, normalize domain values, and resolve relationships
|
||||
before publishing the engine. No provider call or credential availability check
|
||||
belongs in this phase. Publish no partially initialized engine on failure.
|
||||
|
||||
Prefer keyed collections, such as `profiles.<name>`, assembled before one typed
|
||||
loading lifecycle instead of independently decoding each profile and recombining
|
||||
the results. Multiple files can contribute to those collections; this does not
|
||||
require a monolithic file. Reconsider the previous one-typed-document-per-definition
|
||||
recommendation. Settle prompt ID/version representation and catalog format migration
|
||||
in the feature roadmap. Support OS directories, individual files, embedded/custom
|
||||
`fs.FS` sources, and programmatic input through the new source model; source selection
|
||||
and definition discovery remain PromptKit concerns, not a second composition engine.
|
||||
|
||||
ConfigKit expands structural inheritance before typed defaults/decoding. Perform
|
||||
any remaining domain-specific resolution through its recorded lifecycle before
|
||||
semantic validation and result publication. Reuse its presence and provenance;
|
||||
do not add a generic merge/defaulting pass or reconstruct origins after loading.
|
||||
Runtime client assembly and resource interpretation remain outside typed decoding.
|
||||
|
||||
Load and freeze referenced message text with definitions. Validate static prompt
|
||||
structure and template syntax at loading, using the same template function setup
|
||||
as execution; render only when request variables and artifacts exist. Preserve
|
||||
source-relative content resolution and containment without expanding the threat
|
||||
model. A snapshot is a set of captured reads, not an atomic filesystem transaction.
|
||||
|
||||
Resolve each effective content reference using its contributing source's context,
|
||||
not the highest tier's directory. Use ConfigKit's released direct context
|
||||
query rather than a consumer-maintained filesystem registry. Composite origins
|
||||
must not be assigned an arbitrary directory; ambiguous or absent context needs
|
||||
an actionable resource error or explicit consumer-supplied context. ConfigKit
|
||||
does not snapshot external text: PromptKit opens it explicitly and freezes it
|
||||
before engine publication. Caller-owned filesystem lifetimes remain caller-owned;
|
||||
close resource handles acquired by PromptKit without retaining ConfigKit load handles.
|
||||
|
||||
Keep request artifacts and schema compilation at preparation. A request can
|
||||
replace the output contract, so an unused default schema should not need eager
|
||||
compilation. Prepared execution continues to freeze its effective resources.
|
||||
The snapshot covers definitions/message text, not environment values, request
|
||||
files, every schema file, or execution state.
|
||||
|
||||
### Composition And Validation
|
||||
|
||||
Use explicit source categories in the existing useful priority order:
|
||||
programmatic profiles, operator source, application fallback, PromptKit-owned
|
||||
profiles, then external catalogs. Express these as explicit ordered tiers, not
|
||||
source discovery order. Within a tier, mappings compose recursively and overlapping
|
||||
non-mapping contributions conflict, even when equal. Disjoint contributions to
|
||||
the same profile may therefore combine; do not preserve a blanket duplicate-ID
|
||||
rejection. Across tiers, mappings overlay recursively, retaining omitted lower
|
||||
children. Effective prompt identities/versions must remain addressable, with an
|
||||
omitted version ambiguous when multiple versions exist; finalize their keyed
|
||||
representation without reintroducing whole-definition shadowing.
|
||||
|
||||
All sources undergo syntax, acquisition, and composition checks; higher tiers
|
||||
cannot hide those failures. Decode and validate effective values after assembly,
|
||||
not every intermediate tier against the final typed schema. A replaced lower
|
||||
value may consequently have a type that would not fit the final schema. An unused
|
||||
invalid effective definition still fails construction. Catalog ownership,
|
||||
reserved-name, and secret-safety checks remain distinct source-policy requirements,
|
||||
not a reason to impose intermediate typed validation on ordinary inputs.
|
||||
|
||||
Preserve profile templates that become usable through descendants or runtime
|
||||
overrides: eager loading must not require every consumer profile to be an
|
||||
independently runnable endpoint/model pair. Validate supplied fields and graph
|
||||
integrity eagerly (ConfigKit handles missing bases and cycles), and check final execution
|
||||
requirements after runtime selection/overrides. Prompt default-profile references
|
||||
can be checked against the assembled collection; a prompt with no default remains
|
||||
valid for callers that supply a profile explicitly.
|
||||
|
||||
ConfigKit's ordering is typed defaults, source-underlay tiers, strictly composed
|
||||
main input, selected overlay, then CLI overrides. PromptKit's API must map its
|
||||
source priorities explicitly onto supported roles; do not mislabel programmatic
|
||||
input as CLI flags. Underlay-only loading avoids inventing a main document.
|
||||
Imports remain opt-in, and the named-overlay convenience need not be exposed by
|
||||
PromptKit merely because ConfigKit supports it.
|
||||
|
||||
A prompt's `version` is a domain identifier, not automatically a ConfigKit
|
||||
application-schema version. Do not configure `VersionRule` for prompt versions.
|
||||
If a separate configuration schema version is later needed, respect ConfigKit's
|
||||
designated main-input ownership and pre-underlay check timing; do not assume
|
||||
last-wins version selection across catalogs.
|
||||
|
||||
### Source Presence And Released Sibling Inheritance
|
||||
|
||||
Adopt ConfigKit source-layer semantics: absence retains lower input; explicit
|
||||
zero, false, and empty strings replace it; mappings combine recursively; empty
|
||||
mappings retain lower children; sequences replace as units; null explicitly
|
||||
replaces, subject to final schema legality. No reset/delete operator is assumed.
|
||||
Preserve presence through domain resolution, backend/framework
|
||||
default application, request overrides, and outbound request construction.
|
||||
Otherwise accepting `temperature: 0` during decoding would accomplish nothing.
|
||||
|
||||
Use ConfigKit metadata for both file and programmatic input. A final Go value
|
||||
alone does not distinguish absent from explicit zero. Avoid eagerly defaulting
|
||||
profile fields in ways that erase that distinction before relationship resolution.
|
||||
|
||||
Provider `extra_params` maps combine recursively across source tiers too: `{}`
|
||||
does not clear lower keys. Retire the proposed atomic source-map replacement and
|
||||
null-as-unspecified conventions; do not emulate them after loading. An operator
|
||||
overriding only `model` also retains a lower `extends` declaration and other
|
||||
omitted settings. Diagnose and document these consequences explicitly.
|
||||
|
||||
Replace `base_profile` with `extends` on sibling profile mappings. A string names
|
||||
one exact immediate sibling; null cancels the relationship while preserving the
|
||||
child's own settings. Omission retains a lower-tier directive. There is no ancestor
|
||||
scope search or cross-collection reference. Superseded references are history,
|
||||
not active relationships to validate; every effective relationship resolves,
|
||||
including profiles not selected for execution.
|
||||
|
||||
Children recursively overlay their resolved parents with the same value rules:
|
||||
zero/false/empty strings and sequences/null replace, mappings combine, and `{}`
|
||||
does not clear inherited keys. Inherited values count as explicit input for
|
||||
presence queries and absence-based defaults, even when false, zero, or null.
|
||||
Do not repeat inheritance in PromptKit's application-resolution callback. Use
|
||||
`Origin.Inheritance` and retained history for explicit operator inspection.
|
||||
Inherited resource references retain their declaring source's context.
|
||||
|
||||
Enclosing inheritance resolves before nested relationships; nested inherited
|
||||
directives resolve at each destination against its siblings. Defaults and
|
||||
application-derived values do not trigger another inheritance pass. Model this
|
||||
deliberately in PromptKit configuration rather than promising arbitrary graph
|
||||
resolution over runtime values. Partial bases must fit the typed schema and
|
||||
PromptKit's validation; ConfigKit grants them no special validation exemption.
|
||||
|
||||
`extends` is reserved in every input mapping, including provider-parameter maps,
|
||||
across all source kinds. There is no opt-in switch or ordinary-data escape in the
|
||||
released contract. Audit catalog and consumer payload needs before finalizing
|
||||
their format; a literal provider field with that name cannot silently pass through
|
||||
the configuration loader. Do not invent an encoding workaround in this assessment.
|
||||
|
||||
The existing request-level override API remains a separate runtime boundary.
|
||||
The feature roadmap must review its interaction with configured values without
|
||||
silently subjecting runtime request data to configuration inheritance directives.
|
||||
|
||||
### Reloading, Errors, And Application Integration
|
||||
|
||||
Later definition edits take effect through a newly constructed engine. Do not add
|
||||
watchers, mutable registry reloads, or a second lazy compatibility mode. Applications
|
||||
choose how to swap engines and let in-flight work finish. Old prepared handles
|
||||
remain associated with their original engine. Simultaneously active engines have
|
||||
separate backend capacity pools; migration guidance must not imply a global limit
|
||||
across a reload overlap.
|
||||
|
||||
Return configuration failures from construction with file/path/location and
|
||||
useful definition identity where available. Recommend preserving ConfigKit's
|
||||
public `LoadError` through wrapping for `errors.As`, together with a PromptKit
|
||||
configuration category; do not copy its entire diagnostic API into the facade.
|
||||
Domain-only relationship failures remain PromptKit errors with relevant source
|
||||
locations. Start with deterministic fail-fast loading across files and ConfigKit's
|
||||
existing multi-diagnostic behavior within a load, rather than inventing a batch
|
||||
diagnostic subsystem. Do not expose raw credentials or add implicit logging.
|
||||
|
||||
Runtime failures continue to distinguish invalid requests, missing selections,
|
||||
rendering/artifact/schema errors, provider/transport errors, capacity rejection,
|
||||
and content-validation results. Cancellation remains effective at
|
||||
PromptKit-controlled boundaries; ConfigKit's current synchronous `Load` has no
|
||||
context argument, so do not promise interruption inside it or create goroutines
|
||||
to simulate that guarantee.
|
||||
|
||||
Applications using ConfigKit can present the same underlying loading diagnostics
|
||||
for their application settings and PromptKit files. They can also translate
|
||||
application-owned typed settings into PromptKit profiles/backends. Do not require
|
||||
applications to encode Go clients into YAML or adopt ConfigKit merely to call
|
||||
PromptKit's runtime API. The operator-inspection capability below supplies the
|
||||
concrete consumer purpose for exposing effective configuration and its provenance.
|
||||
|
||||
### Explicit Configuration Inspection
|
||||
|
||||
Provide a consumer-facing API for inspecting effective configuration and its
|
||||
provenance so applications can implement targeted CLI commands or debug output.
|
||||
Keep three activities distinct:
|
||||
|
||||
- **Construction:** load and resolve configuration, retain ConfigKit's provenance
|
||||
and inspection state, and publish execution-ready definitions. Resolve any
|
||||
presence and origin information needed for definition construction here.
|
||||
- **Ordinary execution:** use the ready definitions directly. Do not analyze
|
||||
configuration provenance, traverse origin/history records, reconstruct
|
||||
inheritance explanations, or render effective-configuration reports as part of
|
||||
a prompt run. Carry execution-relevant presence explicitly in resolved values
|
||||
rather than querying configuration metadata during each run.
|
||||
- **Explicit inspection:** query effective settings and their contributing
|
||||
sources or explanations only when a consumer invokes the inspection API.
|
||||
Inspection uses the same loaded configuration snapshot as the engine; it does
|
||||
not reread configuration files, open referenced resources, or contact providers.
|
||||
|
||||
Reuse ConfigKit's inspection and redaction facilities rather than maintaining a
|
||||
second provenance system. The API should make targeted queries practical; exact
|
||||
signatures, query scope, and safe result ownership belong in the feature roadmap.
|
||||
Do not imply that configuration inspection predicts all runtime request overrides
|
||||
or reveals the values of execution-time credentials. Existing prompt/profile
|
||||
inspection, prepared-run details, and execution observations have distinct purposes;
|
||||
configuration inspection does not replace them or add provenance to run events.
|
||||
|
||||
Applications decide whether and how to expose inspection through operator tools.
|
||||
PromptKit adds no CLI, implicit debug logging, or automatic explanation output.
|
||||
Redacted configuration can still contain sensitive application content, so callers
|
||||
retain responsibility for access control and disclosure. Retention has a memory
|
||||
cost, but explanation generation and provenance analysis stay off the run path.
|
||||
|
||||
## 4. Simplification Accounting
|
||||
|
||||
| Current implementation | Expected disposition |
|
||||
| --- | --- |
|
||||
| `internal/profile/filesystem_repository.go`: metadata scans, selected strict decode, stream-end checks, malformed-file selection arbitration | Replace with source acquisition, ConfigKit assembly/decoding, and indexed effective values; remove the metadata prepass and per-lookup filesystem path |
|
||||
| `internal/promptdef/filesystem_repository.go`: metadata selection and repeated decoding/content opening | Replace with eager prompt loads and an ID/version index; keep domain normalization and contained content reading |
|
||||
| Profile filesystem, `fs.FS`, eager, memory, and overlay repository variants | Collapse loading into source adapters plus immutable collections; preserve only interfaces that still serve a real boundary |
|
||||
| `internal/profile/resolving_repository.go` | Remove custom graph traversal, cycle detection, and zero-sentinel merging in favor of ConfigKit sibling inheritance; retain only PromptKit domain validation |
|
||||
| `internal/engine` raw repository-stack assembly | Replace with ConfigKit source tiers and one effective typed collection; engine capacity/client assembly remains |
|
||||
| `internal/catalog` profile metadata extraction | Obtain explicit field information from ConfigKit; retain manifests, ownership, namespace, connection-metadata and secret checks |
|
||||
| `internal/filecatalog` | Retain/reduce discovery and containment helpers actually needed by the eager loader; do not claim the entire package disappears |
|
||||
| `internal/domain` source YAML tags and runtime/source coupling | Separate configuration DTOs from execution values where presence requires it; reuse domain validation rather than duplicating it |
|
||||
| Direct `gopkg.in/yaml.v3` use for definition loading | Remove once all such consumers migrate; ConfigKit brings its own private YAML dependency |
|
||||
|
||||
New integration work is real: collection DTOs, ConfigKit calls and error wrapping,
|
||||
source-context access, collection indexing, programmatic mapping ergonomics, and
|
||||
conversion to runtime definitions. Catalog documents may need new keyed shapes;
|
||||
do not hide that migration behind a permanent second loader.
|
||||
Some defensive copies remain necessary at public/injected-client boundaries;
|
||||
ConfigKit's detached load result is not permission to expose shared engine maps.
|
||||
`internal/jsonvalue` still serves programmatic values, request parameters, and
|
||||
prepared schema ownership, so adoption does not eliminate it.
|
||||
|
||||
Maintenance savings depend on actually deleting the old lazy paths and their
|
||||
special-case tests. Move generic YAML/scalar behavior coverage to ConfigKit's
|
||||
ownership; PromptKit retains representative adapter tests plus domain, precedence,
|
||||
snapshot, inheritance, embedded-source, error, and execution integration tests.
|
||||
Do not duplicate ConfigKit's entire parser suite or discard execution regression
|
||||
coverage because construction fixtures need migration.
|
||||
|
||||
Runtime benefit: ordinary runs no longer rediscover definition directories,
|
||||
redecode definitions, reread template text, or traverse inheritance chains.
|
||||
Cost: construction loads even unused definitions and retains their text and
|
||||
resolved values. Retain ConfigKit inspection state needed by the explicit operator
|
||||
API without duplicating its history in runtime definitions or per-run values.
|
||||
These are reasoned tradeoffs, not measured performance
|
||||
claims. Adoption is not justified by the decoder replacement alone.
|
||||
|
||||
### Algorithmic Efficiency
|
||||
|
||||
Efficiency is an explicit design criterion for both the upstream work and this
|
||||
shared-library refactor. Prefer optimal or near-optimal algorithms relative to
|
||||
input and produced-data size, not a blanket O(n) claim with an undefined n.
|
||||
|
||||
- Acquire/parse each accepted source occurrence once, assemble input, and run
|
||||
one typed loading lifecycle rather than decoding intermediate tiers. Reuse
|
||||
captured content instead of reparsing it for metadata, selection, and validation.
|
||||
- Build identity indexes in one pass, with expected constant-time map lookup.
|
||||
Do not scan whole collections for every lookup or precedence decision.
|
||||
- Delegate inheritance expansion to ConfigKit once during loading rather than
|
||||
rebuilding its graph in PromptKit. Assess scaling against input, materialized
|
||||
inherited values, nested destination resolution, and retained history; do not
|
||||
claim unconditional O(V + E) total work from the count of profiles and edges.
|
||||
ConfigKit bounds expansion and retained evidence through its existing budgets;
|
||||
representative catalog sizes must fit them without new per-run processing.
|
||||
- Avoid whole-collection copying per layer, repeated serialization, and repeated
|
||||
prefix/history traversal. Analyze layer processing against total input nodes
|
||||
and produced values/provenance, including intentionally retained history.
|
||||
- Keep configuration discovery, decoding, and provenance analysis out of ordinary
|
||||
execution. Perform explanation work only on explicit inspection calls. Copy only
|
||||
where ownership requires it; do not trade caller isolation for speed.
|
||||
|
||||
These are design targets, not promised complexity for every operation. Sorting
|
||||
for deterministic discovery may require O(F log F) for F files, and processing
|
||||
message text costs work proportional to its bytes. Construction time, retained
|
||||
memory, and frequent-call costs all matter. Prefer straightforward indexes and
|
||||
clear immutable ownership over speculative global caches or intricate optimizers.
|
||||
The implementation plan should include focused benchmarks for consequential
|
||||
repeated-work risks, such as shared-ancestor profiles and many layers, rather than
|
||||
machine-dependent timing gates or benchmarks for every helper.
|
||||
|
||||
## 5. Upstream Readiness And Remaining Opportunities
|
||||
|
||||
Filesystem sources, underlays, programmatic mappings, source contexts, and sibling
|
||||
inheritance are released in v0.5.0. No additional upstream feature is currently a
|
||||
demonstrated prerequisite. Context-aware loading remains a possible future request
|
||||
if concrete consumers need it, not a reason to delay adoption. Existing loading
|
||||
is synchronous; do not simulate cancellation with detached background work.
|
||||
|
||||
Do not propose a public raw AST, permissive identity-only decoder, or dynamic
|
||||
subtree conversion solely to preserve PromptKit's current scanning design.
|
||||
Do not upstream backend/model selection, credential timing for inference,
|
||||
profile catalog reservations, or prompt-template functions.
|
||||
|
||||
ConfigKit and PromptKit both declare LGPL-3.0-or-later. Adoption must still update
|
||||
dependency notices under the OSS policy and inspect the exact selected release
|
||||
and its transitive notices. Their current Go minimums do not require a PromptKit
|
||||
toolchain increase. Use a released module dependency, not a sibling-directory
|
||||
replacement. Select v0.5.0 as the assessed adoption baseline; any newer selection
|
||||
needs its own review. This assessment changes neither dependencies nor licenses.
|
||||
|
||||
## 6. Decisions And Migration
|
||||
|
||||
The agreed direction now incorporates ConfigKit's v0.5.0 contracts. This supersedes
|
||||
the earlier recommendations for independently typed documents, whole-definition
|
||||
shadowing, intermediate typed validation, and atomic maps/null-as-unspecified at
|
||||
the source boundary.
|
||||
|
||||
1. **Loading lifetime and failure scope.** Keep eager construction and frozen
|
||||
definitions/content. Syntax/acquisition/composition failures remain unmaskable;
|
||||
typed and domain validation apply to effective values. A separate reusable
|
||||
public snapshot lifecycle remains possible but is not currently needed.
|
||||
2. **Source organization.** Prefer keyed collections composed across files and
|
||||
tiers before typed loading. Settle exact profile and prompt/version shapes,
|
||||
source-role mapping, and catalog migration in the feature roadmap. Independent
|
||||
typed loads followed by custom combination are not the selected direction.
|
||||
3. **Presence and relationships.** Accept recursive source-tier overlay, retained
|
||||
children for empty mappings, explicit null replacement, and effective metadata.
|
||||
Adopt built-in sibling inheritance, including reserved `extends`, recursive
|
||||
mapping combination, null cancellation, and inherited explicit presence. Do
|
||||
not preserve a parallel PromptKit structural resolver.
|
||||
4. **Diagnostic boundary.** Keep wrapping ConfigKit's public loading errors rather
|
||||
than mirroring them. Retain actionable domain relationship and resource errors
|
||||
without duplicating ConfigKit's generic diagnostic/provenance system.
|
||||
5. **Operator inspection.** Expose effective configuration and provenance through
|
||||
an explicit consumer API backed by the engine's loaded snapshot. Retain the
|
||||
required ConfigKit inspection state at construction; never analyze it during
|
||||
ordinary prompt execution. Finalize targeted query shapes and ownership in the
|
||||
feature roadmap, with presentation remaining application-owned.
|
||||
|
||||
Use released mapping-based programmatic input and direct filesystem resource-context
|
||||
queries. If real consumer examples show
|
||||
mapping construction to be repetitive, consider small typed PromptKit conveniences
|
||||
with explicit presence rather than requiring an upstream typed-ingestion API.
|
||||
|
||||
Remaining PromptKit decisions are the exact collection/version format, catalog
|
||||
migration, programmatic ergonomics, operator-inspection surface, and validation
|
||||
of partial base profiles. Also settle how operators discard unwanted provider
|
||||
settings or old relationships using the released null semantics without a hidden
|
||||
replacement pass. Check the reserved-key impact on arbitrary provider maps. These
|
||||
are integration decisions, not unresolved ConfigKit API proposals.
|
||||
|
||||
Expected consumer migration:
|
||||
|
||||
- Update definitions for ConfigKit's strict scalar rules, rejected YAML
|
||||
anchors/aliases/tags, input limits, and null/type checks. Quote prompt version
|
||||
strings rather than introducing a special numeric-version conversion merely
|
||||
for historical YAML coercion.
|
||||
- Move standalone definitions into the agreed keyed contribution format. Review
|
||||
same-name contributions: disjoint fields within a tier can combine, and higher
|
||||
tiers retain omitted lower fields, including `extends` references. Convert
|
||||
`base_profile` to sibling `extends`; use `extends: null` to cancel a lower-tier
|
||||
relationship, not to erase all settings already supplied to the child.
|
||||
- Review zero/false/empty/null values explicitly. Provider maps merge across
|
||||
tiers, `{}` does not clear them, and null replaces rather than means omission.
|
||||
Use labeled programmatic mappings or approved presence-aware conveniences.
|
||||
Apply those same combination rules to inherited settings. Rename ordinary
|
||||
configuration keys named `extends`; examine provider maps for cases where
|
||||
renaming would change their substantive meaning and resolve those explicitly.
|
||||
- Handle effective-definition errors at construction, fix syntax/composition
|
||||
failures even in overridden inputs, and deliberately rebuild/swap engines
|
||||
after definition changes.
|
||||
- Keep runtime credential lookup optional, avoid baking resolved secrets into
|
||||
definition snapshots, and preserve request-scoped key overrides.
|
||||
- Update maintained catalog data for collection shapes and any incompatible syntax;
|
||||
coordinate catalog releases before the PromptKit dependency update. Preserve
|
||||
maintained profile names and catalog ownership rules, not every old scalar
|
||||
spelling. Do not validate live provider/model availability.
|
||||
- Update error inspection, constructor/source options, and programmatic profile
|
||||
literals as required. Keep representative preparation, execution, inspection,
|
||||
custom backend, fallback, and embedded workflows in the migration guidance.
|
||||
|
||||
The subsequent feature roadmap should carry forward these choices, resolve the
|
||||
remaining PromptKit integration decisions, and specify the target end state.
|
||||
The staged implementation plan can now adopt released ConfigKit support without
|
||||
waiting for upstream feature work, coordinating any catalog format releases before
|
||||
their consumption. Cover deletions/replacements, contract tests, canonical documentation,
|
||||
examples, attribution, and minor-release migration notes. Neither document should
|
||||
become a reason to preserve obsolete compatibility machinery.
|
||||
339
docs/roadmap/configkit.md
Normal file
339
docs/roadmap/configkit.md
Normal file
@@ -0,0 +1,339 @@
|
||||
# ConfigKit Adoption
|
||||
|
||||
Status: agreed feature roadmap. ConfigKit v0.5.0 supplies the required upstream
|
||||
capabilities. The [implementation plan](implementation.md) owns execution steps.
|
||||
The [configuration-boundary ADR](../adr/0001-configuration-boundary.md) records
|
||||
the handoff decision. This document owns the intended end state, not
|
||||
implementation authorization.
|
||||
|
||||
## Purpose
|
||||
|
||||
Use ConfigKit to standardize PromptKit's configuration semantics and simplify
|
||||
its loading machinery, replacing custom logic wherever ConfigKit is the
|
||||
appropriate owner. Preserve substantive consumer capabilities, not historical
|
||||
file syntax, loading order, or Go signatures. Pre-1.0 breaking changes are
|
||||
acceptable and must have migration guidance.
|
||||
|
||||
**This is an intentional breaking change. We are intentionally replacing PromptKit's configuration handling code in favor of adopting ConfigKit's simpler, standardized API. Our goal is to arrive at a new version of PromptKit that is leaner, simpler, and uses a ConfigKit-standard configuration syntax. We MUST NOT give any weight to preserving arbitrary implementation details from PromptKit's current codebase--those should be discarded in favor of the simplest viable implementation that supports PromptKit's substantive LLM-focused consumer capabilities.**
|
||||
|
||||
Applications using both libraries should share configuration interpretation,
|
||||
layering, inheritance, presence, and inspection semantics. PromptKit should
|
||||
concentrate on LLM-specific validation, resource interpretation, and execution.
|
||||
The [adoption assessment](configkit-assessment.md) records the supporting analysis;
|
||||
this roadmap owns the agreed feature end state. Where the earlier assessment
|
||||
differs, this roadmap and the maintainer's direction above take precedence.
|
||||
|
||||
Do not preserve source-option wrappers, repository interfaces, incidental error
|
||||
ordering, or old field shapes solely because they exist today. Reuse existing
|
||||
code only when it remains the simplest appropriate owner of a substantive
|
||||
capability. Migration documentation replaces compatibility machinery, not vice versa.
|
||||
|
||||
## Baseline And Dependency
|
||||
|
||||
The current PromptKit runtime baseline is v0.11.2. Its
|
||||
[source implementation](../internal/sources.md), [format contract](../formats.md),
|
||||
and [consumer guide](../consumers/pkg-promptkit.md) describe existing behavior,
|
||||
not the target described here.
|
||||
|
||||
The reviewed ConfigKit release is v0.5.0 at
|
||||
`dc09959cca4e105f49ec316c6d4ba585bd2340bd`. It provides native `fs.FS` sources,
|
||||
`LoadLayered`, source-backed underlays, labeled programmatic mappings, passive
|
||||
resource contexts, and built-in sibling inheritance. Its released configuration
|
||||
contract and GoDoc own exact upstream semantics. No further upstream feature is
|
||||
currently a demonstrated prerequisite. Review any newer version before selection.
|
||||
|
||||
Adopt a released Go module, not a local `replace` or workspace requirement.
|
||||
Update attribution and transitive license notices under the OSS policy. This
|
||||
feature does not change either project's license or require a Go minimum increase
|
||||
relative to the reviewed versions.
|
||||
|
||||
## Substantive Capabilities To Preserve
|
||||
|
||||
- Named/versioned prompts, declared-input inspection, templates, inline and
|
||||
referenced message content, variables, input helpers, roles, and cache control.
|
||||
- Maintained catalogs, application defaults, operator configuration, programmatic
|
||||
profiles, aliases/inheritance, built-in/custom/local backends, and endpoint-only
|
||||
profiles.
|
||||
- Preparation without generation, ordinary execution, and frozen single-claim
|
||||
prepared execution; independent prompt and profile inspection.
|
||||
- Runtime execution-setting, reasoning, session, credential, and output-contract
|
||||
overrides; appended already-rendered messages.
|
||||
- Inline/file input artifacts and injected artifact, model, and HTTP collaborators.
|
||||
- Backend capacity management, cancellation, transient retries, structural output
|
||||
validation, provider-native structured output, bounded repair, and observations.
|
||||
- Results, effective identities/settings, usage, hashes, and meaningful errors.
|
||||
|
||||
Retain optional environment credentials and direct request credentials. ConfigKit
|
||||
secret resolution must not introduce a new requirement that an upstream API key
|
||||
exist. Domain validation must not probe provider/model availability.
|
||||
|
||||
## Target Configuration Model
|
||||
|
||||
### One Shared Loading Lifecycle
|
||||
|
||||
Construction supplies selected configuration inputs to ConfigKit, which composes,
|
||||
overlays, expands inheritance, and decodes them. PromptKit interprets and validates
|
||||
the final effective configuration at the ConfigKit handoff and produces
|
||||
execution-ready definitions. ConfigKit's effective-value resolution and validation
|
||||
hooks may implement that handoff. Publish no partially initialized engine on failure.
|
||||
|
||||
ConfigKit owns generic parsing, combination, presence, origins, and inheritance.
|
||||
PromptKit owns input selection/discovery, definition indexing, LLM-specific rules,
|
||||
and translation of final values into runtime settings. Input adapters select or
|
||||
encode contributions; they do not inspect or validate raw source contents.
|
||||
Effective presence/null information and resource context supplied by ConfigKit
|
||||
support interpretation at handoff. Historical inspection remains an explicitly
|
||||
invoked operator capability, separate from domain enforcement.
|
||||
Runtime clients and callbacks remain normal Go dependencies, not YAML types.
|
||||
|
||||
Use `backends.<id>`, `profiles.<id>`, and `prompts.<id>.<version>` keyed mappings
|
||||
assembled before typed decoding rather than independently decoding definitions and combining them
|
||||
afterward. Keys own identity; do not duplicate `id`/`version` fields. These paths
|
||||
describe mapping levels, not dot-splitting of literal IDs or version keys.
|
||||
Multiple files can contribute to a collection; a monolithic file is not required.
|
||||
When a prompt ID has multiple versions, selection without a version remains
|
||||
ambiguous. This shape fits sibling inheritance without extra identity reconciliation.
|
||||
|
||||
Supply explicit source groups through a small construction surface compatible
|
||||
with ConfigKit's `Input`/underlay concepts. Offer convenient file/directory/embedded
|
||||
entry points where useful, not a parallel set of historical loading semantics.
|
||||
|
||||
Accept ConfigKit's labeled mapping sources for programmatic configuration. Add
|
||||
only small typed conveniences justified by the minimal local-backend and
|
||||
application-profile workflows. Helpers encode explicit presence and feed the same
|
||||
loading lifecycle; they neither merge configurations nor infer omission from zeros.
|
||||
Backend helpers also produce inputs to this lifecycle. Backend definitions,
|
||||
including catalog connection and capacity settings, cross the same ConfigKit
|
||||
handoff as prompts and profiles. The immutable runtime registry is built from
|
||||
effective backend values. HTTP/model clients and other runtime collaborators
|
||||
remain outside typed configuration.
|
||||
|
||||
Keep the public package a deliberate facade and implementation private. Do not
|
||||
expose internal engine/domain types or introduce a parallel generic loader merely
|
||||
to preserve old source options. ConfigKit's supported public types may be used
|
||||
where intentional integration is simpler than maintaining equivalent wrappers.
|
||||
|
||||
### Sources, Tiers, And Presence
|
||||
|
||||
Support ordinary files/directories, embedded or caller-supplied `fs.FS`, and
|
||||
programmatic definitions. Explicitly selected directories may be discovered by
|
||||
PromptKit; ConfigKit handles their configuration contributions. Do not infer
|
||||
priority from traversal order.
|
||||
|
||||
Retain the useful priority categories, lowest to highest: external
|
||||
catalogs, PromptKit-owned profiles, application fallback, operator configuration,
|
||||
and programmatic configuration. Express intentional priority as ordered tiers.
|
||||
Within each tier, independent contributions compose strictly; overlapping
|
||||
non-mapping values conflict, even when equal. Disjoint fields under the same
|
||||
identity can combine. Across tiers, mappings recursively overlay.
|
||||
These rules also govern backend configuration; consumers can add backend IDs or
|
||||
overlay existing definitions through their selected tiers. Registry construction
|
||||
validates the resulting values, not the identity of their contributors.
|
||||
|
||||
Use ConfigKit's source roles honestly: underlay groups are ordered defaults,
|
||||
main input overlays underlays, and optional named overlay/CLI facilities retain
|
||||
their upstream meaning. Underlay-only loading is supported; do not manufacture
|
||||
a main document or disguise programmatic values as CLI flags. Exposing every
|
||||
ConfigKit convenience through PromptKit is not a requirement.
|
||||
|
||||
Adopt the shared value rules: omission retains lower input; explicit zero, false,
|
||||
empty string, sequence, and null replace; mappings combine recursively; empty
|
||||
mappings do not clear lower children. Null remains explicit and subject to the
|
||||
final schema. Provider-parameter maps obey these rules too. No hidden whole-profile
|
||||
replacement, arbitrary merge callback, or new reset/delete operator belongs here.
|
||||
|
||||
Preserve presence into execution-ready settings so an explicit zero does not
|
||||
disappear during backend/default resolution or outbound request construction.
|
||||
Use ConfigKit metadata at construction, not as a per-run lookup mechanism.
|
||||
|
||||
### Sibling Inheritance
|
||||
|
||||
Use ConfigKit's reserved `extends` rather than PromptKit's `base_profile` and
|
||||
custom resolver. A string identifies an immediate sibling mapping; null cancels
|
||||
the relationship without deleting the child's own settings. Layering completes
|
||||
before inheritance. Only effective relationships resolve, but all of them do,
|
||||
including entries never selected for execution.
|
||||
|
||||
Children recursively overlay resolved parents. Inherited values count as explicit
|
||||
input. Origins and resource context remain attached to their declaring source.
|
||||
Nested relationships follow ConfigKit's destination-local rules; defaults and
|
||||
application-derived values do not trigger another inheritance pass.
|
||||
|
||||
The keyword is reserved throughout configuration, including dynamic provider
|
||||
maps; it is not opt-in. Keep runtime request provider-parameter maps ordinary
|
||||
JSON-compatible data outside configuration parsing. Consumers needing a literal
|
||||
provider field named `extends` supply it at that runtime boundary. Do not rename
|
||||
the outbound key or introduce a second opaque configuration format to escape it.
|
||||
Document and test this distinction, including any affected maintained payloads.
|
||||
|
||||
### Validation And Errors
|
||||
|
||||
ConfigKit owns acquisition, syntax, composition, inheritance, and typed-decoding
|
||||
failures. PromptKit preserves those errors and validates final effective domain
|
||||
values. An unused invalid effective definition fails construction; a lower-layer
|
||||
value legitimately replaced by ConfigKit is not a separate validation target.
|
||||
This boundary applies equally to application and maintained catalog configuration.
|
||||
PromptKit does not perform catalog preflight, raw-source or intermediate-state
|
||||
validation, or history-based enforcement, including equivalent test-only checks.
|
||||
Catalog maintainers own publishing checks in their respective repositories.
|
||||
|
||||
Domain validators remain shared with runtime request validation where appropriate.
|
||||
Allow schema-valid partial profiles without an `abstract` flag. Validate supplied
|
||||
domain values at loading; final runnable-target requirements are checked after
|
||||
profile selection and runtime overrides at execution preparation.
|
||||
Do not require credentials, artifacts, or provider connectivity during construction.
|
||||
|
||||
Preserve absence versus explicit null in resolved settings. Null on an optional
|
||||
provider setting deliberately suppresses it; it must not restore the same setting
|
||||
from backend defaults. A null provider map suppresses the configured lower map,
|
||||
while null entries inside a provider map remain literal JSON null values. This
|
||||
supports reusable bases and deliberate removal of stale settings without changing
|
||||
ConfigKit's merge rules. Framework-required settings such as timeouts have explicit
|
||||
field-specific null semantics rather than blanket nil defaulting.
|
||||
|
||||
Preserve ConfigKit public loading errors through wrapping for `errors.As`, with a
|
||||
PromptKit configuration category. Use source-aware domain/resource failures where
|
||||
ConfigKit is not the owner. Do not mirror its diagnostic framework or promise
|
||||
aggregation beyond what the underlying operation provides. Runtime operational
|
||||
errors remain distinct from generated-content validation failures.
|
||||
|
||||
Prompt versions are domain identifiers, not ConfigKit `VersionRule` declarations.
|
||||
Do not introduce an application schema-version policy merely to adopt ConfigKit.
|
||||
|
||||
## Snapshot, Resource, And Execution Boundaries
|
||||
|
||||
The engine captures effective definitions and referenced message text during
|
||||
construction. Validate static template syntax then; render with request data
|
||||
later. Resolve content references from the effective field's contributing source
|
||||
context, not the last layer's directory. Ambiguous/contextless resource references
|
||||
need an explicit source context or an actionable error, not an invented directory.
|
||||
|
||||
ConfigKit returns passive resource contexts, not snapshots of referenced text.
|
||||
PromptKit explicitly opens and closes its resource handles. Caller filesystem
|
||||
ownership remains unchanged. Preserve appropriate source containment without
|
||||
claiming that every `fs.FS` is a security sandbox. Captured reads are not an atomic
|
||||
transaction over a concurrently changing filesystem.
|
||||
|
||||
Definition changes require new engine construction. No watcher, background
|
||||
refresh, or lazy compatibility mode is added. Applications own engine replacement;
|
||||
prepared handles remain bound to their original engine and overlapping engines
|
||||
retain separate capacity pools.
|
||||
|
||||
Request artifacts and schema preparation remain invocation work. A request can
|
||||
replace the output contract, so construction need not compile unused schemas.
|
||||
Prepared execution continues to freeze its effective resources. Environment
|
||||
credential lookup remains at the established execution/preparation boundary,
|
||||
not frozen into the configuration snapshot.
|
||||
|
||||
Ordinary execution uses indexed ready definitions and resolved presence values.
|
||||
It performs no configuration discovery, parsing, inheritance traversal, or
|
||||
provenance analysis. Context cancellation remains cooperative at controlled
|
||||
boundaries; do not add detached work to pretend synchronous ConfigKit loading is
|
||||
instantly interruptible.
|
||||
|
||||
## Explicit Operator Inspection
|
||||
|
||||
Add an API consumers can use to implement targeted CLI commands or debug output
|
||||
for effective configuration, contributing sources, and explanations. Retain
|
||||
ConfigKit inspection state with the engine's configuration snapshot and reuse
|
||||
its redaction and provenance facilities.
|
||||
|
||||
Inspection is explicitly invoked and uses that snapshot without reloading files,
|
||||
opening resources, or contacting providers. It does not predict every runtime
|
||||
override, reveal execution-time credential values, or replace prepared-run details
|
||||
and execution observations. No provenance traversal, explanation generation, or
|
||||
configuration-report rendering occurs during ordinary prompt runs.
|
||||
|
||||
Expose a read-only inspection view backed by retained ConfigKit state, with
|
||||
targeted path queries, source/history information, and explicit redacted effective
|
||||
and explanation output. Reuse suitable public ConfigKit inspection values rather
|
||||
than mirroring them. Do not expose mutable runtime definitions, resource capabilities
|
||||
merely for diagnostics, or request credentials. The scope is generic targeted
|
||||
inspection, not just narrow profile/prompt origin summaries.
|
||||
|
||||
PromptKit ships no CLI and emits no automatic debug reports. Applications own
|
||||
presentation, access control, and disclosure; redaction does not make all prompt
|
||||
or configuration content safe to publish.
|
||||
|
||||
## Required Refactoring And Coordination
|
||||
|
||||
- Replace profile/prompt metadata prepasses, selected-file decoding, and per-call
|
||||
scans with ConfigKit collection loading and immutable indexed definitions.
|
||||
- Remove the custom profile inheritance resolver and error-driven repository
|
||||
overlay stack. Do not retain an old-format compatibility engine alongside the
|
||||
new semantics.
|
||||
- Route catalog and programmatic backend definitions through ConfigKit; retire
|
||||
the separate backend-manifest decoder and pre-handoff registration validation.
|
||||
Retain backend endpoint, capacity, and provider-parameter domain validation on
|
||||
the final effective values.
|
||||
- Retain only needed discovery and message-resource adapters; remove direct YAML
|
||||
dependency usage once no remaining owner needs it. Preserve JSON Schema,
|
||||
runtime JSON-value ownership, domain validation, and execution collaborators.
|
||||
- Adapt maintained catalogs and PromptKit-owned definitions to the agreed keyed
|
||||
format and released inheritance semantics, including keyed backend assets.
|
||||
Preserve maintained name commitments and the intended shipped backend settings.
|
||||
Catalog source-policy enforcement belongs to catalog maintainers. Coordinate
|
||||
compatible catalog releases before consuming the changed asset format.
|
||||
Publish new keyed assets directly in the existing data-only catalog projects,
|
||||
recording the format transition rather than maintaining duplicate legacy assets
|
||||
or a permanent conversion adapter. Catalog maintainers select release/version
|
||||
handling under their own policies; old consumers must not be assumed capable
|
||||
of decoding the new shape. Pin the compatible released versions in PromptKit.
|
||||
This document requests no immediate changes in those repositories.
|
||||
- Keep configuration conversion, normalization, and inspection connected to
|
||||
ConfigKit's lifecycle instead of reconstructing effective values/history after
|
||||
loading. Translate once into execution-owned values with necessary copying.
|
||||
- Update canonical formats, consumer guidance, GoDoc, internal inventory, examples,
|
||||
architecture boundaries, dependencies, and OSS notices when implementation lands.
|
||||
|
||||
## Efficiency And Validation Expectations
|
||||
|
||||
Target linear or near-linear work relative to actual input, output, and retained
|
||||
history, accounting separately for sorting, resource reads, and domain work.
|
||||
Acquire/parse each accepted source occurrence once, avoid decoding intermediate
|
||||
tiers, and index definition selection. Do not duplicate inheritance expansion or
|
||||
copy complete accumulated collections for every added layer.
|
||||
|
||||
Materialized inherited data and provenance can exceed source size. Exercise real
|
||||
catalog shapes within ConfigKit's budgets; do not claim unconditional O(n) runtime
|
||||
or memory. Prefer clear ownership and straightforward indexes over global caches
|
||||
or elaborate optimization. Operator inspection may perform work proportional to
|
||||
the requested report; that work remains off the execution path.
|
||||
|
||||
Tests should use real ConfigKit with offline sources and synthetic provider
|
||||
collaborators. PromptKit owns its adapter, domain, lifecycle, and execution tests,
|
||||
not a duplicate ConfigKit parser/graph suite. Retire tests whose only purpose was
|
||||
the removed loading machinery or source-policy enforcement. Exercise catalog
|
||||
integration through the final assembled configuration and resulting behavior.
|
||||
Preserve meaningful execution regressions.
|
||||
|
||||
Representative acceptance evidence must cover:
|
||||
|
||||
- Minimal local-backend construction and execution without elaborate configuration.
|
||||
- Embedded defaults plus operator and programmatic contributions, including
|
||||
accurate inherited relative-resource origins.
|
||||
- Profile inheritance, changing model/backend, canceling old relationships, and
|
||||
discarding unwanted settings with documented semantics.
|
||||
- Explicit zero/false/null behavior reaching execution, construction failures,
|
||||
independent snapshots, and unchanged substantive preparation/execution features.
|
||||
- Targeted inspection with useful redacted origins and no provenance work during
|
||||
ordinary runs.
|
||||
|
||||
Use focused scaling/allocation benchmarks for large collections, shared bases,
|
||||
and many tiers where they expose repeated-work risks. Do not add timing-based
|
||||
release gates or provider-network tests.
|
||||
|
||||
## Migration And Completion
|
||||
|
||||
Ship as an intentionally incompatible minor release with concrete migration
|
||||
guidance. Cover keyed files, `base_profile` to `extends`, reserved-key behavior,
|
||||
recursive maps, explicit null versus omission, changed error timing, programmatic
|
||||
construction, backend layering in place of registration-only restrictions, and
|
||||
explicit engine replacement. Explain that overriding a model
|
||||
does not discard lower endpoint, reasoning, provider, or inheritance settings.
|
||||
|
||||
The end state is one shared configuration model with the old loading/resolution
|
||||
machinery removed, useful operator inspection, and all substantive execution
|
||||
workflows still available. The six previously open design choices have been
|
||||
approved and incorporated into the owning sections above. This is not a
|
||||
compatibility-preserving reinterpretation of the existing configuration engine.
|
||||
@@ -38,33 +38,8 @@ consumers.
|
||||
|
||||
## Ideas
|
||||
|
||||
### Public bounded output repair
|
||||
|
||||
After the codebase-audit remediations are complete, Promptkit should make its
|
||||
bounded output-repair capability available through the public engine. A
|
||||
consumer should be able to request a limited number of corrective generation
|
||||
attempts when JSON or JSON Schema output fails content validation, without
|
||||
having to reproduce Promptkit's generation, validation, capacity, and result-
|
||||
accounting orchestration.
|
||||
|
||||
- Repair is validation recovery, not a general provider retry, failover, or
|
||||
backoff policy. Transport failures, cancellation, and operational schema or
|
||||
validation errors must retain their ordinary error behavior.
|
||||
- Repair must stop after the first valid result or the configured attempt
|
||||
bound. Exhausting the bound should preserve the final invalid result and its
|
||||
validation diagnostics rather than inventing success.
|
||||
- Initial generation and every repair attempt must use the same resolved
|
||||
backend, effective execution settings and presence semantics, session,
|
||||
credential boundary, structured-output contract, and backend-capacity
|
||||
policy.
|
||||
- Results should report the number of repair attempts and cumulative usage for
|
||||
every model call made by the run.
|
||||
- Ordinary and prepared execution should expose coherent behavior, including
|
||||
cancellation, frozen prepared state, error identity, and capacity lifetime.
|
||||
|
||||
Select this work only after the accepted audit findings affecting shared
|
||||
execution invariants, validation, orchestration, transport, and repair
|
||||
internals have been remediated.
|
||||
No ideas are currently awaiting selection. Active feature work belongs in its
|
||||
focused roadmap rather than this catalog.
|
||||
|
||||
## Entry Format
|
||||
|
||||
|
||||
642
docs/roadmap/implementation.md
Normal file
642
docs/roadmap/implementation.md
Normal file
@@ -0,0 +1,642 @@
|
||||
# ConfigKit Adoption Implementation Plan
|
||||
|
||||
Status: ready for staged implementation under the agreed configuration boundary.
|
||||
This is a planning document; writing it does not implement the feature or
|
||||
authorize publishing dependencies.
|
||||
|
||||
## Authority And Working Method
|
||||
|
||||
The [ConfigKit feature roadmap](configkit.md) owns the intended end state and
|
||||
approved policy. Its intentional-breaking-change direction is authoritative:
|
||||
preserve substantive prompting capabilities, not the old loader, signatures,
|
||||
source wrappers, incidental error ordering, or configuration syntax.
|
||||
The [configuration-boundary ADR](../adr/0001-configuration-boundary.md) records
|
||||
the ownership decision: PromptKit interprets and validates the final effective
|
||||
configuration supplied by ConfigKit. Presence/null information and effective
|
||||
resource context are part of that handoff. Source history serves explicit
|
||||
operator inspection, not validation or catalog enforcement. This applies to
|
||||
tests as well as production code.
|
||||
|
||||
Implement the stages below in order, one stage per coding-agent prompt. Each
|
||||
stage includes its tests and affected current-state documentation. Read
|
||||
[development](../development.md) and the applicable policies before starting;
|
||||
in particular, follow [agents](../policy/agents.md),
|
||||
[architecture](../policy/architecture.md), [testing](../policy/testing.md),
|
||||
[documentation](../policy/documentation.md), and [OSS](../policy/oss.md).
|
||||
Catalog work also follows each catalog repository's own instructions and release
|
||||
procedure. Do not turn this plan's internal organization into new public promises.
|
||||
|
||||
Begin each stage by inspecting the worktree and earlier stages. Commit completed
|
||||
units under the agent policy. Keep stages buildable; temporary private wiring
|
||||
needed to connect successive stages must be removed by Stage 11. Do not ship a
|
||||
legacy mode, dual loader, or old-format conversion layer. Do not add prototype
|
||||
applications or a generic configuration framework of our own.
|
||||
|
||||
Planning baselines:
|
||||
|
||||
- PromptKit runtime v0.11.2, commit
|
||||
`12d6d80024e80c552d122a076c22429bf6f2f6c2`; approved roadmap at `521291d`.
|
||||
- ConfigKit v0.5.0, commit
|
||||
`dc09959cca4e105f49ec316c6d4ba585bd2340bd`, module
|
||||
`gitea.maximumdirect.net/eric/ConfigKit`.
|
||||
- Both catalog dependencies currently select `/v2` at `v2.20260926.0`.
|
||||
|
||||
Use ConfigKit v0.5.0 for this work. Do not silently upgrade to a different
|
||||
contract. The catalog asset migration needs coordinated releases, not a committed
|
||||
workspace or local replacement. Changing the backend asset contract is broader
|
||||
than the catalog profile-data compatibility exception: migrate the catalogs to
|
||||
`/v3` with `v3.YYYYMMDD.N` tags, preserving their date/revision convention and
|
||||
updating their own release documentation accordingly. Select exact dates and
|
||||
unused revisions at publication; record versions and full commits consumed.
|
||||
Publishing any repository requires separate authority.
|
||||
|
||||
## Implementation Decisions
|
||||
|
||||
### Construction Surface
|
||||
|
||||
Keep `NewEngine(Config, ...Option)` and the execution/injection surface where it
|
||||
still serves its substantive purpose. Replace profile/prompt source options with
|
||||
these fields and helpers; this is an intentional API break:
|
||||
|
||||
```go
|
||||
type ConfigurationSources struct {
|
||||
Underlays []configkit.Underlay
|
||||
Main *configkit.Input
|
||||
}
|
||||
|
||||
// Add Sources; remove PromptDir and ProfileDir.
|
||||
// Retain SchemaDir, Timeout, HTTPClient, and TransientRetries.
|
||||
type Config struct { /* existing non-definition settings; Sources ConfigurationSources */ }
|
||||
|
||||
func DirectoryInputs(directory string) ([]configkit.Input, error)
|
||||
func FilesystemInputs(label string, filesystem fs.FS, root string) ([]configkit.Input, error)
|
||||
```
|
||||
|
||||
The pointer distinguishes omitted main input without inspecting ConfigKit's
|
||||
opaque Source. Nil main becomes `configkit.NoSource()`; a supplied invalid Input
|
||||
is not silently ignored. Prepend two private underlays: external catalogs, then
|
||||
PromptKit-owned definitions. Append caller underlays in their supplied order;
|
||||
main is last and keeps its ConfigKit meaning. Imports follow each Input's flag.
|
||||
Do not expose named overlays, CLI overrides, or VersionRule through another
|
||||
PromptKit abstraction in this feature.
|
||||
|
||||
Show the usual application categories as caller underlays ordered fallback,
|
||||
operator, programmatic, with main omitted. For a single highest-priority document,
|
||||
main is also useful. These are explicit selections, not directory-name heuristics
|
||||
or a second automatic precedence algorithm. Each directory helper returns inputs
|
||||
for one strict-composition group, recursively finding `.yaml` and `.yml` files;
|
||||
sorted discovery makes diagnostics reproducible, not one file higher priority.
|
||||
An explicitly selected missing/unreadable root fails; an empty directory supplies
|
||||
no contributions. Helpers set imports false; callers may enable them explicitly.
|
||||
Use one ConfigKit filesystem namespace per helper invocation and root resource
|
||||
contexts at the supplied directory. Single files use ConfigKit directly.
|
||||
|
||||
Remove `WithPromptFS`, `WithPromptFile`, `WithProfileFS`, `WithProfileFile`,
|
||||
`WithFallbackProfileFS`, `WithProfiles`, and `WithBackend` at public cutover.
|
||||
Retain schema source options and runtime collaborator injection. Backend
|
||||
configuration moves into ConfigKit inputs as specified below. Do not require any
|
||||
prompts to exist merely to construct an engine for profile inspection.
|
||||
|
||||
Programmatic configuration uses labeled `configkit.Mapping` inputs. Replace the
|
||||
old zero-sentinel `Profile`/`OpenAICompatibleProfileConfig` pair with one small
|
||||
`ProfileConfig` convenience and a source builder:
|
||||
|
||||
```go
|
||||
type ProfileConfig struct {
|
||||
Extends, Backend, Endpoint, Model *string
|
||||
Temperature, TopP *float64
|
||||
MaxTokens, TimeoutSeconds *int
|
||||
ServiceTier, ReasoningEffort, APIKeyEnv *string
|
||||
APIKeyRequired *bool
|
||||
ExtraParams map[string]any
|
||||
}
|
||||
func ProfileSource(label string, profiles map[string]ProfileConfig) configkit.Source
|
||||
```
|
||||
|
||||
Nil convenience fields mean omitted; nonnil pointers preserve zero/false/empty.
|
||||
Nil ExtraParams omits the map; a nonnil empty map supplies an empty mapping, not a
|
||||
clear operation. Use raw Mapping for explicit null, prompt definitions, or richer
|
||||
configuration. Do not introduce a public generic optional-value type just for
|
||||
this helper. Convert fields explicitly to ConfigKit-supported built-in values,
|
||||
then call Mapping for snapshot ownership and load-time errors. Do not marshal
|
||||
through YAML or infer absence from a value. Helpers encode values; domain
|
||||
validation occurs on the final configuration at handoff.
|
||||
|
||||
Replace the registration-only Backend value with a presence-aware convenience:
|
||||
|
||||
```go
|
||||
type BackendConfig struct {
|
||||
Endpoint, APIKeyEnv *string
|
||||
ConcurrencyLimit, QueueCapacity *int
|
||||
ExtraParams map[string]any
|
||||
}
|
||||
func BackendSource(label string, backends map[string]BackendConfig) configkit.Source
|
||||
func LocalBackend(endpoint string, concurrencyLimit int) BackendConfig
|
||||
```
|
||||
|
||||
Use the same omission/pointer/map conventions as ProfileSource. LocalBackend
|
||||
supplies endpoint and concurrency pointers and leaves the remaining fields
|
||||
omitted; callers choose the ID as the map key. Neither helper validates settings
|
||||
before ConfigKit processes them. For example, place BackendSource with
|
||||
`map[string]BackendConfig{"local": LocalBackend(endpoint, 4)}` in an application
|
||||
underlay. Raw Mapping supports explicit null and combined backend/profile/prompt
|
||||
documents. Remove the old Backend registration value and its reservation rules;
|
||||
retain useful built-in ID constants. All backend contributions follow the same
|
||||
strict-composition and overlay rules as other configuration.
|
||||
|
||||
### Typed Configuration And Execution Values
|
||||
|
||||
Add `internal/configuration` as the cohesive ConfigKit adapter. Its private
|
||||
document DTO has `backends: map[string]backendConfig`,
|
||||
`profiles: map[string]profileConfig`, and
|
||||
`prompts: map[string]map[string]promptConfig`. Collection entries are mappings,
|
||||
not nullable definition pointers. No duplicate identity fields; ConfigKit consumes
|
||||
`extends`, so no DTO field or second resolver handles it. Use `configkit.Keys`,
|
||||
`Key`, and `Index` for semantic paths, never split IDs on dots.
|
||||
|
||||
Profile DTO fields match ProfileConfig's YAML spellings (`backend`,
|
||||
`reasoning_effort`, etc.). Nullable pointers plus ConfigKit presence metadata
|
||||
distinguish absent, value, and explicit null. Store that distinction in small
|
||||
private execution-owned fields or presence bits, not retained metadata queries.
|
||||
Runtime/domain packages must not import ConfigKit. Do not duplicate its tree,
|
||||
merge engine, inheritance graph, diagnostics, or presence framework.
|
||||
|
||||
Backend DTO fields match BackendConfig using `endpoint`, `api_key_env`,
|
||||
`concurrency_limit`, `queue_capacity`, and `extra_params`. Validate the final
|
||||
endpoint as required, absolute HTTP(S) with the existing URL rules. An absent or
|
||||
null API-key environment name means no lookup; a supplied name follows existing
|
||||
normalization and naming rules. Absent/null concurrency means zero (unlimited);
|
||||
supplied values must be nonnegative. Absent/null queue capacity selects the
|
||||
existing default of 1024 for positive concurrency and no queue for unlimited
|
||||
concurrency. Explicit queue capacity, including zero, requires positive
|
||||
concurrency and must satisfy existing nonnegative/overflow checks. Null provider
|
||||
maps mean no configured extra parameters; entries may contain JSON null.
|
||||
Required endpoint failures, unknown fields, and domain errors apply to final
|
||||
definitions, including unused ones. Registry construction consumes these final
|
||||
normalized values; it does not replay registration or source-reservation checks.
|
||||
|
||||
Use ConfigKit's Resolve/Validate callbacks only on its effective post-layering,
|
||||
post-inheritance typed values. These public lifecycle hooks implement the domain
|
||||
handoff and record normalization; they are not hooks for inspecting earlier
|
||||
contributions. Final presence lookup and resource-context lookup are sufficient
|
||||
for interpretation. History/Explain/HistoricalPaths are operator inspection
|
||||
facilities, not inputs to construction-time policy checks.
|
||||
|
||||
Use the existing prompt fields under each keyed version, without `id`/`version`.
|
||||
Use pointers for `content`, `content_file`, and optional clearable strings;
|
||||
preserve existing input, cache-control, output-contract, and role rules.
|
||||
`messages` must be nonempty; each message has exactly one nonnull content source.
|
||||
An explicitly supplied empty inline string is a valid template. Null content
|
||||
removes that selection; null content_file removes its reference. Output remains a
|
||||
required mapping with valid format and validation mode. Optional description,
|
||||
session template, default profile, cache control, inputs, and schema path can be
|
||||
cleared where their domain permits absence. Null required role/format/mode or a
|
||||
null whole definition is invalid. Scalar `required` and `repair_attempts` retain
|
||||
their false/zero defaults and are not nullable. Null inputs means no declarations.
|
||||
Unknown fields are ConfigKit errors, including old `id`, `version`, and
|
||||
`base_profile`. Prompt versions must be string mapping keys.
|
||||
|
||||
Literal definition keys remain exact; require nonblank identity but do not
|
||||
normalize/rekey maps after ConfigKit has resolved relationships. Update selection
|
||||
to use exact nonblank IDs/versions rather than silently trimming them into a
|
||||
different identity. Scalar normalization such as endpoint trimming is recorded
|
||||
through ConfigKit's resolution callbacks when
|
||||
mutating its typed configuration. Message roles and cache control are normalized
|
||||
on execution-owned copies instead: ConfigKit v0.5.0 treats sequence mutations as
|
||||
whole-value replacements, losing unchanged content_file origins. Inspection
|
||||
retains the effective input message metadata and its source context. Field
|
||||
references to definition IDs remain exact as well.
|
||||
|
||||
Profile field rules after configuration overlay/inheritance:
|
||||
|
||||
| Fields | Absent | Explicit value | Explicit null |
|
||||
| --- | --- | --- | --- |
|
||||
| `backend` | No selected backend | Exact registered ID; blank means none | No backend |
|
||||
| `endpoint` | Selected backend endpoint, if any | Normalize/validate nonblank URL; empty clears | Clear, do not recover backend endpoint |
|
||||
| `model` | Unspecified | Nonblank supplied model; empty clears | Unspecified |
|
||||
| `temperature`, `top_p`, `max_tokens` | Unspecified | Validate existing bounds; zero is explicit and sent | Suppress outbound control |
|
||||
| `reasoning_effort`, `service_tier` | Unspecified | Existing normalization; blank suppresses | Suppress outbound control |
|
||||
| `timeout_seconds` | Framework 600-second baseline | Nonnegative; zero disables generation deadline | Reset to framework baseline, not lower profile timeout |
|
||||
| `api_key_env` | Selected backend's optional lookup name | Trimmed value; blank disables lookup | Disable lookup, do not recover backend name |
|
||||
| `api_key_required` | False | Explicit boolean, including false overriding true | False |
|
||||
| `extra_params` | Backend defaults | ConfigKit-composed profile map, recursively overlaid on backend map | Suppress entire configured provider map |
|
||||
|
||||
Profile API-key environment names follow the existing lookup contract, including
|
||||
host-supported non-shell-identifier names. Do not extend the backend's naming
|
||||
regular expression to profiles. Trimming, explicit clearing, optional lookup,
|
||||
and direct request-key precedence remain unchanged.
|
||||
|
||||
Backend registry defaults are applied once during snapshot construction, after
|
||||
effective profile inheritance. Keep them out of the inheritance input: a parent's
|
||||
derived endpoint/key/defaults must not follow a child that changes backend.
|
||||
Provider map null entries are literal JSON null; empty mappings retain children.
|
||||
For backend-to-profile provider-map overlay, use a narrow JSON-value merge helper
|
||||
with the same recursive-map/value-replacement semantics; it is not a configuration
|
||||
loader. Both backend and profile configuration maps have already passed through
|
||||
ConfigKit's reserved-key semantics; only request-level maps bypass that lifecycle.
|
||||
|
||||
Record any defaults/normalizations written into the typed configuration via
|
||||
ConfigKit Resolve, with dependencies on relevant configuration paths. Do not
|
||||
project runtime collaborators or credentials into the configuration tree just to
|
||||
report them. Inspection includes effective backend configuration. It describes
|
||||
resolved configuration, while InspectProfile and preparation describe
|
||||
backend-completed execution settings. Make this boundary
|
||||
explicit rather than inventing synthetic provenance for registry defaults.
|
||||
|
||||
Retain APIKeyRequired's direct-key workflow: true suppresses a backend-derived
|
||||
environment source. An explicitly configured profile environment source, or a
|
||||
request environment override, remains usable; false restores ordinary optional
|
||||
credential semantics. No credential values belong in the DTO, inspection, or
|
||||
construction-time environment resolution. Raw profile `api_key` remains invalid.
|
||||
|
||||
Validate all effective supplied values and referenced backend IDs at construction.
|
||||
Do not require a complete model/endpoint there or reject unused partial profiles.
|
||||
Do not require a prompt default_profile to be runnable before a request can
|
||||
override it. Final target completeness is checked after runtime overrides.
|
||||
InspectProfile can report a partial target without credential or provider access.
|
||||
Keep request override contracts otherwise unchanged, including complete nonempty
|
||||
runtime ExtraParams replacement and reasoning clearing. A runtime literal
|
||||
`extra_params["extends"]` remains ordinary provider data. Do not expand this work
|
||||
into a redesign of every request override.
|
||||
|
||||
### Snapshot And Inspection Design
|
||||
|
||||
The loader returns indexed, execution-owned definitions plus a private retained
|
||||
ConfigKit result. An adapter may implement the runner's narrow lookup interfaces
|
||||
over maps during migration; no repository chain, fallback-on-error, or per-call
|
||||
configuration load survives. Preserve defensive ownership across injected clients
|
||||
and public results. Cache prompt definition hashes at construction; hash effective
|
||||
captured content, not a stale resource path. Hash bytes need not match the old
|
||||
configuration format. Preserve rendered-prompt hashing semantics.
|
||||
|
||||
Resolve each effective content_file using its exact field origin and ConfigKit
|
||||
SourceContext. Join its relative name to BaseDir inside the returned rooted FS;
|
||||
reject absolute/outside-root paths and close every opened handle. Preserve path
|
||||
text, including meaningful whitespace, after separately checking nonblankness.
|
||||
Do not choose the child's source for an inherited reference. If there is no unique
|
||||
contributing source with context, fail with an actionable path/source error; the
|
||||
caller can supply an explicit Input filesystem context or override the field.
|
||||
Retain no resource capability for the purpose of public diagnostics.
|
||||
|
||||
Capture message text and parse message/session template syntax at construction
|
||||
with the same function names/options used for runtime rendering. Do not invoke
|
||||
input helpers or require Vars/artifacts while parsing. Reuse compiled templates
|
||||
where practical, binding request-specific input functions on safe per-call clones;
|
||||
never mutate a shared template function closure with request state. Runtime
|
||||
missing-variable/input failures remain runtime errors.
|
||||
|
||||
Expose a concrete read-only facade value:
|
||||
|
||||
```go
|
||||
func (e *Engine) InspectConfiguration() (ConfigurationInspection, error)
|
||||
func (v ConfigurationInspection) Metadata() configkit.Metadata
|
||||
func (v ConfigurationInspection) Sources() []configkit.SourceInfo
|
||||
func (v ConfigurationInspection) Diagnostics() []configkit.Diagnostic
|
||||
func (v ConfigurationInspection) Explain(path configkit.Path) (configkit.Explanation, bool)
|
||||
func (v ConfigurationInspection) WriteEffective(w io.Writer) error
|
||||
func (v ConfigurationInspection) WriteExplanation(w io.Writer, path configkit.Path) error
|
||||
```
|
||||
|
||||
Store an unexported narrow implementation reference; do not expose Result.Config,
|
||||
ConfigKit SourceContext, mutable runtime values, or internal types. Delegate
|
||||
queries and redaction to the retained result. Nil Engine returns ErrInvalidConfig;
|
||||
the zero inspection view has empty query results and writer operations return
|
||||
ErrInvalidConfig. Writer errors remain inspectable. Mark credential-shaped dynamic
|
||||
keys sensitive using Definition.Secret; the existing secret-key vocabulary can
|
||||
serve as a redaction predicate, not a source-rejection rule. Do not redact
|
||||
`api_key_env` names as credential values. This is not a
|
||||
promise that arbitrary prompt/provider content is safe to disclose.
|
||||
|
||||
Keep the runner independent of this view. Inspection calls perform no discovery,
|
||||
resource reads, environment credential lookup, or provider operations. Schemas,
|
||||
artifacts, preparation ownership, generation admission, retry/repair, observations,
|
||||
and cancellation remain at their established runtime boundaries.
|
||||
|
||||
## Stage 1 — Add The Dependency And Typed Loading Core
|
||||
|
||||
**Completed:** 2026-09-30.
|
||||
|
||||
Scope: module metadata, `internal/configuration` DTOs, real ConfigKit loading,
|
||||
and its focused tests. Leave engine assembly on the old path for this stage.
|
||||
|
||||
1. Add ConfigKit v0.5.0; update go.sum and factual OSS notices for it and its
|
||||
selected transitive dependencies. Preserve project licenses and the Go floor.
|
||||
2. Add keyed backend/profile/prompt DTOs and a private load entry point accepting
|
||||
ConfigKit Inputs and underlays. Call LoadLayered once, with no VersionRule or
|
||||
secret resolution bindings for API credentials. Use callback-scoped
|
||||
Resolve/Validate correctly.
|
||||
3. Implement effective presence extraction and the field rules above. Validate
|
||||
final domain values through existing helpers, resolving backend references
|
||||
against the final backend map; profiles have no final target requirement yet.
|
||||
Record normalization/default mutations, not arbitrary DTO bookkeeping fields.
|
||||
4. Preserve `*configkit.LoadError` through wrapping. Domain callback messages
|
||||
identify paths/problems without interpolating raw values or credentials.
|
||||
5. Add representative real-loader tests: keyed literal IDs, strict group conflict,
|
||||
recursive cross-tier overlay, a shared base with explicit zero/null/false, an
|
||||
unused invalid effective definition, and preservation of a ConfigKit load
|
||||
error. Cover an invalid lower-layer domain value replaced by a valid final
|
||||
value: construction succeeds. Do not reproduce ConfigKit's parser/inheritance
|
||||
matrix or add preflight/source-policy tests.
|
||||
|
||||
Exit: focused tests and `go test ./...` pass; old engine behavior is not presented
|
||||
as migrated. Internal inventory documents only the loader now actually present.
|
||||
|
||||
## Stage 2 — Add Source Discovery And Prompt Resource Capture
|
||||
|
||||
**Completed:** 2026-09-30.
|
||||
|
||||
Scope: source selection helpers and construction-time prompt materialization.
|
||||
|
||||
1. Implement private directory/FS discovery behind the chosen helper signatures;
|
||||
wrap discovered paths as ConfigKit sources, retaining explicit root context.
|
||||
Reuse useful filecatalog path handling without carrying profile metadata scans.
|
||||
2. Materialize effective prompt message text using field origins and passive
|
||||
contexts. Move reusable prompt/input/cache/output validation into the loader
|
||||
or shared domain owner; do not copy entire legacy parsers.
|
||||
3. Build immutable prompt indexes by exact ID/version and an ambiguity indicator
|
||||
for omitted versions. Capture declared-input inspection data and prompt hashes.
|
||||
4. Validate static template syntax using the renderer's grammar/function setup.
|
||||
Refactor that setup into one private owner if needed; do not render at load.
|
||||
5. Test native and embedded sources, imported/inherited relative content from a
|
||||
different directory, explicit mapping context, contextless failure, handle
|
||||
closure, malformed static syntax, and resource mutation after capture. Use
|
||||
minimal fixtures, sharing scenarios where the same adapter is exercised.
|
||||
|
||||
Exit: loader creates usable frozen prompt definitions without request data.
|
||||
`go test ./internal/configuration ./internal/prompt ./internal/promptdef` passes;
|
||||
existing execution tests still work through the old assembly until cutover.
|
||||
|
||||
## Stage 3 — Make Execution Settings Presence-Aware
|
||||
|
||||
**Completed:** 2026-09-30.
|
||||
|
||||
Scope: domain settings, target resolution, wire-presence propagation, profile
|
||||
inspection. No new source discovery in execution packages.
|
||||
|
||||
1. Build the immutable backend registry from the final loaded backend map. Add
|
||||
execution-owned presence/suppression state and construct indexed profiles
|
||||
from loader output plus that registry. Apply registry-derived
|
||||
values after inheritance, using the field table above.
|
||||
2. Replace scalar-zero/blank merge assumptions in target resolution. Configuration
|
||||
explicit zero reaches injected clients and the built-in request body; null does
|
||||
not resurrect a provider setting. Extend existing TargetPresence plumbing
|
||||
through preparation/results as necessary rather than creating a parallel path.
|
||||
3. Preserve request-level override precedence, API-key availability behavior,
|
||||
endpoint-only profiles, backend identity/capacity, and partial-profile runtime
|
||||
completion. Move final completeness checks after overrides. InspectProfile
|
||||
returns partial settings without treating missing model/endpoint as load errors.
|
||||
4. Test explicit numeric zero versus omission, map overlay and null, backend
|
||||
change without derived defaults leaking from parent, timeout null/zero,
|
||||
credential-source suppression, and completion through request overrides.
|
||||
Include one built-in-client wire test, not just an injected-client assertion.
|
||||
|
||||
Exit: focused domain/usecase/client tests pass. Any temporary adaptation for the
|
||||
old assembly is private, clearly limited to reaching Stage 7, and does not become
|
||||
a compatibility promise. Update affected target/presence GoDoc with this change.
|
||||
|
||||
## Stage 4 — Migrate The OpenRouter Catalog Assets
|
||||
|
||||
**Completed:** 2026-09-30. Published OpenRouter catalog `v3.20260930.0`
|
||||
from `f17d208cb7ad1fb043cda2266cd3011d11b40aa0`.
|
||||
|
||||
Scope: `../promptkit-backend-openrouter` only. Catalog maintainers own source and
|
||||
publishing checks here; PromptKit consumes its configuration through ConfigKit.
|
||||
|
||||
1. Read that repository's instructions; preserve unrelated work. Convert each
|
||||
profile asset to `profiles: {literal-id: {settings...}}`, preserving inventory
|
||||
and values, changing `base_profile` to `extends`, and removing embedded ID.
|
||||
Keep per-profile files and the exported FS/Root package; no dependency on
|
||||
PromptKit and no second legacy asset tree.
|
||||
2. Audit zero/empty values that previously meant unspecified. Remove those when
|
||||
omission was intended; retain intentional explicit values. Review any dynamic
|
||||
provider `extends` keys rather than silently changing outbound payloads.
|
||||
3. Replace `catalog/backend.json` with `catalog/backend.yml` containing a
|
||||
`backends: {openrouter: {settings...}}` mapping. Preserve endpoint, API-key
|
||||
environment name, concurrency, queue, and extra-parameter values; remove the
|
||||
duplicate id and old manifest schema_version. The file is an ordinary ConfigKit
|
||||
input, not a separately decoded manifest. Update embedded assets, module path
|
||||
and imports to `/v3`, tests, and catalog format/release documentation, with an
|
||||
incompatible-format migration note. Keep FS/Root as the minimal export surface.
|
||||
Catalog-local publishing checks stay in that repository. No provider probes.
|
||||
4. Validate under its procedure and commit. If publication is authorized, publish
|
||||
the next unused `v3.YYYYMMDD.N`; otherwise report the exact commit awaiting
|
||||
publication. The date/revision is selected then, not guessed in this plan.
|
||||
|
||||
Exit: one reviewed migrated catalog commit; existing immutable tags still serve
|
||||
old consumers. The new major module path identifies the changed asset contract;
|
||||
old consumers retain the published `/v2` assets.
|
||||
|
||||
## Stage 5 — Migrate The Rakestrawhome Catalog Assets
|
||||
|
||||
**Completed:** 2026-09-30. Published Rakestrawhome catalog `v3.20260930.0`
|
||||
from `993443d88c451901f4c917fa00c204fd3b54d28d`.
|
||||
|
||||
Scope: `../promptkit-backend-rakestrawhome` only. Apply the same keyed-format,
|
||||
presence, notice, validation, and publication treatment as Stage 4 to its smaller
|
||||
inventory, including the `/v3` transition and keyed `backends.rakestrawhome`
|
||||
definition in `catalog/backend.yml`. Preserve backend endpoint, credentials
|
||||
metadata, and capacity policy.
|
||||
Do not couple its date/revision to OpenRouter's release.
|
||||
|
||||
Exit: both compatible catalog versions are published and ordinarily resolvable
|
||||
before Stage 6 pins them. If permission or publication is unavailable, stop at
|
||||
this external dependency checkpoint; do not commit a local replace, guessed
|
||||
version, or legacy-to-new runtime converter to bypass it.
|
||||
|
||||
## Stage 6 — Integrate Catalog Sources And Maintained Profiles
|
||||
|
||||
**Completed:** 2026-09-30.
|
||||
|
||||
Scope: `internal/catalog`, maintained definitions, catalog integration tests,
|
||||
dependency pins/notices. Keep this separate from public API migration.
|
||||
|
||||
1. Pin the published compatible catalog versions and exact attribution commits.
|
||||
Update imports to `/v3`. Replace manifest decoding with ConfigKit file Inputs.
|
||||
2. Supply backend and profile assets as ConfigKit Inputs in the first
|
||||
underlay. Supply PromptKit-owned keyed Mapping definitions as the next group:
|
||||
standard extends the selected catalog's current maintained base; auto retains
|
||||
its current direct model/backend definition. Preserve both literal names.
|
||||
3. Retire source-layout allowlists, catalog preflight, source ownership/reserved
|
||||
prefix checks, connection-field prohibitions, and secret-key source rejection.
|
||||
The adapter selects known asset entry points and discovers configuration files;
|
||||
ConfigKit acquires their contents. Domain validation sees only the final
|
||||
assembled configuration. Consumers may overlay catalog backends, profiles,
|
||||
and their bases through the same configuration model.
|
||||
4. Test real imported assets offline with the new loader, maintained-name
|
||||
resolution, consumer override of a maintained base/backend, and rejection of
|
||||
an invalid final backend. Test a replaced lower-layer domain error succeeds.
|
||||
Do not independently validate catalog-only intermediate configurations, parse
|
||||
raw assets in PromptKit tests, or audit origins to establish ownership.
|
||||
|
||||
Exit: catalog and owned definitions participate in the new keyed configuration
|
||||
model without a legacy profile parser on that path. Catalog integration tests
|
||||
pass against released modules with `GOWORK=off` and no replacements.
|
||||
|
||||
## Stage 7 — Cut Over Engine Assembly And Public Construction
|
||||
|
||||
**Completed:** 2026-09-30.
|
||||
|
||||
Scope: `engine.go`, configuration facade/helpers, `internal/engine`, facade
|
||||
adapters, and migration of their direct test setup.
|
||||
|
||||
1. Introduce the Construction Surface exactly as specified above. Remove the
|
||||
superseded source/profile options and convenience types rather than keeping
|
||||
deprecated pass-through wrappers. Keep intentional runtime injection APIs.
|
||||
2. Load the combined backend/profile/prompt configuration, construct its runtime
|
||||
registry and indexed snapshot, then assemble renderer, validator, capacity
|
||||
manager, and runner. Return no Engine on construction failure.
|
||||
Translate construction errors to ErrInvalidConfig while preserving LoadError
|
||||
via errors.As. Resource errors include semantic path and safe source context.
|
||||
3. Connect indexed prompt/profile lookups to preparation and inspection. Missing
|
||||
selections keep meaningful not-found errors; prompt version ambiguity stays a
|
||||
selection error. Old parse/load error categories need not be manufactured for
|
||||
errors now detected at construction.
|
||||
4. Move root test fixtures/setup and maintained example construction to ConfigKit
|
||||
sources, keyed definitions, ProfileSource, and BackendSource so all consumers
|
||||
still compile. Update local-backend examples to choose their ID as a map key.
|
||||
Update expected timing only for the deliberate eager-validation change. Keep
|
||||
behavioral assertions for prompting, overrides, and errors.
|
||||
5. Add public tests for a minimal custom local backend and programmatic profile,
|
||||
plus embedded defaults/operator/programmatic tiers. No elaborate consumer
|
||||
setup should be needed merely to send one prompt.
|
||||
|
||||
Exit: the public Engine uses only the new loading path. Root/package tests pass;
|
||||
all package/example sources compile. Update GoDoc and consumer construction
|
||||
guidance in this stage so removed options are not presented as usable.
|
||||
|
||||
## Stage 8 — Complete Frozen Rendering And Lifecycle Integration
|
||||
|
||||
**Completed:** 2026-09-30.
|
||||
|
||||
Scope: renderer/runtime consumption of the snapshot, ownership, and lifecycle
|
||||
regressions. Stage 2 already owns acquisition and static syntax validation.
|
||||
|
||||
1. Use captured text and parsed templates without per-run configuration reads,
|
||||
parsing, or inheritance. Bind input helpers per invocation safely; keep missing
|
||||
request data errors and direct-session template bypass behavior.
|
||||
2. Verify preparation and execution still handle artifacts and schema compilation
|
||||
at invocation time, honor output overrides, freeze prepared resources, and
|
||||
keep prepared handles bound to their engine. Credentials remain at runtime.
|
||||
3. Test an old engine after configuration/resource edits and a newly constructed
|
||||
engine observing the edits. Test concurrent renders with different Vars/input
|
||||
helpers and no data leakage. Use an FS that rejects reads after construction
|
||||
to demonstrate no configuration/content-file reopening during ordinary runs;
|
||||
do not incorrectly prohibit request artifact/schema reads.
|
||||
4. Preserve appended-message ordering, structural repair, retries, admission,
|
||||
cancellation, observers, and result metadata through existing regression
|
||||
tests; add only distinct missing cross-boundary cases.
|
||||
|
||||
Exit: focused race tests pass for assembled rendering/preparation and lifecycle.
|
||||
No retained callbacks capture ConfigKit resolution contexts or request state.
|
||||
|
||||
## Stage 9 — Expose Explicit Configuration Inspection
|
||||
|
||||
**Completed:** 2026-09-30.
|
||||
|
||||
Scope: the read-only facade and retained ConfigKit inspection adapter.
|
||||
|
||||
1. Implement ConfigurationInspection and its specified methods without exposing
|
||||
Result.Config or filesystem capabilities. Retain a private narrow view of the
|
||||
ConfigKit result; no regeneration of origins from runtime values.
|
||||
2. Ensure effective/explanation reports use ConfigKit redaction, including the
|
||||
selected sensitivity predicate. Preserve safe value-free domain diagnostics.
|
||||
3. Test a targeted literal-key path with layered/inherited history, detached
|
||||
returned slices/records, writer failures, zero/nil behavior, and redacted secret
|
||||
shaped provider values. Demonstrate inspection still works after source removal
|
||||
and makes no filesystem/provider calls.
|
||||
4. Document construction inspection versus InspectProfile and prepared details.
|
||||
Consumers choose report disclosure; do not add CLI commands or automatic logs.
|
||||
|
||||
Exit: public inspection tests pass, including concurrent inspection/execution.
|
||||
The runner has no reference to the inspector or ConfigKit metadata types.
|
||||
|
||||
## Stage 10 — Migrate Examples And Canonical Format Guidance
|
||||
|
||||
**Completed:** 2026-09-30.
|
||||
|
||||
Scope: maintained examples and user-facing documentation.
|
||||
|
||||
1. Complete the preparation, execution, and observation example migration begun
|
||||
at public cutover. Preserve their distinct existing semantic assertions; extend
|
||||
the preparation example with a small explicit inspection query if useful.
|
||||
2. Update `docs/formats.md` to own the new keyed syntax, source-group semantics,
|
||||
inheritance/presence/null table, eager validation, snapshot lifetime, and
|
||||
resource-context rules. Link ConfigKit for generic grammar; do not copy its
|
||||
complete parser/inheritance contract.
|
||||
3. Update README and `docs/consumers/pkg-promptkit.md` with minimal local-backend,
|
||||
file/embedded source, programmatic Mapping/ProfileSource, explicit inspection,
|
||||
and engine-replacement workflows. Explain main versus ordered underlays and
|
||||
backend layering in place of registration-only restrictions.
|
||||
4. Add concise old-to-new migration guidance in the consumer guide: removed API
|
||||
options, exact IDs, `base_profile` to `extends`, zero/false/null, recursive maps,
|
||||
no clear via `{}`, reserved `extends` runtime escape, failure timing, and the
|
||||
fact that changing a model/backend does not clear unrelated explicit fields.
|
||||
Remove old automatic reload claims. Release notes later link to this owner.
|
||||
|
||||
Exit: all three offline examples run with their semantic checks; documentation
|
||||
links and snippets match the implemented API. No unreleased version is tagged or
|
||||
published, and no speculative release-note version is invented.
|
||||
|
||||
## Stage 11 — Remove Superseded Machinery And Consolidate Tests
|
||||
|
||||
**Completed:** 2026-09-30.
|
||||
|
||||
Scope: dead loading/resolution code and its obsolete tests, module cleanup, and
|
||||
internal architectural documentation.
|
||||
|
||||
1. Delete lazy prompt/profile metadata prepasses, filesystem repositories,
|
||||
fallback-on-error overlays, custom base-chain resolver/depth cap, zero-sentinel
|
||||
profile merging, and temporary stage adapters once no caller needs them.
|
||||
Remove the separate backend-manifest decoder, source-policy audits, and
|
||||
pre-handoff backend registration validation. Keep final-value domain checks.
|
||||
Retain narrow lookup interfaces only where they still simplify real consumers.
|
||||
2. Retain useful filecatalog/schema utilities and JSON-value ownership/validation.
|
||||
Remove direct yaml.v3 use and dependency if no remaining owner requires it;
|
||||
do not claim all YAML dependencies disappear when ConfigKit still uses YAML.
|
||||
3. Move meaningful regressions from deleted package tests to their new owning
|
||||
boundary. Retire tests about old internals, repeated parser coverage,
|
||||
source-policy enforcement, or superseded semantics. Do not replace lost runtime
|
||||
protection with loader-only tests. Keep private assembly checks separate from
|
||||
external facade tests.
|
||||
4. Update `docs/internal/overview.md`, `docs/internal/sources.md`, relevant renderer
|
||||
and engine documents, and the implemented-structure paragraphs in architecture
|
||||
policy. Describe the new package/dependency boundary without turning names or
|
||||
chosen adapters into permanent invariants. Update OSS notices with module tidy.
|
||||
|
||||
Exit: no legacy source path remains, no internal package imports the public root,
|
||||
runtime/domain packages do not import ConfigKit, and `go test ./...` passes with
|
||||
the reduced implementation/test inventory.
|
||||
|
||||
## Stage 12 — Validate Scale And The Complete Consumer Workflow
|
||||
|
||||
**Completed:** 2026-09-30.
|
||||
|
||||
Scope: final integration evidence and high-confidence simplification, not another
|
||||
feature expansion or release gate.
|
||||
|
||||
1. Add focused benchmarks for construction with many independent definitions,
|
||||
shared inherited bases, and several tiers; add an already-constructed selection
|
||||
benchmark. Report allocations and input/output shape; inherited materialized
|
||||
output/history may dominate. Do not impose timing thresholds or new caps.
|
||||
2. Inspect for repeated full-map copies, per-run provenance calls, rescans,
|
||||
repeated inheritance, and unnecessary double parsing. Fix demonstrated local
|
||||
inefficiencies with straightforward ownership/indexing; no global cache.
|
||||
3. Run the repository's existing complete local check script once on the final
|
||||
candidate, including race tests and offline examples. Investigate failures;
|
||||
do not weaken meaningful tests to get a green result. Stage intended new
|
||||
files before the script's tracked-file checks.
|
||||
4. Review the feature-roadmap capability list against actual public workflows:
|
||||
independent inspection, partial profile completion, optional credentials,
|
||||
source contexts, artifacts/schemas, prepared ownership, backend capacity,
|
||||
overrides, appended messages, repair/retries, observations, and hashes.
|
||||
5. Report completed stages, selected catalog versions, validation evidence, and
|
||||
remaining issues. Minor-release documentation/version selection and roadmap
|
||||
retirement belong to the separately authorized release preparation workflow.
|
||||
|
||||
Exit: the feature roadmap's end state is demonstrably implemented, with one
|
||||
configuration lifecycle, explicit operator inspection, no legacy loader, and
|
||||
no newly invented provider checks or human release gates.
|
||||
|
||||
The catalog-validation placement question is resolved by the configuration-boundary
|
||||
ADR. No design question remains open here. Catalog publication authority and
|
||||
ordinary module availability remain operational checkpoints for the catalog stages.
|
||||
126
docs/roadmap/remediation.md
Normal file
126
docs/roadmap/remediation.md
Normal file
@@ -0,0 +1,126 @@
|
||||
# ConfigKit Adoption Review Findings
|
||||
|
||||
Review date: 2026-10-03. Reviewed implementation commits `2b847ac` through
|
||||
`66c55b7` against the [feature roadmap](configkit.md),
|
||||
[implementation plan](implementation.md), and accepted
|
||||
[configuration-boundary ADR](../adr/0001-configuration-boundary.md).
|
||||
|
||||
## Summary
|
||||
|
||||
The principal architecture is in place: one ConfigKit load assembles backend,
|
||||
profile, and prompt configuration; PromptKit validates effective values, captures
|
||||
resources/templates, and builds execution-owned indexes. Catalog preflight and
|
||||
legacy profile resolution are removed. Operator inspection delegates to the
|
||||
retained ConfigKit result separately from ordinary execution.
|
||||
|
||||
Both findings have been addressed. The original review evidence is retained
|
||||
below; the resolution section records the changes and regression coverage.
|
||||
|
||||
## 1. P2 — Message Normalization Loses Content-File Resource Origins
|
||||
|
||||
Location: `internal/configuration/prompts.go`, particularly `resolvePrompts`
|
||||
lines 69–99 and `readContent`'s effective-origin lookup.
|
||||
|
||||
When a role or cache-control value needs normalization, `resolvePrompts` records
|
||||
a normalization operation for the entire `messages` sequence in addition to the
|
||||
field-level operations. The resulting metadata no longer provides the source
|
||||
location needed by `readContent` for an unchanged `content_file`. Construction
|
||||
then rejects an otherwise valid prompt.
|
||||
|
||||
Confirmed through the public API using FilesystemInputs over an fs.FS containing
|
||||
this document and a readable `body.txt`:
|
||||
|
||||
```yaml
|
||||
prompts:
|
||||
p:
|
||||
v1:
|
||||
messages:
|
||||
- role: " USER "
|
||||
content_file: body.txt
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
```
|
||||
|
||||
NewEngine fails with
|
||||
`/prompts/p/v1/messages/0/content_file: content_file needs one contributing source with a filesystem context`.
|
||||
Changing only the role to `user` succeeds. Role trimming/lowercasing and
|
||||
source-relative message capture are both intended capabilities; requiring
|
||||
already-normalized metadata is an accidental restriction.
|
||||
|
||||
Required outcome:
|
||||
|
||||
- Preserve ConfigKit's effective resource context while normalizing neighboring
|
||||
message metadata. Review the broad sequence normalization record and how the
|
||||
final input origin is retained across domain normalization.
|
||||
- Use effective handoff metadata/context, not historical traversal, raw-source
|
||||
inspection, source rereading, or an invented fallback directory.
|
||||
- Add focused regressions combining file-backed content with role and
|
||||
cache-control normalization, including inherited/imported content. Existing
|
||||
tests cover normalization and resource capture separately but miss this
|
||||
interaction. Verify inspection remains accurate for untouched content fields.
|
||||
|
||||
## 2. P2 — Profiles Gain An Unrequested Environment-Name Restriction
|
||||
|
||||
Location: `internal/configuration/load.go`, profile validation at lines 278–279,
|
||||
using the package's `envName` regular expression.
|
||||
|
||||
Profile `api_key_env` values now must match `[A-Za-z_][A-Za-z0-9_]*`. The previous
|
||||
profile path accepted environment lookup names without this shell-identifier
|
||||
restriction; the regular expression previously governed backend registration.
|
||||
The refactor extends it to profiles even though arbitrary environment names
|
||||
supported by the host remain usable through Go's environment lookup and the
|
||||
request override path.
|
||||
|
||||
Confirmed through NewEngine with an otherwise valid endpoint/model profile:
|
||||
`api_key_env: NORMAL_KEY` succeeds, while `api_key_env: service.api-key` fails
|
||||
construction. The latter is a usable environment name on the reviewed Linux
|
||||
platform. This restriction is unrelated to configuration layering, ConfigKit
|
||||
typing, or optional credential availability.
|
||||
|
||||
The implementation plan's reference to existing naming rules is not evidence
|
||||
of an express user decision to introduce this new profile restriction. The
|
||||
requested compatibility break permits adopting ConfigKit semantics; it does
|
||||
not itself authorize additional unrelated name constraints.
|
||||
|
||||
Required outcome:
|
||||
|
||||
- Remove the newly introduced profile shell-identifier check, preserving the
|
||||
established trimming, explicit clearing, optional environment lookup, and
|
||||
direct-key precedence behavior.
|
||||
- Add a public-path regression for a supported non-shell-identifier environment
|
||||
name and for its unset/empty optional behavior, with controlled test environment
|
||||
state and a synthetic credential. Keep any prerequisite validation limited to
|
||||
actual supported environment lookup requirements.
|
||||
- Correct the plan's profile-field guidance so later work does not reintroduce
|
||||
this restriction. Treat any broader change to existing backend naming policy
|
||||
separately rather than silently extending or weakening it in this fix.
|
||||
|
||||
## Validation Performed
|
||||
|
||||
- `go test ./...` passed on Go 1.26.8.
|
||||
- `GOTOOLCHAIN=go1.25.5 go test ./...` passed on the declared minimum toolchain.
|
||||
- `go test -race ./...` passed.
|
||||
- Separate offline public-API reproductions confirmed both findings. Missing
|
||||
message/session variables also remained runtime errors in targeted checks.
|
||||
|
||||
The passing suite does not cover the two failing cases above. No provider
|
||||
requests were made. Beyond these findings, the reviewed implementation did not
|
||||
show another actionable ownership-boundary violation or strengthened guarantee.
|
||||
|
||||
## Resolution
|
||||
|
||||
1. Message role/cache-control normalization now occurs on execution-owned copies
|
||||
during materialization. ConfigKit's captured message sequence and its origins
|
||||
remain intact. The imported/inherited content regression now includes role
|
||||
and cache-control normalization and asserts the unchanged field's source in
|
||||
inspection metadata. This also removes duplicate cache-control normalization.
|
||||
2. Removed the profile environment-name regular expression check, leaving backend
|
||||
validation unchanged. A public engine/HTTP-transport regression covers a
|
||||
non-shell environment name, unset/empty/whitespace values, explicit clearing,
|
||||
trimming, and direct-key precedence. The implementation plan now states this
|
||||
distinction explicitly.
|
||||
|
||||
Both regressions failed against the reviewed implementation before the fixes.
|
||||
After remediation, the full ordinary and race-enabled suites, affected packages
|
||||
on Go 1.25.5, `go vet ./...`, and documentation/whitespace checks passed.
|
||||
@@ -1,71 +0,0 @@
|
||||
# Structured Generation Errors
|
||||
|
||||
## Purpose
|
||||
|
||||
Promptkit should give downstream applications actionable, machine-readable
|
||||
details when the built-in OpenAI-compatible client receives a non-success HTTP
|
||||
response. Today the client reports only the status code and discards the
|
||||
provider response body. This makes ordinary configuration failures—such as an
|
||||
unsupported strict JSON Schema keyword—unnecessarily difficult to diagnose.
|
||||
|
||||
## Target End State
|
||||
|
||||
Failures from the built-in transport are available through a public typed error
|
||||
that works with `errors.As` while continuing to match `ErrLLMGenerate` through
|
||||
`errors.Is`. The error should expose:
|
||||
|
||||
- the HTTP status code;
|
||||
- a normalized provider error code or type when supplied; and
|
||||
- a bounded provider message extracted from a recognized OpenAI-compatible
|
||||
JSON error envelope.
|
||||
|
||||
The ordinary `Error()` string should remain safe and concise: it should include
|
||||
the status and provider code or type, but not automatically include the
|
||||
provider message. Consumers that deliberately want the provider's diagnostic
|
||||
text can retrieve it from the typed error and apply their own disclosure and
|
||||
logging policy.
|
||||
|
||||
This contract should be available for both ordinary and prepared execution.
|
||||
Errors returned by injected model clients must continue to preserve their own
|
||||
identity and should not be converted into fabricated HTTP details.
|
||||
|
||||
## Safety And Compatibility Boundaries
|
||||
|
||||
- Never expose the raw response body, response headers, endpoint, credentials,
|
||||
request messages, schema document, or generated content through this API.
|
||||
- Read only a small fixed maximum response body, reject malformed or
|
||||
unrecognized envelopes, normalize invalid UTF-8 and control characters, and
|
||||
cap every retained diagnostic field independently.
|
||||
- Treat the extracted provider message as untrusted and potentially sensitive:
|
||||
its GoDoc must tell consumers not to log or display it without applying their
|
||||
own policy.
|
||||
- Preserve the existing generic behavior when a response is empty, non-JSON,
|
||||
oversized, or does not match a recognized error envelope.
|
||||
- Do not assign retryability from an HTTP status. Promptkit supplies facts;
|
||||
downstream applications retain retry and presentation policy.
|
||||
|
||||
## Recommended API Direction
|
||||
|
||||
Prefer one immutable public `GenerationError` value, constructed internally and
|
||||
carrying accessors for HTTP status, provider code or type, and provider message.
|
||||
This keeps the exact representation evolvable while giving consumers an
|
||||
idiomatic `errors.As` contract. Public Go declarations and GoDoc should own the
|
||||
final exact names and semantics.
|
||||
|
||||
The internal OpenAI-compatible client should parse only the conventional
|
||||
top-level `error` envelope and pass normalized details through the use-case and
|
||||
public error-mapping layers. The integration documentation should continue to
|
||||
own wire behavior; the public declarations should own the consumer contract.
|
||||
|
||||
## Acceptance Criteria
|
||||
|
||||
- A downstream consumer can distinguish a provider HTTP 400 from other
|
||||
generation failures and obtain a bounded provider explanation when present.
|
||||
- The typed error still satisfies `errors.Is(err, ErrLLMGenerate)`.
|
||||
- Existing cancellation, capacity, validation, and injected-client error
|
||||
identities remain unchanged.
|
||||
- Tests cover recognized string and numeric provider codes, absent and malformed
|
||||
envelopes, oversized bodies and fields, control characters, and error-chain
|
||||
behavior without making live provider requests.
|
||||
- Current-state GoDoc and the OpenAI-compatible integration and internal-client
|
||||
documents are updated only when the implementation lands.
|
||||
563
engine.go
563
engine.go
@@ -6,27 +6,17 @@ import (
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
artifactadapter "gitea.maximumdirect.net/eric/promptkit/internal/artifact"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/backend"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/capacity"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/defaults"
|
||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/llm"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/profile"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/profile/builtin"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/prompt"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/promptdef"
|
||||
internalengine "gitea.maximumdirect.net/eric/promptkit/internal/engine"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/observation"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/usecase"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/validate"
|
||||
)
|
||||
|
||||
// ErrInvalidConfig identifies invalid engine construction, including missing
|
||||
// required configuration, invalid options or backend registrations, and a nil
|
||||
// effective configuration, invalid options, and a nil
|
||||
// Engine receiver.
|
||||
var ErrInvalidConfig = errors.New("invalid engine configuration")
|
||||
|
||||
@@ -45,17 +35,17 @@ var (
|
||||
// nor the selected prompt's default profile is present. Such an error also
|
||||
// matches ErrInvalidRequest.
|
||||
ErrProfileRequired = errors.New("profile selection is required")
|
||||
// ErrPromptLoad identifies a failure to read, decode, validate, select, or
|
||||
// hash a prompt definition, except for the not-found case represented by
|
||||
// ErrPromptNotFound.
|
||||
// ErrPromptLoad identifies a prompt selection failure other than the
|
||||
// not-found case represented by ErrPromptNotFound. Invalid definitions
|
||||
// instead fail construction with ErrInvalidConfig.
|
||||
ErrPromptLoad = errors.New("failed to load prompt definition")
|
||||
// ErrProfileLoad identifies a failure to read, decode, validate, or select
|
||||
// an execution profile or resolve its backend, except for the profile
|
||||
// not-found case represented by ErrProfileNotFound.
|
||||
// ErrProfileLoad identifies a profile selection or resolution failure other
|
||||
// than the not-found case represented by ErrProfileNotFound. Invalid
|
||||
// definitions instead fail construction with ErrInvalidConfig.
|
||||
ErrProfileLoad = errors.New("failed to load execution profile")
|
||||
// ErrAPIKeyEnvMissing identifies an APIKeyEnv whose environment variable is
|
||||
// unset or empty when no direct RunRequest.APIKey takes precedence. Such an
|
||||
// error also matches ErrInvalidRequest.
|
||||
// ErrAPIKeyEnvMissing identifies an explicitly required APIKeyEnv whose
|
||||
// environment variable is unset or empty after direct RunRequest.APIKey
|
||||
// precedence is applied. Such an error also matches ErrInvalidRequest.
|
||||
ErrAPIKeyEnvMissing = errors.New("api_key_env points to an unset environment variable")
|
||||
// ErrArtifactLoad identifies a failure to resolve an input artifact. Errors
|
||||
// returned by an injected ArtifactReader remain available through errors.Is.
|
||||
@@ -69,9 +59,23 @@ var (
|
||||
// request, an LLM or provider rate-limit response, or ErrLLMGenerate.
|
||||
ErrCapacityExceeded = errors.New("backend capacity exceeded")
|
||||
// ErrLLMGenerate identifies a model-client failure or a nil successful
|
||||
// response. Errors returned by an injected LLMClient remain available
|
||||
// through errors.Is.
|
||||
// response. A built-in OpenAI-compatible non-2xx response is available as a
|
||||
// [GenerationError]. Errors returned by an injected LLMClient remain
|
||||
// available through errors.Is.
|
||||
ErrLLMGenerate = errors.New("failed to generate output")
|
||||
// ErrTransport identifies a failed HTTP request by the built-in model
|
||||
// client. Generation errors in this category also match ErrLLMGenerate and
|
||||
// retain the transport cause for errors.Is and errors.As. It does not imply
|
||||
// that the provider received no work.
|
||||
ErrTransport = errors.New("model transport failed")
|
||||
// ErrMalformedResponse identifies an invalid successful model response or
|
||||
// a nil successful response from an injected client. Generation errors in
|
||||
// this category also match ErrLLMGenerate.
|
||||
ErrMalformedResponse = errors.New("malformed model response")
|
||||
// ErrRetriesExhausted identifies a built-in generation sequence whose
|
||||
// positive transient retry budget ended with another eligible HTTP status.
|
||||
// RetryExhaustedError retains the final cause and actual attempt count.
|
||||
ErrRetriesExhausted = errors.New("transient model retries exhausted")
|
||||
// ErrValidation identifies an operational failure to load or compile a
|
||||
// schema or validate output. A completed validation whose Status is
|
||||
// ValidationFailed is returned in RunResult without this error.
|
||||
@@ -81,28 +85,32 @@ var (
|
||||
// Engine inspects prompts and profiles and prepares and runs Promptkit prompt
|
||||
// requests.
|
||||
//
|
||||
// An Engine is safe for concurrent calls to [Engine.InspectPrompt],
|
||||
// An Engine is safe for concurrent calls to [Engine.InspectConfiguration],
|
||||
// [Engine.InspectPrompt],
|
||||
// [Engine.InspectProfile], [Engine.Prepare], [Engine.PrepareExecution],
|
||||
// [Engine.Run], and [Engine.RunPrepared]. Each Engine owns independent
|
||||
// backend-capacity pools that coordinate Run and RunPrepared admission and
|
||||
// model generation. Injected collaborators may still be invoked concurrently
|
||||
// across different backend pools or for unlimited backends.
|
||||
type Engine struct {
|
||||
runner *usecase.Runner
|
||||
internal *internalengine.Engine
|
||||
}
|
||||
|
||||
// Config selects the directory-backed sources and built-in model-client
|
||||
// transport used by [NewEngine]. Config has no stable JSON representation.
|
||||
// ConfigurationSources selects ordered ConfigKit underlays and an optional
|
||||
// highest-priority main input. Each underlay's inputs strictly compose within
|
||||
// their group; later underlays overlay earlier ones. A nil Main uses
|
||||
// ConfigKit's empty source. A non-nil Main is passed through unchanged and
|
||||
// may fail construction if invalid.
|
||||
type ConfigurationSources struct {
|
||||
Underlays []configkit.Underlay
|
||||
Main *configkit.Input
|
||||
}
|
||||
|
||||
// Config selects configuration sources and built-in model-client transport
|
||||
// used by [NewEngine]. Config has no stable JSON representation.
|
||||
type Config struct {
|
||||
// PromptDir is the directory searched recursively for prompt definitions.
|
||||
// It is required unless a WithPromptFS or WithPromptFile option supplies the
|
||||
// prompt source.
|
||||
PromptDir string
|
||||
// ProfileDir is an optional ordinary configured source whose profiles take
|
||||
// precedence over application fallback and embedded built-in profiles. An
|
||||
// empty value selects the lower-precedence sources unless a profile-source
|
||||
// option supplies the ordinary source.
|
||||
ProfileDir string
|
||||
// Sources combines maintained definitions with caller-selected inputs.
|
||||
Sources ConfigurationSources
|
||||
// SchemaDir is the root for JSON Schema files. An empty value uses the
|
||||
// current directory. WithSchemaFS or WithSchemaFile replaces this source.
|
||||
SchemaDir string
|
||||
@@ -115,17 +123,19 @@ type Config struct {
|
||||
// Timeout or the 10-minute default. The supplied client is not mutated. This
|
||||
// field is ignored when WithLLMClient is used.
|
||||
HTTPClient *http.Client
|
||||
// TransientRetries selects additional built-in HTTP attempts after eligible
|
||||
// provider statuses. Nil defaults to two; zero disables retries. Negative
|
||||
// values are invalid. Injected LLMClient implementations are never retried
|
||||
// automatically.
|
||||
TransientRetries *int
|
||||
}
|
||||
|
||||
// Option customizes engine construction.
|
||||
//
|
||||
// NewEngine applies options in argument order and ignores nil options. Within
|
||||
// each prompt-source, ordinary-profile-source, fallback-profile-source,
|
||||
// in-memory-profile, schema-source, model-client, and artifact-reader
|
||||
// category, the last non-nil valid option replaces earlier options in that
|
||||
// category. WithBackend is the additive exception: unique registrations
|
||||
// accumulate, and a repeated backend ID is an error rather than a replacement.
|
||||
// An invalid option fails construction even if a later option would replace it.
|
||||
// each schema-source, model-client, and artifact-reader category, the last
|
||||
// non-nil valid option replaces earlier options. An invalid option fails
|
||||
// construction even if a later option would replace it.
|
||||
type Option interface {
|
||||
apply(*engineOptions) error
|
||||
}
|
||||
@@ -137,20 +147,7 @@ func (f optionFunc) apply(options *engineOptions) error {
|
||||
}
|
||||
|
||||
type engineOptions struct {
|
||||
llmClient llm.Client
|
||||
artifactReader artifactadapter.Reader
|
||||
promptDefs promptdef.Repository
|
||||
profiles profile.Repository
|
||||
fallbackProfiles profile.Repository
|
||||
memoryProfiles profile.Repository
|
||||
backends []domain.Backend
|
||||
validator validate.Validator
|
||||
promptSource bool
|
||||
profileSource bool
|
||||
fallbackProfileSource bool
|
||||
memorySource bool
|
||||
validatorSource bool
|
||||
artifactSource bool
|
||||
internal internalengine.Options
|
||||
}
|
||||
|
||||
// WithLLMClient replaces the built-in model client used by [Engine.Run] and
|
||||
@@ -166,7 +163,7 @@ func WithLLMClient(client LLMClient) Option {
|
||||
if client == nil {
|
||||
return ErrInvalidConfig
|
||||
}
|
||||
options.llmClient = publicLLMClientAdapter{client: client}
|
||||
options.internal.SetLLMClient(publicLLMClientAdapter{client: client})
|
||||
return nil
|
||||
})
|
||||
}
|
||||
@@ -181,140 +178,7 @@ func WithArtifactReader(reader ArtifactReader) Option {
|
||||
if reader == nil {
|
||||
return ErrInvalidConfig
|
||||
}
|
||||
options.artifactReader = publicArtifactReaderAdapter{reader: reader}
|
||||
options.artifactSource = true
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// WithPromptFS loads prompt definitions from fsys under root.
|
||||
//
|
||||
// The source uses the same strict prompt YAML rules as configured prompt
|
||||
// directories, and prompt content_file paths resolve within this source.
|
||||
// fsys must be non-nil and root must be non-empty; otherwise NewEngine fails
|
||||
// with ErrInvalidConfig. This option replaces Config.PromptDir and earlier
|
||||
// prompt-source options.
|
||||
func WithPromptFS(fsys fs.FS, root string) Option {
|
||||
return optionFunc(func(options *engineOptions) error {
|
||||
if fsys == nil {
|
||||
return ErrInvalidConfig
|
||||
}
|
||||
if strings.TrimSpace(root) == "" {
|
||||
return ErrInvalidConfig
|
||||
}
|
||||
options.promptDefs = promptdef.NewFSRepository(fsys, root)
|
||||
options.promptSource = true
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// WithPromptFile loads prompt definitions from the single prompt file at path.
|
||||
//
|
||||
// Relative prompt content_file paths resolve from the file's directory. path
|
||||
// must name an existing non-directory file when NewEngine applies the option.
|
||||
// This option replaces Config.PromptDir and earlier prompt-source options.
|
||||
func WithPromptFile(path string) Option {
|
||||
return optionFunc(func(options *engineOptions) error {
|
||||
fsys, root, err := fileSource(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
options.promptDefs = promptdef.NewFileRepository(fsys, root, filepath.Dir(path))
|
||||
options.promptSource = true
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// WithProfileFS loads execution profiles from fsys under root.
|
||||
//
|
||||
// Profiles from this ordinary configured source take precedence over
|
||||
// application fallback and built-in profiles. Profile YAML must use api_key_env
|
||||
// for environment-based credentials; raw API keys are rejected. fsys must be
|
||||
// non-nil and root must be non-empty; otherwise NewEngine fails with
|
||||
// ErrInvalidConfig. This option replaces Config.ProfileDir and earlier file or
|
||||
// FS profile-source options, but remains below WithProfiles in precedence.
|
||||
func WithProfileFS(fsys fs.FS, root string) Option {
|
||||
return optionFunc(func(options *engineOptions) error {
|
||||
if fsys == nil {
|
||||
return ErrInvalidConfig
|
||||
}
|
||||
if strings.TrimSpace(root) == "" {
|
||||
return ErrInvalidConfig
|
||||
}
|
||||
options.profiles = profile.NewFSRepository(fsys, root)
|
||||
options.profileSource = true
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// WithProfileFile loads execution profiles from the single profile file at path.
|
||||
//
|
||||
// The profile takes precedence over application fallback and built-in profiles.
|
||||
// Profile YAML must use api_key_env for environment-based credentials; raw API
|
||||
// keys are rejected. path must name an existing non-directory file when
|
||||
// NewEngine applies the option. This option replaces Config.ProfileDir and
|
||||
// earlier file or FS profile-source options, but remains below WithProfiles in
|
||||
// precedence.
|
||||
func WithProfileFile(path string) Option {
|
||||
return optionFunc(func(options *engineOptions) error {
|
||||
fsys, root, err := fileSource(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
options.profiles = profile.NewFSRepository(fsys, root)
|
||||
options.profileSource = true
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// WithFallbackProfileFS supplies application-owned fallback profile
|
||||
// definitions from fsys under root.
|
||||
//
|
||||
// Profile lookup checks, in order, profiles supplied by WithProfiles; the
|
||||
// ordinary configured source selected by WithProfileFile, WithProfileFS, or
|
||||
// Config.ProfileDir; this fallback source; and Promptkit's embedded built-in
|
||||
// profiles. Each source supplies a complete profile definition; profile fields
|
||||
// are not merged between sources. Only an absent profile ID proceeds to the
|
||||
// next source. A matching read, parse, duplicate, validation, or credential
|
||||
// format failure stops resolution.
|
||||
//
|
||||
// Files use the ordinary strict profile YAML and api_key_env credential rules.
|
||||
// Loading and validation are lazy: NewEngine validates this option's arguments
|
||||
// but does not read profile files. fsys must be non-nil and root must be
|
||||
// nonblank; otherwise NewEngine returns an error matching ErrInvalidConfig.
|
||||
// Repeating this option replaces the earlier valid fallback source.
|
||||
//
|
||||
// This option controls profile-definition lookup, not provider or generation
|
||||
// failover.
|
||||
func WithFallbackProfileFS(fsys fs.FS, root string) Option {
|
||||
return optionFunc(func(options *engineOptions) error {
|
||||
if fsys == nil {
|
||||
return ErrInvalidConfig
|
||||
}
|
||||
if strings.TrimSpace(root) == "" {
|
||||
return ErrInvalidConfig
|
||||
}
|
||||
options.fallbackProfiles = profile.NewFSRepository(fsys, root)
|
||||
options.fallbackProfileSource = true
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// WithProfiles configures in-memory profiles that take precedence over
|
||||
// ordinary configured, application fallback, and built-in profiles.
|
||||
//
|
||||
// NewEngine validates and copies every profile. IDs must be unique within one
|
||||
// call. An invalid profile, duplicate ID, or unsupported ExtraParams value
|
||||
// makes construction fail with ErrInvalidConfig. Repeating WithProfiles
|
||||
// replaces the complete earlier in-memory set rather than merging it.
|
||||
func WithProfiles(profiles ...Profile) Option {
|
||||
return optionFunc(func(options *engineOptions) error {
|
||||
repo, err := newMemoryProfileRepository(profiles)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
options.memoryProfiles = repo
|
||||
options.memorySource = true
|
||||
options.internal.SetArtifactReader(publicArtifactReaderAdapter{reader: reader})
|
||||
return nil
|
||||
})
|
||||
}
|
||||
@@ -327,15 +191,7 @@ func WithProfiles(profiles ...Profile) Option {
|
||||
// replaces Config.SchemaDir and earlier schema-source options.
|
||||
func WithSchemaFS(fsys fs.FS, root string) Option {
|
||||
return optionFunc(func(options *engineOptions) error {
|
||||
if fsys == nil {
|
||||
return ErrInvalidConfig
|
||||
}
|
||||
if strings.TrimSpace(root) == "" {
|
||||
return ErrInvalidConfig
|
||||
}
|
||||
options.validator = validate.NewFSValidator(fsys, root)
|
||||
options.validatorSource = true
|
||||
return nil
|
||||
return options.internal.SetSchemaFS(fsys, root)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -346,28 +202,24 @@ func WithSchemaFS(fsys fs.FS, root string) Option {
|
||||
// replaces Config.SchemaDir and earlier schema-source options.
|
||||
func WithSchemaFile(path string) Option {
|
||||
return optionFunc(func(options *engineOptions) error {
|
||||
fsys, root, err := fileSource(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
options.validator = validate.NewFSValidator(fsys, root)
|
||||
options.validatorSource = true
|
||||
return nil
|
||||
return options.internal.SetSchemaFile(path)
|
||||
})
|
||||
}
|
||||
|
||||
// NewEngine constructs an Engine from configuration and options.
|
||||
//
|
||||
// Options are applied in order according to [Option]. PromptDir is required
|
||||
// unless a prompt-source option is present. Construction validates option
|
||||
// arguments, in-memory profiles, and backend registrations but defers reading
|
||||
// and validating prompt, file-backed profile, and schema contents until Prepare
|
||||
// or Run needs them.
|
||||
// ConfigKit composes maintained catalog inputs, PromptKit-owned definitions,
|
||||
// caller underlays, and the optional main input in that order. Construction
|
||||
// validates the final effective backend, profile, and prompt definitions and
|
||||
// captures referenced prompt content. A missing prompt collection is allowed.
|
||||
// Schema documents and request artifacts remain runtime resources.
|
||||
//
|
||||
// NewEngine returns an error matching ErrInvalidConfig for invalid
|
||||
// configuration, options, or backend-capacity policies. Each constructed
|
||||
// configuration or options, preserving a ConfigKit LoadError when loading
|
||||
// fails. Each constructed
|
||||
// Engine has independent backend-capacity pools. Construction does not perform
|
||||
// model requests or require credentials.
|
||||
// model requests or require credentials. A negative Config.TransientRetries
|
||||
// is invalid.
|
||||
func NewEngine(cfg Config, opts ...Option) (*Engine, error) {
|
||||
var options engineOptions
|
||||
for _, opt := range opts {
|
||||
@@ -375,108 +227,18 @@ func NewEngine(cfg Config, opts ...Option) (*Engine, error) {
|
||||
continue
|
||||
}
|
||||
if err := opt.apply(&options); err != nil {
|
||||
return nil, fmt.Errorf("%w: %v", ErrInvalidConfig, err)
|
||||
return nil, fmt.Errorf("%w: %w", ErrInvalidConfig, err)
|
||||
}
|
||||
}
|
||||
|
||||
promptDefs := options.promptDefs
|
||||
if !options.promptSource {
|
||||
if strings.TrimSpace(cfg.PromptDir) == "" {
|
||||
return nil, fmt.Errorf("%w: prompt directory is required", ErrInvalidConfig)
|
||||
}
|
||||
promptDefs = promptdef.NewFilesystemRepository(cfg.PromptDir)
|
||||
}
|
||||
|
||||
profiles := newProfileRepository(cfg.ProfileDir, options)
|
||||
|
||||
backendRegistry, err := backend.NewRegistry(options.backends)
|
||||
assembled, err := internalengine.New(
|
||||
internalengine.Config{Sources: internalengine.ConfigurationSources{Underlays: cfg.Sources.Underlays, Main: cfg.Sources.Main}, SchemaDir: cfg.SchemaDir, Timeout: cfg.Timeout, HTTPClient: cfg.HTTPClient, TransientRetries: copyIntPtr(cfg.TransientRetries)},
|
||||
options.internal,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: failed to construct backend registry: %v", ErrInvalidConfig, err)
|
||||
return nil, fmt.Errorf("%w: %w", ErrInvalidConfig, err)
|
||||
}
|
||||
|
||||
capacityManager, err := capacity.NewManager(backendRegistry.CapacityPolicies())
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: failed to construct backend capacity manager: %v", ErrInvalidConfig, err)
|
||||
}
|
||||
|
||||
validator := options.validator
|
||||
if !options.validatorSource {
|
||||
schemaDir := cfg.SchemaDir
|
||||
if strings.TrimSpace(schemaDir) == "" {
|
||||
schemaDir = defaults.SchemaDirDefault
|
||||
}
|
||||
validator = validate.NewStandardValidator(schemaDir)
|
||||
}
|
||||
|
||||
llmClient := options.llmClient
|
||||
if llmClient == nil {
|
||||
var err error
|
||||
llmClient, err = llm.NewOpenAICompatibleClient(llm.OpenAICompatibleConfig{
|
||||
Timeout: cfg.Timeout,
|
||||
HTTPClient: cfg.HTTPClient,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: %v", ErrInvalidConfig, err)
|
||||
}
|
||||
}
|
||||
llmClient = capacity.NewClient(capacityManager, llmClient)
|
||||
|
||||
artifacts := options.artifactReader
|
||||
if !options.artifactSource {
|
||||
artifacts = artifactadapter.NewCompositeReader()
|
||||
}
|
||||
|
||||
return &Engine{
|
||||
runner: usecase.NewRunner(
|
||||
promptDefs,
|
||||
profiles,
|
||||
backendRegistry,
|
||||
artifacts,
|
||||
prompt.NewGoRenderer(),
|
||||
llmClient,
|
||||
validator,
|
||||
capacityManager,
|
||||
),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func newProfileRepository(profileDir string, options engineOptions) profile.Repository {
|
||||
repository := builtin.NewRepository()
|
||||
|
||||
if options.fallbackProfileSource {
|
||||
repository = profile.NewOverlayRepository(options.fallbackProfiles, repository)
|
||||
}
|
||||
|
||||
if options.profileSource {
|
||||
repository = profile.NewOverlayRepository(options.profiles, repository)
|
||||
} else if strings.TrimSpace(profileDir) != "" {
|
||||
repository = profile.NewOverlayRepository(profile.NewFilesystemRepository(profileDir), repository)
|
||||
}
|
||||
|
||||
if options.memorySource {
|
||||
repository = profile.NewOverlayRepository(options.memoryProfiles, repository)
|
||||
}
|
||||
|
||||
return repository
|
||||
}
|
||||
|
||||
func fileSource(name string) (fs.FS, string, error) {
|
||||
if strings.TrimSpace(name) == "" {
|
||||
return nil, "", ErrInvalidConfig
|
||||
}
|
||||
dir := filepath.Dir(name)
|
||||
base := filepath.Base(name)
|
||||
if base == "." || base == string(filepath.Separator) {
|
||||
return nil, "", ErrInvalidConfig
|
||||
}
|
||||
info, err := os.Stat(name)
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("%w: failed to access source file %q: %v", ErrInvalidConfig, name, err)
|
||||
}
|
||||
if info.IsDir() {
|
||||
return nil, "", fmt.Errorf("%w: source path %q must be a file", ErrInvalidConfig, name)
|
||||
}
|
||||
return os.DirFS(dir), filepath.ToSlash(base), nil
|
||||
return &Engine{internal: assembled}, nil
|
||||
}
|
||||
|
||||
// InspectPrompt resolves one explicit prompt definition without selecting a
|
||||
@@ -484,9 +246,9 @@ func fileSource(name string) (fs.FS, string, error) {
|
||||
//
|
||||
// InspectPrompt requires a nonblank promptID. It passes nonblank promptID and
|
||||
// promptVersion values unchanged to the engine's ordinary, case-sensitive
|
||||
// prompt selection. An empty version succeeds only when that source has one
|
||||
// selected ID; a nonempty version selects one exact ID/version pair. The
|
||||
// configured prompt source is used without merging, fallback, or enumeration.
|
||||
// prompt selection. An empty version succeeds only when that ID has one
|
||||
// version; a nonempty version selects one exact ID/version pair from the
|
||||
// captured configuration snapshot.
|
||||
//
|
||||
// A successful result proves that the selected definition and any referenced
|
||||
// message content files were structurally loaded. Inputs are returned in
|
||||
@@ -494,32 +256,32 @@ func fileSource(name string) (fs.FS, string, error) {
|
||||
// resolved. OutputContract is the normalized declared contract, with a JSON
|
||||
// Schema path when declared but without loading or compiling that schema.
|
||||
// PromptHash is the same opaque equality value as PreparedRun.PromptHash for
|
||||
// the selected definition and observed source state; its spelling, length,
|
||||
// the selected captured definition; its spelling, length,
|
||||
// encoding, algorithm, and security properties are not contracts.
|
||||
//
|
||||
// This method does not return prompt bodies, templates, source paths, schemas,
|
||||
// rendered messages, or execution settings. It does not resolve a profile or
|
||||
// credential, read artifacts or schemas, render, validate, admit capacity,
|
||||
// contact a provider, or generate model output. The returned PromptInspection
|
||||
// and its input slice are caller-owned. Filesystem-backed inspection is a
|
||||
// point-in-time lookup and does not freeze a definition for later execution.
|
||||
// and its input slice are caller-owned. Inspection does not reload configuration
|
||||
// or content files; construct another Engine to observe source changes.
|
||||
//
|
||||
// A nil Engine returns an error matching ErrInvalidConfig. A blank prompt ID
|
||||
// matches ErrInvalidRequest. An absent exact ID or version matches
|
||||
// ErrPromptNotFound and not ErrPromptLoad. Malformed, unreadable, duplicate,
|
||||
// ambiguous, referenced-content, or hashing failures match ErrPromptLoad.
|
||||
// Cancellation during lookup matches ErrPromptLoad while preserving the
|
||||
// context error. InspectPrompt returns no partial result on error.
|
||||
// ErrPromptNotFound and not ErrPromptLoad. An ambiguous omitted version matches
|
||||
// ErrPromptLoad. Invalid definitions and resource failures reject construction
|
||||
// instead. Cancellation during lookup preserves the context error. InspectPrompt
|
||||
// returns no partial result on error.
|
||||
func (e *Engine) InspectPrompt(
|
||||
ctx context.Context,
|
||||
promptID string,
|
||||
promptVersion string,
|
||||
) (*PromptInspection, error) {
|
||||
if e == nil || e.runner == nil {
|
||||
if e == nil || e.internal == nil {
|
||||
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
|
||||
}
|
||||
|
||||
inspection, err := e.runner.InspectPrompt(ctx, promptID, promptVersion)
|
||||
inspection, err := e.internal.InspectPrompt(ctx, promptID, promptVersion)
|
||||
if err != nil {
|
||||
return nil, mapPublicError(err)
|
||||
}
|
||||
@@ -529,10 +291,9 @@ func (e *Engine) InspectPrompt(
|
||||
// InspectProfile resolves one explicit profile without selecting a prompt or
|
||||
// starting execution work.
|
||||
//
|
||||
// InspectProfile trims surrounding whitespace from profileID and looks up the
|
||||
// resulting nonblank ID exactly and case-sensitively through the engine's
|
||||
// in-memory, ordinary configured-source, application fallback, and built-in
|
||||
// profile precedence. It applies the framework timeout baseline, selected
|
||||
// InspectProfile looks up a nonblank profileID exactly and case-sensitively
|
||||
// in the captured effective configuration. It applies the framework timeout
|
||||
// baseline, selected
|
||||
// backend, and then selected profile to EffectiveModelParams without a request
|
||||
// override. BackendID is empty for an endpoint-only profile.
|
||||
//
|
||||
@@ -543,23 +304,22 @@ func (e *Engine) InspectPrompt(
|
||||
// absent or blank named environment variable is not an error.
|
||||
//
|
||||
// The returned ProfileInspection and all nested mutable values are
|
||||
// caller-owned. Filesystem-backed inspection is a point-in-time lookup and
|
||||
// does not freeze the profile for a later execution. This method does not load
|
||||
// caller-owned. Inspection does not reload configuration; construct another
|
||||
// Engine to observe source changes. This method does not load
|
||||
// a prompt, render, read artifacts or schemas, admit backend capacity, contact
|
||||
// a provider, or generate model output.
|
||||
//
|
||||
// A nil Engine returns an error matching ErrInvalidConfig. A blank profile ID
|
||||
// matches ErrInvalidRequest. An absent exact ID matches ErrProfileNotFound and
|
||||
// not ErrProfileLoad. Malformed or unreadable profile data, an unknown backend,
|
||||
// or an invalid resolved target matches ErrProfileLoad. Cancellation during
|
||||
// profile loading matches ErrProfileLoad while preserving the context error.
|
||||
// not ErrProfileLoad. Invalid definitions or unknown backends reject
|
||||
// construction instead. Cancellation preserves the context error.
|
||||
// InspectProfile returns no partial result on error.
|
||||
func (e *Engine) InspectProfile(ctx context.Context, profileID string) (*ProfileInspection, error) {
|
||||
if e == nil || e.runner == nil {
|
||||
if e == nil || e.internal == nil {
|
||||
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
|
||||
}
|
||||
|
||||
inspection, err := e.runner.InspectProfile(ctx, profileID)
|
||||
inspection, err := e.internal.InspectProfile(ctx, profileID)
|
||||
if err != nil {
|
||||
return nil, mapPublicError(err)
|
||||
}
|
||||
@@ -567,6 +327,8 @@ func (e *Engine) InspectProfile(ctx context.Context, profileID string) (*Profile
|
||||
}
|
||||
|
||||
// Prepare resolves and renders a prompt request without calling an LLM.
|
||||
// It appends any validated RunRequest.AppendedMessages after rendered
|
||||
// definition messages in the returned caller-owned snapshot.
|
||||
//
|
||||
// Prepare selects the prompt and profile, resolves any selected backend and
|
||||
// effective execution settings, resolves the output contract, loads and hashes
|
||||
@@ -582,9 +344,10 @@ func (e *Engine) InspectProfile(ctx context.Context, profileID string) (*Profile
|
||||
// applicable. Cancellation is passed to the active collaborator and is
|
||||
// reported in the applicable operation category; no general errors.Is
|
||||
// relationship to ctx.Err is promised. Prepare returns no partial result on
|
||||
// error.
|
||||
// error. It validates RunRequest.TransientRetries but makes no model attempts.
|
||||
func (e *Engine) Prepare(ctx context.Context, req RunRequest) (*PreparedRun, error) {
|
||||
if e == nil || e.runner == nil {
|
||||
ctx = observation.WithoutExecutionState(ctx)
|
||||
if e == nil || e.internal == nil {
|
||||
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
|
||||
}
|
||||
|
||||
@@ -593,7 +356,7 @@ func (e *Engine) Prepare(ctx context.Context, req RunRequest) (*PreparedRun, err
|
||||
return nil, fmt.Errorf("%w: %v", ErrInvalidRequest, err)
|
||||
}
|
||||
|
||||
prepared, err := e.runner.Prepare(ctx, domainReq)
|
||||
prepared, err := e.internal.Prepare(ctx, domainReq)
|
||||
if err != nil {
|
||||
return nil, mapPublicError(err)
|
||||
}
|
||||
@@ -601,7 +364,8 @@ func (e *Engine) Prepare(ctx context.Context, req RunRequest) (*PreparedRun, err
|
||||
}
|
||||
|
||||
// PrepareExecution completely prepares a prompt request without calling the
|
||||
// configured LLMClient or reserving backend admission capacity.
|
||||
// configured LLMClient or reserving backend admission capacity. Validated
|
||||
// RunRequest.AppendedMessages are included in the frozen effective messages.
|
||||
//
|
||||
// The returned opaque handle is bound to this Engine and permits one
|
||||
// [Engine.RunPrepared] invocation. Preparation freezes the selected sources,
|
||||
@@ -614,9 +378,12 @@ func (e *Engine) Prepare(ctx context.Context, req RunRequest) (*PreparedRun, err
|
||||
// returns does not invalidate the handle or propagate to RunPrepared.
|
||||
// PrepareExecution returns the same error categories as [Engine.Prepare] and
|
||||
// returns no handle on error. A nil Engine returns an error matching
|
||||
// ErrInvalidConfig.
|
||||
// ErrInvalidConfig. The effective transient retry budget is frozen privately
|
||||
// for RunPrepared and is absent from Details. A supplied Observer is retained
|
||||
// privately for RunPrepared; preparation itself emits no events.
|
||||
func (e *Engine) PrepareExecution(ctx context.Context, req RunRequest) (*PreparedExecution, error) {
|
||||
if e == nil || e.runner == nil {
|
||||
ctx = observation.WithoutExecutionState(ctx)
|
||||
if e == nil || e.internal == nil {
|
||||
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
|
||||
}
|
||||
|
||||
@@ -625,7 +392,7 @@ func (e *Engine) PrepareExecution(ctx context.Context, req RunRequest) (*Prepare
|
||||
return nil, fmt.Errorf("%w: %v", ErrInvalidRequest, err)
|
||||
}
|
||||
|
||||
prepared, err := e.runner.PrepareExecution(ctx, domainReq)
|
||||
prepared, err := e.internal.PrepareExecution(ctx, domainReq)
|
||||
if err != nil {
|
||||
return nil, mapPublicError(err)
|
||||
}
|
||||
@@ -633,26 +400,68 @@ func (e *Engine) PrepareExecution(ctx context.Context, req RunRequest) (*Prepare
|
||||
}
|
||||
|
||||
// Run prepares a request, invokes the configured LLMClient, and validates the
|
||||
// generated output.
|
||||
// generated output. Each call uses the Engine's captured configuration and
|
||||
// composes a fresh effective prompt with any validated
|
||||
// RunRequest.AppendedMessages.
|
||||
//
|
||||
// A content-validation failure is a successful run whose
|
||||
// RunResult.Validation has Status ValidationFailed. An inability to perform
|
||||
// validation returns an error matching ErrValidation and no partial result.
|
||||
// The public Engine does not perform output repair, so validation is
|
||||
// single-pass even when OutputContract.RepairAttempts is positive.
|
||||
// RunResult.Validation has Status ValidationFailed. When its output contract
|
||||
// has a positive repair budget, a failed eligible validation can make bounded
|
||||
// additional model calls and stops at the first valid candidate. Exhaustion
|
||||
// returns the final failed validation result with cumulative usage and actual
|
||||
// repair attempts. An inability to generate or validate returns an error and
|
||||
// no partial result.
|
||||
//
|
||||
// The built-in client retries eligible provider statuses using the effective
|
||||
// Config or request budget. With R additional transient attempts and P repair
|
||||
// generations, one run can dispatch at most (R+1)×(P+1) Promptkit-controlled
|
||||
// model attempts. Each dispatched attempt receives its own generation and
|
||||
// HTTP-client timeout; the caller context governs the full operation.
|
||||
// Injected LLMClient implementations are not retried automatically.
|
||||
// With Observer, Run emits started before facade request conversion or a nil
|
||||
// engine check, and finished on every normal return after public error mapping
|
||||
// and resource release. The event RunID matches a successful result. Without
|
||||
// Observer, result timing retains its existing convention; observed wall
|
||||
// duration can be slightly longer because it spans the facade boundary.
|
||||
//
|
||||
// Run can return every error category documented by [Engine.Prepare], plus
|
||||
// ErrCapacityExceeded and ErrLLMGenerate. An engine admission rejection is
|
||||
// ErrCapacityExceeded and ErrLLMGenerate. Built-in HTTP execution failures
|
||||
// additionally match ErrTransport; malformed successful responses and nil
|
||||
// successful injected responses additionally match ErrMalformedResponse.
|
||||
// An engine admission rejection is
|
||||
// discoverable as [CapacityError] and still matches ErrCapacityExceeded. It
|
||||
// occurs before artifacts, schemas, rendering, or model generation because the
|
||||
// selected backend's admission capacity is full; it does not match
|
||||
// ErrInvalidRequest or ErrLLMGenerate. Errors from injected clients remain
|
||||
// available through errors.Is. Cancellation while waiting for model-generation
|
||||
// capacity matches both ErrLLMGenerate and the context error. Cancellation
|
||||
// otherwise follows the active collaborator's documented behavior. A nil
|
||||
// Engine returns ErrInvalidConfig. Run returns no partial result on error.
|
||||
// ErrInvalidRequest or ErrLLMGenerate. A built-in OpenAI-compatible non-2xx
|
||||
// response is discoverable as [GenerationError]. Eligible-status exhaustion
|
||||
// with a positive budget is discoverable as [RetryExhaustedError]. Injected
|
||||
// errors remain available through errors.Is. Cancellation while waiting for
|
||||
// model-generation capacity matches both ErrLLMGenerate and the context error.
|
||||
// Cancellation otherwise follows the active collaborator's documented
|
||||
// behavior. A nil Engine returns ErrInvalidConfig. Run returns no partial
|
||||
// result on error.
|
||||
func (e *Engine) Run(ctx context.Context, req RunRequest) (*RunResult, error) {
|
||||
if e == nil || e.runner == nil {
|
||||
ctx = observation.WithoutExecutionState(ctx)
|
||||
if req.Observer == nil {
|
||||
return e.run(ctx, req)
|
||||
}
|
||||
recorder, idErr := observation.New(func(event domain.ExecutionEvent) { req.Observer(fromDomainExecutionEvent(event)) })
|
||||
recorder.Start()
|
||||
if idErr != nil {
|
||||
err := fmt.Errorf("failed to create run id: %w", idErr)
|
||||
terminal := observedFinish(nil, err)
|
||||
recorder.Finish(observation.FinishData{Outcome: terminal.Outcome, Failure: terminal.Failure})
|
||||
return nil, err
|
||||
}
|
||||
result, err := e.run(observation.WithRecorder(ctx, recorder), req)
|
||||
terminal := observedFinish(result, err)
|
||||
recorder.Finish(observation.FinishData{Outcome: terminal.Outcome, Failure: terminal.Failure,
|
||||
StatusCode: terminal.StatusCode, RetriesExhausted: terminal.RetriesExhausted})
|
||||
return result, err
|
||||
}
|
||||
|
||||
func (e *Engine) run(ctx context.Context, req RunRequest) (*RunResult, error) {
|
||||
if e == nil || e.internal == nil {
|
||||
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
|
||||
}
|
||||
|
||||
@@ -661,7 +470,7 @@ func (e *Engine) Run(ctx context.Context, req RunRequest) (*RunResult, error) {
|
||||
return nil, fmt.Errorf("%w: %v", ErrInvalidRequest, err)
|
||||
}
|
||||
|
||||
result, err := e.runner.Run(ctx, domainReq)
|
||||
result, err := e.internal.Run(ctx, domainReq)
|
||||
if err != nil {
|
||||
return nil, mapPublicError(err)
|
||||
}
|
||||
@@ -681,18 +490,30 @@ func (e *Engine) Run(ctx context.Context, req RunRequest) (*RunResult, error) {
|
||||
//
|
||||
// The supplied context governs this execution attempt independently of the
|
||||
// preparation context. It covers credential revalidation, admission,
|
||||
// generation, validation, and any internal repair. Result timing begins after
|
||||
// the claim and excludes preparation and consumer-held delay.
|
||||
// generation, validation, and any bounded output repair. Result timing begins
|
||||
// after the claim and excludes preparation and consumer-held delay.
|
||||
//
|
||||
// RunPrepared can return ErrInvalidRequest, ErrAPIKeyEnvMissing,
|
||||
// ErrCapacityExceeded, ErrLLMGenerate, or ErrValidation as applicable while
|
||||
// preserving documented collaborator and context identities. An engine
|
||||
// admission rejection is discoverable as [CapacityError] and still matches
|
||||
// ErrCapacityExceeded. A completed content-validation rejection is returned
|
||||
// in RunResult, not as an operational error. An operational error returns no
|
||||
// partial RunResult.
|
||||
// ErrCapacityExceeded. A built-in OpenAI-compatible non-2xx response is
|
||||
// discoverable as [GenerationError]. Built-in HTTP execution failures match
|
||||
// ErrTransport; malformed successful responses and nil successful injected
|
||||
// responses match ErrMalformedResponse. Positive-budget eligible-status
|
||||
// exhaustion is discoverable as [RetryExhaustedError]. A completed
|
||||
// content-validation rejection, including repair exhaustion, is returned in
|
||||
// RunResult, not as an operational
|
||||
// error. RunPrepared uses the retry choice frozen during PrepareExecution. An
|
||||
// operational error returns no partial RunResult. A retained Observer begins
|
||||
// only after a successful atomic claim, with identities from the frozen
|
||||
// snapshot, and receives a terminal event on normal return after public error
|
||||
// mapping. Invalid, foreign, discarded, or reused handles emit no events.
|
||||
// Callbacks run outside the handle and capacity locks; panics propagate and
|
||||
// do not promise a terminal event.
|
||||
func (e *Engine) RunPrepared(ctx context.Context, prepared *PreparedExecution) (*RunResult, error) {
|
||||
if e == nil || e.runner == nil {
|
||||
ctx = observation.WithoutExecutionState(ctx)
|
||||
if e == nil || e.internal == nil {
|
||||
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
|
||||
}
|
||||
|
||||
@@ -700,9 +521,21 @@ func (e *Engine) RunPrepared(ctx context.Context, prepared *PreparedExecution) (
|
||||
if prepared != nil {
|
||||
internal = prepared.internal
|
||||
}
|
||||
result, err := e.runner.RunPrepared(ctx, internal)
|
||||
var recorder *observation.Recorder
|
||||
result, err := e.internal.RunPrepared(observation.WithClaimSink(ctx, &recorder), internal)
|
||||
if err != nil {
|
||||
return nil, mapPublicError(err)
|
||||
mapped := mapPublicError(err)
|
||||
if recorder != nil {
|
||||
terminal := observedFinish(nil, mapped)
|
||||
recorder.Finish(observation.FinishData{Outcome: terminal.Outcome, Failure: terminal.Failure,
|
||||
StatusCode: terminal.StatusCode, RetriesExhausted: terminal.RetriesExhausted})
|
||||
}
|
||||
return nil, mapped
|
||||
}
|
||||
return fromDomainRunResult(result), nil
|
||||
publicResult := fromDomainRunResult(result)
|
||||
if recorder != nil {
|
||||
terminal := observedFinish(publicResult, nil)
|
||||
recorder.Finish(observation.FinishData{Outcome: terminal.Outcome})
|
||||
}
|
||||
return publicResult, nil
|
||||
}
|
||||
|
||||
3587
engine_test.go
3587
engine_test.go
File diff suppressed because it is too large
Load Diff
45
errors.go
45
errors.go
@@ -6,6 +6,8 @@ import (
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/capacity"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/generation"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/llm"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/profile"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/promptdef"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/usecase"
|
||||
@@ -20,11 +22,42 @@ func mapPublicError(err error) error {
|
||||
strings.TrimSpace(internalCapacityError.BackendID) != "" {
|
||||
return &CapacityError{BackendID: internalCapacityError.BackendID}
|
||||
}
|
||||
var internalExhausted *generation.ExhaustedError
|
||||
if errors.As(err, &internalExhausted) && internalExhausted != nil {
|
||||
mapped := newRetryExhaustedError(internalExhausted.Attempts(), mapPublicError(internalExhausted.Unwrap()))
|
||||
if errors.Is(err, usecase.ErrValidation) && !errors.Is(mapped, ErrValidation) {
|
||||
return fmt.Errorf("%w: %w", ErrValidation, mapped)
|
||||
}
|
||||
if publicErr := publicErrorFor(err); publicErr != nil && !errors.Is(mapped, publicErr) {
|
||||
return fmt.Errorf("%w: %w", publicErr, mapped)
|
||||
}
|
||||
return mapped
|
||||
}
|
||||
publicErr := publicErrorFor(err)
|
||||
var providerHTTPError *llm.ProviderHTTPError
|
||||
if errors.As(err, &providerHTTPError) && providerHTTPError != nil {
|
||||
generationErr := newGenerationError(
|
||||
providerHTTPError.StatusCode(),
|
||||
providerHTTPError.ProviderCode(),
|
||||
providerHTTPError.ProviderType(),
|
||||
providerHTTPError.ProviderMessage(),
|
||||
)
|
||||
if publicErr != nil && !errors.Is(publicErr, ErrLLMGenerate) {
|
||||
return fmt.Errorf("%w: %w", publicErr, generationErr)
|
||||
}
|
||||
return generationErr
|
||||
}
|
||||
if publicErr == nil {
|
||||
return err
|
||||
}
|
||||
return fmt.Errorf("%w: %w", publicErr, err)
|
||||
switch {
|
||||
case errors.Is(err, llm.ErrTransport):
|
||||
return fmt.Errorf("%w: %w: %w", publicErr, ErrTransport, err)
|
||||
case errors.Is(err, llm.ErrMalformedResponse):
|
||||
return fmt.Errorf("%w: %w: %w", publicErr, ErrMalformedResponse, err)
|
||||
default:
|
||||
return fmt.Errorf("%w: %w", publicErr, err)
|
||||
}
|
||||
}
|
||||
|
||||
func publicErrorFor(err error) error {
|
||||
@@ -39,10 +72,6 @@ func publicErrorFor(err error) error {
|
||||
return ErrPromptLoad
|
||||
case errors.Is(err, usecase.ErrProfileLoad):
|
||||
return ErrProfileLoad
|
||||
case errors.Is(err, promptdef.ErrInvalidYAML), errors.Is(err, promptdef.ErrInvalidPromptDefinition):
|
||||
return ErrPromptLoad
|
||||
case isProfileLoadCause(err):
|
||||
return ErrProfileLoad
|
||||
case errors.Is(err, usecase.ErrAPIKeyEnvMissing):
|
||||
return errors.Join(ErrInvalidRequest, ErrAPIKeyEnvMissing)
|
||||
case errors.Is(err, capacity.ErrCapacityExceeded):
|
||||
@@ -61,9 +90,3 @@ func publicErrorFor(err error) error {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func isProfileLoadCause(err error) bool {
|
||||
return errors.Is(err, profile.ErrInvalidYAML) ||
|
||||
errors.Is(err, profile.ErrInvalidProfile) ||
|
||||
errors.Is(err, profile.ErrRawAPIKeyNotAllowed)
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/llm"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/usecase"
|
||||
)
|
||||
|
||||
@@ -48,3 +49,34 @@ func TestMapPublicErrorTranslatesCapacityError(t *testing.T) {
|
||||
t.Fatalf("mapped backend ID changed with source error: %q", publicErr.BackendID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMapPublicErrorPreservesValidationAroundGenerationError(t *testing.T) {
|
||||
internalErr := fmt.Errorf(
|
||||
"%w: %w",
|
||||
usecase.ErrValidation,
|
||||
&llm.ProviderHTTPError{},
|
||||
)
|
||||
|
||||
err := mapPublicError(internalErr)
|
||||
if !errors.Is(err, ErrValidation) {
|
||||
t.Fatalf("mapped error=%v, want ErrValidation", err)
|
||||
}
|
||||
if !errors.Is(err, ErrLLMGenerate) {
|
||||
t.Fatalf("mapped error=%v, want ErrLLMGenerate", err)
|
||||
}
|
||||
var generationErr *GenerationError
|
||||
if !errors.As(err, &generationErr) || generationErr == nil {
|
||||
t.Fatalf("mapped error=%v, want GenerationError", err)
|
||||
}
|
||||
var leakedInternalErr *llm.ProviderHTTPError
|
||||
if errors.As(err, &leakedInternalErr) {
|
||||
t.Fatalf("mapped error exposes internal ProviderHTTPError: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMapPublicErrorDoesNotTreatRequestEncodingAsTransport(t *testing.T) {
|
||||
err := mapPublicError(fmt.Errorf("%w: %w", usecase.ErrLLMGenerate, llm.ErrRequestFailed))
|
||||
if !errors.Is(err, ErrLLMGenerate) || errors.Is(err, ErrTransport) {
|
||||
t.Fatalf("request encoding category = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
168
examples/go-library/observe/main.go
Normal file
168
examples/go-library/observe/main.go
Normal file
@@ -0,0 +1,168 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"embed"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
//go:embed prompts/*.yaml schemas/*.json
|
||||
var assets embed.FS
|
||||
|
||||
type scriptedTransport struct {
|
||||
proposalCalls int
|
||||
}
|
||||
|
||||
func (t *scriptedTransport) RoundTrip(req *http.Request) (*http.Response, error) {
|
||||
if err := req.Context().Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var request struct {
|
||||
Model string `json:"model"`
|
||||
}
|
||||
if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
switch request.Model {
|
||||
case "proposal-model":
|
||||
t.proposalCalls++
|
||||
if t.proposalCalls == 1 {
|
||||
return httpResponse(http.StatusServiceUnavailable, `{"error":{}}`), nil
|
||||
}
|
||||
return modelResponse(`{"title":"Queue cleanup","risk":"low"}`, 10, 8), nil
|
||||
case "assessment-model":
|
||||
return modelResponse(`{"recommendation":"accept","confidence":0.9}`, 7, 5), nil
|
||||
default:
|
||||
return nil, fmt.Errorf("unexpected model in offline transport")
|
||||
}
|
||||
}
|
||||
|
||||
func modelResponse(content string, promptTokens, completionTokens int) *http.Response {
|
||||
body, _ := json.Marshal(map[string]any{
|
||||
"choices": []any{map[string]any{"message": map[string]string{"content": content}}},
|
||||
"usage": map[string]int{"prompt_tokens": promptTokens, "completion_tokens": completionTokens},
|
||||
})
|
||||
return httpResponse(http.StatusOK, string(body))
|
||||
}
|
||||
|
||||
func httpResponse(status int, body string) *http.Response {
|
||||
return &http.Response{StatusCode: status, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(body))}
|
||||
}
|
||||
|
||||
type eventLog struct{ events []promptkit.ExecutionEvent }
|
||||
|
||||
func (l *eventLog) record(event promptkit.ExecutionEvent) { l.events = append(l.events, event) }
|
||||
|
||||
func (l *eventLog) terminal() (promptkit.ExecutionEvent, error) {
|
||||
if len(l.events) == 0 || l.events[len(l.events)-1].Kind != promptkit.ExecutionFinished {
|
||||
return promptkit.ExecutionEvent{}, fmt.Errorf("execution did not finish")
|
||||
}
|
||||
return l.events[len(l.events)-1], nil
|
||||
}
|
||||
|
||||
type summary struct {
|
||||
ProposalAttempts uint64 `json:"proposal_attempts"`
|
||||
ProposalTransientRetries uint64 `json:"proposal_transient_retries"`
|
||||
ProposalUsageComplete bool `json:"proposal_usage_complete"`
|
||||
AssessmentAttempts uint64 `json:"assessment_attempts"`
|
||||
AssessmentTransientRetries uint64 `json:"assessment_transient_retries"`
|
||||
AssessmentOutcome string `json:"assessment_outcome"`
|
||||
}
|
||||
|
||||
func run() (summary, error) {
|
||||
transport := &scriptedTransport{}
|
||||
inputs, err := promptkit.FilesystemInputs("example prompts", assets, "prompts")
|
||||
if err != nil {
|
||||
return summary{}, err
|
||||
}
|
||||
backendID, proposalModel, assessmentModel := "offline-backend", "proposal-model", "assessment-model"
|
||||
inputs = append(inputs,
|
||||
configkit.Input{Source: promptkit.BackendSource("example backend", map[string]promptkit.BackendConfig{
|
||||
backendID: promptkit.LocalBackend("https://example.invalid/v1", 1),
|
||||
})},
|
||||
configkit.Input{Source: promptkit.ProfileSource("example profiles", map[string]promptkit.ProfileConfig{
|
||||
"proposal-profile": {Backend: &backendID, Model: &proposalModel},
|
||||
"assessment-profile": {Backend: &backendID, Model: &assessmentModel},
|
||||
})},
|
||||
)
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{
|
||||
Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: inputs}}},
|
||||
HTTPClient: &http.Client{Transport: transport},
|
||||
}, promptkit.WithSchemaFS(assets, "schemas"))
|
||||
if err != nil {
|
||||
return summary{}, err
|
||||
}
|
||||
|
||||
var proposalLog eventLog
|
||||
proposal, err := engine.Run(context.Background(), promptkit.RunRequest{
|
||||
PromptID: "example.proposal", Inputs: map[string]promptkit.ArtifactRef{"idea": promptkit.Inline("Reduce queue cleanup toil.")}, Observer: proposalLog.record,
|
||||
})
|
||||
if err != nil {
|
||||
return summary{}, err
|
||||
}
|
||||
if !proposal.Validation.IsValid {
|
||||
return summary{}, fmt.Errorf("proposal failed structural validation")
|
||||
}
|
||||
proposalTerminal, err := proposalLog.terminal()
|
||||
if err != nil {
|
||||
return summary{}, err
|
||||
}
|
||||
var assessmentLog eventLog
|
||||
assessment, err := engine.Run(context.Background(), promptkit.RunRequest{
|
||||
PromptID: "example.assessment", Inputs: map[string]promptkit.ArtifactRef{"proposal": promptkit.Inline(proposal.RawOutput)}, Observer: assessmentLog.record,
|
||||
})
|
||||
if err != nil {
|
||||
return summary{}, err
|
||||
}
|
||||
if !assessment.Validation.IsValid {
|
||||
return summary{}, fmt.Errorf("assessment failed structural validation")
|
||||
}
|
||||
assessmentTerminal, err := assessmentLog.terminal()
|
||||
if err != nil {
|
||||
return summary{}, err
|
||||
}
|
||||
var proposalValue struct {
|
||||
Risk string `json:"risk"`
|
||||
}
|
||||
var assessmentValue struct {
|
||||
Recommendation string `json:"recommendation"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(proposal.RawOutput), &proposalValue); err != nil {
|
||||
return summary{}, err
|
||||
}
|
||||
if err := json.Unmarshal([]byte(assessment.RawOutput), &assessmentValue); err != nil {
|
||||
return summary{}, err
|
||||
}
|
||||
decision := "rejected"
|
||||
if proposalValue.Risk == "low" && assessmentValue.Recommendation == "accept" {
|
||||
decision = "accepted"
|
||||
}
|
||||
return summary{
|
||||
ProposalAttempts: proposalTerminal.Attempts,
|
||||
ProposalTransientRetries: proposalTerminal.TransientRetries,
|
||||
ProposalUsageComplete: proposalTerminal.Usage.PromptTokens.Complete,
|
||||
AssessmentAttempts: assessmentTerminal.Attempts,
|
||||
AssessmentTransientRetries: assessmentTerminal.TransientRetries,
|
||||
AssessmentOutcome: decision,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func main() {
|
||||
value, err := run()
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
if err := json.NewEncoder(os.Stdout).Encode(value); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
18
examples/go-library/observe/prompts/assessment.yaml
Normal file
18
examples/go-library/observe/prompts/assessment.yaml
Normal file
@@ -0,0 +1,18 @@
|
||||
prompts:
|
||||
example.assessment:
|
||||
"1.0.0":
|
||||
default_profile: assessment-profile
|
||||
description: Assess a structured proposal independently of schema validity.
|
||||
inputs:
|
||||
- name: proposal
|
||||
required: true
|
||||
content_type: application/json
|
||||
messages:
|
||||
- role: system
|
||||
content: Return a concise assessment as JSON.
|
||||
- role: user
|
||||
content: '{{input "proposal"}}'
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: assessment.schema.json
|
||||
18
examples/go-library/observe/prompts/proposal.yaml
Normal file
18
examples/go-library/observe/prompts/proposal.yaml
Normal file
@@ -0,0 +1,18 @@
|
||||
prompts:
|
||||
example.proposal:
|
||||
"1.0.0":
|
||||
default_profile: proposal-profile
|
||||
description: Propose a bounded improvement from a synthetic idea.
|
||||
inputs:
|
||||
- name: idea
|
||||
required: true
|
||||
content_type: text/plain
|
||||
messages:
|
||||
- role: system
|
||||
content: Return a concise proposal as JSON.
|
||||
- role: user
|
||||
content: '{{input "idea"}}'
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: proposal.schema.json
|
||||
10
examples/go-library/observe/schemas/assessment.schema.json
Normal file
10
examples/go-library/observe/schemas/assessment.schema.json
Normal file
@@ -0,0 +1,10 @@
|
||||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"type": "object",
|
||||
"required": ["recommendation", "confidence"],
|
||||
"properties": {
|
||||
"recommendation": {"type": "string"},
|
||||
"confidence": {"type": "number"}
|
||||
},
|
||||
"additionalProperties": false
|
||||
}
|
||||
10
examples/go-library/observe/schemas/proposal.schema.json
Normal file
10
examples/go-library/observe/schemas/proposal.schema.json
Normal file
@@ -0,0 +1,10 @@
|
||||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"type": "object",
|
||||
"required": ["title", "risk"],
|
||||
"properties": {
|
||||
"title": {"type": "string"},
|
||||
"risk": {"type": "string"}
|
||||
},
|
||||
"additionalProperties": false
|
||||
}
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
@@ -18,18 +19,23 @@ type summary struct {
|
||||
}
|
||||
|
||||
func main() {
|
||||
engine, err := promptkit.NewEngine(
|
||||
promptkit.Config{},
|
||||
promptkit.WithPromptFile("examples/go-library/prepare/prompt.yaml"),
|
||||
promptkit.WithProfiles(promptkit.Profile{
|
||||
ID: "offline-example",
|
||||
Endpoint: "https://example.invalid/v1",
|
||||
Model: "offline-model",
|
||||
}),
|
||||
)
|
||||
endpoint, model := "https://example.invalid/v1", "offline-model"
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: promptkit.ProfileSource("example profile", map[string]promptkit.ProfileConfig{
|
||||
"offline-example": {Endpoint: &endpoint, Model: &model},
|
||||
})}}}},
|
||||
Main: &configkit.Input{Source: configkit.File("examples/go-library/prepare/prompt.yaml")},
|
||||
}})
|
||||
if err != nil {
|
||||
exit(err)
|
||||
}
|
||||
view, err := engine.InspectConfiguration()
|
||||
if err != nil {
|
||||
exit(err)
|
||||
}
|
||||
if _, found := view.Explain(configkit.Keys("profiles", "offline-example", "model")); !found {
|
||||
exit(fmt.Errorf("configured profile model was not found"))
|
||||
}
|
||||
|
||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||
PromptID: "example.prepare",
|
||||
|
||||
@@ -1,16 +1,17 @@
|
||||
id: example.prepare
|
||||
version: "1.0.0"
|
||||
default_profile: offline-example
|
||||
description: Prepare a prompt without contacting a model provider.
|
||||
inputs:
|
||||
- name: note
|
||||
required: true
|
||||
content_type: text/plain
|
||||
messages:
|
||||
- role: system
|
||||
content: Summarize the note in one sentence.
|
||||
- role: user
|
||||
content: '{{input "note"}}'
|
||||
output:
|
||||
format: text
|
||||
validation_mode: basic
|
||||
prompts:
|
||||
example.prepare:
|
||||
"1.0.0":
|
||||
default_profile: offline-example
|
||||
description: Prepare a prompt without contacting a model provider.
|
||||
inputs:
|
||||
- name: note
|
||||
required: true
|
||||
content_type: text/plain
|
||||
messages:
|
||||
- role: system
|
||||
content: Summarize the note in one sentence.
|
||||
- role: user
|
||||
content: '{{input "note"}}'
|
||||
output:
|
||||
format: text
|
||||
validation_mode: basic
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
@@ -38,16 +39,13 @@ type summary struct {
|
||||
}
|
||||
|
||||
func main() {
|
||||
engine, err := promptkit.NewEngine(
|
||||
promptkit.Config{},
|
||||
promptkit.WithPromptFile("examples/go-library/run/prompt.yaml"),
|
||||
promptkit.WithProfiles(promptkit.Profile{
|
||||
ID: "offline-example",
|
||||
Endpoint: "https://example.invalid/v1",
|
||||
Model: "offline-model",
|
||||
}),
|
||||
promptkit.WithLLMClient(deterministicClient{}),
|
||||
)
|
||||
endpoint, model := "https://example.invalid/v1", "offline-model"
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: promptkit.ProfileSource("example profile", map[string]promptkit.ProfileConfig{
|
||||
"offline-example": {Endpoint: &endpoint, Model: &model},
|
||||
})}}}},
|
||||
Main: &configkit.Input{Source: configkit.File("examples/go-library/run/prompt.yaml")},
|
||||
}}, promptkit.WithLLMClient(deterministicClient{}))
|
||||
if err != nil {
|
||||
exit(err)
|
||||
}
|
||||
|
||||
@@ -1,16 +1,17 @@
|
||||
id: example.run
|
||||
version: "1.0.0"
|
||||
default_profile: offline-example
|
||||
description: Run a prompt with a deterministic injected model client.
|
||||
inputs:
|
||||
- name: note
|
||||
required: true
|
||||
content_type: text/plain
|
||||
messages:
|
||||
- role: system
|
||||
content: Summarize the note in one sentence.
|
||||
- role: user
|
||||
content: '{{input "note"}}'
|
||||
output:
|
||||
format: text
|
||||
validation_mode: basic
|
||||
prompts:
|
||||
example.run:
|
||||
"1.0.0":
|
||||
default_profile: offline-example
|
||||
description: Run a prompt with a deterministic injected model client.
|
||||
inputs:
|
||||
- name: note
|
||||
required: true
|
||||
content_type: text/plain
|
||||
messages:
|
||||
- role: system
|
||||
content: Summarize the note in one sentence.
|
||||
- role: user
|
||||
content: '{{input "note"}}'
|
||||
output:
|
||||
format: text
|
||||
validation_mode: basic
|
||||
|
||||
@@ -18,7 +18,7 @@ func (r RunRequest) GoString() string {
|
||||
|
||||
func (r RunRequest) redactedString() string {
|
||||
return fmt.Sprintf(
|
||||
"promptkit.RunRequest{PromptID:%q PromptVersion:%q ProfileID:%q APIKeySet:%t Inputs:%d Vars:%d ExecutionSet:%t ValidationSet:%t}",
|
||||
"promptkit.RunRequest{PromptID:%q PromptVersion:%q ProfileID:%q APIKeySet:%t Inputs:%d Vars:%d ExecutionSet:%t ValidationSet:%t AppendedMessages:%d}",
|
||||
r.PromptID,
|
||||
r.PromptVersion,
|
||||
r.ProfileID,
|
||||
@@ -27,6 +27,7 @@ func (r RunRequest) redactedString() string {
|
||||
len(r.Vars),
|
||||
r.Execution != nil,
|
||||
r.Validation != nil,
|
||||
len(r.AppendedMessages),
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
177
frozen_rendering_contract_test.go
Normal file
177
frozen_rendering_contract_test.go
Normal file
@@ -0,0 +1,177 @@
|
||||
package promptkit_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"testing/fstest"
|
||||
|
||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
const capturedPromptDocument = `
|
||||
prompts:
|
||||
captured:
|
||||
"1":
|
||||
default_profile: offline
|
||||
session_id: '{{.session}}'
|
||||
messages:
|
||||
- role: user
|
||||
content_file: body.tmpl
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
`
|
||||
|
||||
func capturedPromptEngine(t *testing.T, source fs.FS) *promptkit.Engine {
|
||||
t.Helper()
|
||||
inputs, err := promptkit.FilesystemInputs("application prompts", source, "config")
|
||||
if err != nil {
|
||||
t.Fatalf("select prompt inputs: %v", err)
|
||||
}
|
||||
endpoint, model := "https://example.invalid/v1", "offline-model"
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||
Underlays: []configkit.Underlay{
|
||||
{Inputs: inputs},
|
||||
{Inputs: []configkit.Input{{Source: promptkit.ProfileSource("offline profile", map[string]promptkit.ProfileConfig{
|
||||
"offline": {Endpoint: &endpoint, Model: &model},
|
||||
})}}},
|
||||
},
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatalf("construct engine: %v", err)
|
||||
}
|
||||
return engine
|
||||
}
|
||||
|
||||
func TestEngineCapturesPromptContentUntilReconstruction(t *testing.T) {
|
||||
files := fstest.MapFS{
|
||||
"config/prompt.yaml": &fstest.MapFile{Data: []byte(capturedPromptDocument)},
|
||||
"config/body.tmpl": &fstest.MapFile{Data: []byte("before {{.name}}")},
|
||||
}
|
||||
first := capturedPromptEngine(t, files)
|
||||
files["config/body.tmpl"] = &fstest.MapFile{Data: []byte("after {{.name}}")}
|
||||
request := promptkit.RunRequest{PromptID: "captured", Vars: map[string]string{"name": "Ada", "session": "session-1"}}
|
||||
oldPrepared, err := first.Prepare(context.Background(), request)
|
||||
if err != nil {
|
||||
t.Fatalf("prepare with existing engine: %v", err)
|
||||
}
|
||||
second := capturedPromptEngine(t, files)
|
||||
newPrepared, err := second.Prepare(context.Background(), request)
|
||||
if err != nil {
|
||||
t.Fatalf("prepare with rebuilt engine: %v", err)
|
||||
}
|
||||
if oldPrepared.Messages[0].Content != "before Ada" || newPrepared.Messages[0].Content != "after Ada" || oldPrepared.PromptHash == newPrepared.PromptHash {
|
||||
t.Fatalf("captured content or hash did not follow engine lifetime: old=%+v new=%+v", oldPrepared, newPrepared)
|
||||
}
|
||||
}
|
||||
|
||||
type unreadableAfterConstructionFS struct {
|
||||
fs.FS
|
||||
closed atomic.Bool
|
||||
reads atomic.Int64
|
||||
}
|
||||
|
||||
func (source *unreadableAfterConstructionFS) Open(name string) (fs.File, error) {
|
||||
if source.closed.Load() {
|
||||
source.reads.Add(1)
|
||||
return nil, fs.ErrPermission
|
||||
}
|
||||
return source.FS.Open(name)
|
||||
}
|
||||
|
||||
func TestCapturedTemplatesBindInputsIndependentlyAcrossConcurrentPreparations(t *testing.T) {
|
||||
source := &unreadableAfterConstructionFS{FS: fstest.MapFS{
|
||||
"config/prompt.yaml": &fstest.MapFile{Data: []byte(capturedPromptDocument)},
|
||||
"config/body.tmpl": &fstest.MapFile{Data: []byte(`{{.name}}: {{input "note"}}`)},
|
||||
}}
|
||||
engine := capturedPromptEngine(t, source)
|
||||
source.closed.Store(true)
|
||||
|
||||
const calls = 24
|
||||
var wait sync.WaitGroup
|
||||
errs := make(chan error, calls)
|
||||
for index := 0; index < calls; index++ {
|
||||
wait.Add(1)
|
||||
go func(index int) {
|
||||
defer wait.Done()
|
||||
name := fmt.Sprintf("caller-%d", index)
|
||||
body := fmt.Sprintf("body-%d", index)
|
||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||
PromptID: "captured", SessionID: "direct-session",
|
||||
Vars: map[string]string{"name": name},
|
||||
Inputs: map[string]promptkit.ArtifactRef{"note": promptkit.Inline(body)},
|
||||
})
|
||||
if err == nil && (prepared.Messages[0].Content != name+": "+body || prepared.SessionID != "direct-session") {
|
||||
err = fmt.Errorf("unexpected prepared content: %+v", prepared)
|
||||
}
|
||||
errs <- err
|
||||
}(index)
|
||||
}
|
||||
wait.Wait()
|
||||
close(errs)
|
||||
for err := range errs {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if source.reads.Load() != 0 {
|
||||
t.Fatalf("preparation reopened captured source %d times", source.reads.Load())
|
||||
}
|
||||
|
||||
_, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||
PromptID: "captured", Vars: map[string]string{"name": "Ada"},
|
||||
Inputs: map[string]promptkit.ArtifactRef{"note": promptkit.Inline("body")},
|
||||
})
|
||||
if !errors.Is(err, promptkit.ErrPromptRender) {
|
||||
t.Fatalf("missing session variable = %v, want runtime render error", err)
|
||||
}
|
||||
_, err = engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||
PromptID: "captured", SessionID: "direct-session", Vars: map[string]string{"name": "Ada"},
|
||||
})
|
||||
if !errors.Is(err, promptkit.ErrPromptRender) {
|
||||
t.Fatalf("missing input helper artifact = %v, want runtime render error", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreparationStillReadsRequestArtifactsAndSchemasAfterConstruction(t *testing.T) {
|
||||
endpoint, model := "https://example.invalid/v1", "offline-model"
|
||||
schemas := fstest.MapFS{}
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: promptkit.ProfileSource("profile", map[string]promptkit.ProfileConfig{
|
||||
"offline": {Endpoint: &endpoint, Model: &model},
|
||||
})}}}},
|
||||
Main: &configkit.Input{Source: configkit.Mapping("structured prompt", map[string]any{
|
||||
"prompts": map[string]any{"structured": map[string]any{"1": map[string]any{
|
||||
"default_profile": "offline",
|
||||
"inputs": []any{map[string]any{"name": "note", "required": true}},
|
||||
"messages": []any{map[string]any{"role": "user", "content": `{{input "note"}}`}},
|
||||
"output": map[string]any{"format": "json", "validation_mode": "json_schema", "schema_path": "schema.json"},
|
||||
}}},
|
||||
})},
|
||||
}}, promptkit.WithSchemaFS(schemas, "."))
|
||||
if err != nil {
|
||||
t.Fatalf("construct engine before request resources exist: %v", err)
|
||||
}
|
||||
schemas["schema.json"] = &fstest.MapFile{Data: []byte(`{"$schema":"https://json-schema.org/draft/2020-12/schema","type":"object"}`)}
|
||||
artifactPath := filepath.Join(t.TempDir(), "note.txt")
|
||||
if err := os.WriteFile(artifactPath, []byte("request artifact"), 0o600); err != nil {
|
||||
t.Fatalf("write request artifact: %v", err)
|
||||
}
|
||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "structured", Inputs: map[string]promptkit.ArtifactRef{
|
||||
"note": promptkit.File(artifactPath),
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatalf("prepare with resources supplied after construction: %v", err)
|
||||
}
|
||||
if len(prepared.Messages) != 1 || prepared.Messages[0].Content != "request artifact" || prepared.StructuredOutput == nil || prepared.StructuredOutput.JSONSchema == nil {
|
||||
t.Fatalf("runtime artifact or schema was not prepared: %+v", prepared)
|
||||
}
|
||||
}
|
||||
87
generation_error.go
Normal file
87
generation_error.go
Normal file
@@ -0,0 +1,87 @@
|
||||
package promptkit
|
||||
|
||||
import "fmt"
|
||||
|
||||
// GenerationError reports a non-2xx response from Promptkit's built-in
|
||||
// OpenAI-compatible client during [Engine.Run] or [Engine.RunPrepared].
|
||||
//
|
||||
// Engine-produced values are immutable, caller-owned values. Use errors.Is to
|
||||
// match [ErrLLMGenerate] and errors.As with a *GenerationError target to obtain
|
||||
// this type. The four provider accessors expose untrusted provider-controlled
|
||||
// values that can contain sensitive request or schema fragments. Applications
|
||||
// must apply their own disclosure policy before logging, displaying, or
|
||||
// returning them to another caller.
|
||||
//
|
||||
// Accessors, Error, GoString, and Unwrap are safe on a nil receiver and a zero
|
||||
// value. Default and Go-syntax formatting deliberately redact provider details.
|
||||
// GenerationError has no stable JSON representation.
|
||||
type GenerationError struct {
|
||||
statusCode int
|
||||
providerCode string
|
||||
providerType string
|
||||
providerMessage string
|
||||
}
|
||||
|
||||
func newGenerationError(statusCode int, providerCode, providerType, providerMessage string) *GenerationError {
|
||||
return &GenerationError{
|
||||
statusCode: statusCode,
|
||||
providerCode: providerCode,
|
||||
providerType: providerType,
|
||||
providerMessage: providerMessage,
|
||||
}
|
||||
}
|
||||
|
||||
// StatusCode returns the received provider HTTP status code, or zero for a nil
|
||||
// receiver or zero value.
|
||||
func (e *GenerationError) StatusCode() int {
|
||||
if e == nil {
|
||||
return 0
|
||||
}
|
||||
return e.statusCode
|
||||
}
|
||||
|
||||
// ProviderCode returns the normalized provider error code, if present. Its
|
||||
// value is untrusted and may contain sensitive data.
|
||||
func (e *GenerationError) ProviderCode() string {
|
||||
if e == nil {
|
||||
return ""
|
||||
}
|
||||
return e.providerCode
|
||||
}
|
||||
|
||||
// ProviderType returns the normalized provider error type, if present. Its
|
||||
// value is untrusted and may contain sensitive data.
|
||||
func (e *GenerationError) ProviderType() string {
|
||||
if e == nil {
|
||||
return ""
|
||||
}
|
||||
return e.providerType
|
||||
}
|
||||
|
||||
// ProviderMessage returns the bounded normalized provider diagnostic, if
|
||||
// present. Its value is untrusted and may contain sensitive data.
|
||||
func (e *GenerationError) ProviderMessage() string {
|
||||
if e == nil {
|
||||
return ""
|
||||
}
|
||||
return e.providerMessage
|
||||
}
|
||||
|
||||
// Error returns a redacted diagnostic that is not a parsing contract.
|
||||
func (e *GenerationError) Error() string {
|
||||
if e == nil || e.statusCode == 0 {
|
||||
return ErrLLMGenerate.Error()
|
||||
}
|
||||
return fmt.Sprintf("%s: provider returned HTTP status %d", ErrLLMGenerate, e.statusCode)
|
||||
}
|
||||
|
||||
// GoString returns the same redacted diagnostic as Error.
|
||||
func (e *GenerationError) GoString() string {
|
||||
return e.Error()
|
||||
}
|
||||
|
||||
// Unwrap returns ErrLLMGenerate. It is safe to call on a nil receiver or zero
|
||||
// value.
|
||||
func (e *GenerationError) Unwrap() error {
|
||||
return ErrLLMGenerate
|
||||
}
|
||||
229
generation_error_contract_test.go
Normal file
229
generation_error_contract_test.go
Normal file
@@ -0,0 +1,229 @@
|
||||
package promptkit_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
func TestBuiltInGenerationError(t *testing.T) {
|
||||
const (
|
||||
codeMarker = "provider-code-marker"
|
||||
typeMarker = "provider-type-marker"
|
||||
messageMarker = "provider-message-marker"
|
||||
)
|
||||
engine := newBuiltInGenerationErrorEngine(t, http.StatusUnprocessableEntity,
|
||||
`{"error":{"code":"`+codeMarker+`","type":"`+typeMarker+`","message":"`+messageMarker+`"}}`)
|
||||
|
||||
result, err := engine.Run(context.Background(), simpleGenerationRequest())
|
||||
if result != nil {
|
||||
t.Fatalf("Run result = %#v, want nil", result)
|
||||
}
|
||||
assertGenerationError(t, err, http.StatusUnprocessableEntity, codeMarker, typeMarker, messageMarker)
|
||||
|
||||
preparedEngine := newBuiltInGenerationErrorEngine(t, http.StatusServiceUnavailable, `{"error":{}}`)
|
||||
prepared, err := preparedEngine.PrepareExecution(context.Background(), simpleGenerationRequest())
|
||||
if err != nil {
|
||||
t.Fatalf("PrepareExecution: %v", err)
|
||||
}
|
||||
result, err = preparedEngine.RunPrepared(context.Background(), prepared)
|
||||
if result != nil {
|
||||
t.Fatalf("RunPrepared result = %#v, want nil", result)
|
||||
}
|
||||
assertGenerationError(t, err, http.StatusServiceUnavailable, "", "", "")
|
||||
}
|
||||
|
||||
func TestBuiltInGenerationErrorWithAppendedMessages(t *testing.T) {
|
||||
const messageMarker = "combined-request-provider-marker"
|
||||
engine := newBuiltInGenerationErrorEngine(t, http.StatusUnprocessableEntity,
|
||||
`{"error":{"message":"`+messageMarker+`"}}`)
|
||||
request := simpleGenerationRequest()
|
||||
request.AppendedMessages = []promptkit.RenderedMessage{
|
||||
{Role: promptkit.RoleAssistant, Content: "previous response"},
|
||||
{Role: promptkit.RoleUser, Content: "consumer correction"},
|
||||
}
|
||||
|
||||
result, err := engine.Run(context.Background(), request)
|
||||
if result != nil {
|
||||
t.Fatalf("Run result = %#v, want nil", result)
|
||||
}
|
||||
assertGenerationError(t, err, http.StatusUnprocessableEntity, "", "", messageMarker)
|
||||
}
|
||||
|
||||
func TestBuiltInRepairGenerationError(t *testing.T) {
|
||||
const (
|
||||
codeMarker = "repair-code-marker"
|
||||
typeMarker = "repair-type-marker"
|
||||
messageMarker = "repair-message-marker"
|
||||
)
|
||||
calls := 0
|
||||
config := simpleConfig("message")
|
||||
config.HTTPClient = &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
||||
calls++
|
||||
if calls == 1 {
|
||||
body := `{"choices":[{"message":{"content":"not-json"}}]}`
|
||||
return &http.Response{StatusCode: http.StatusOK, ContentLength: int64(len(body)), Body: io.NopCloser(strings.NewReader(body))}, nil
|
||||
}
|
||||
body := `{"error":{"code":"` + codeMarker + `","type":"` + typeMarker + `","message":"` + messageMarker + `"}}`
|
||||
return &http.Response{StatusCode: http.StatusUnprocessableEntity, ContentLength: int64(len(body)), Body: io.NopCloser(strings.NewReader(body))}, nil
|
||||
})}
|
||||
engine, err := promptkit.NewEngine(config)
|
||||
if err != nil {
|
||||
t.Fatalf("NewEngine: %v", err)
|
||||
}
|
||||
req := simpleGenerationRequest()
|
||||
req.Validation = &promptkit.OutputContract{
|
||||
Format: promptkit.FormatJSON,
|
||||
ValidationMode: promptkit.ValidationJSON,
|
||||
RepairAttempts: 1,
|
||||
}
|
||||
|
||||
result, err := engine.Run(context.Background(), req)
|
||||
if result != nil {
|
||||
t.Fatalf("Run result = %#v, want nil", result)
|
||||
}
|
||||
if calls != 2 {
|
||||
t.Fatalf("provider calls = %d, want 2", calls)
|
||||
}
|
||||
assertGenerationError(t, err, http.StatusUnprocessableEntity, codeMarker, typeMarker, messageMarker)
|
||||
}
|
||||
|
||||
func TestBuiltInGenerationFailureCategories(t *testing.T) {
|
||||
transportCause := errors.New("private transport diagnostic")
|
||||
config := simpleConfig("message")
|
||||
config.HTTPClient = &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
||||
return nil, transportCause
|
||||
})}
|
||||
engine, err := promptkit.NewEngine(config)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
result, err := engine.Run(context.Background(), simpleGenerationRequest())
|
||||
if result != nil || !errors.Is(err, promptkit.ErrLLMGenerate) || !errors.Is(err, promptkit.ErrTransport) || !errors.Is(err, transportCause) {
|
||||
t.Fatalf("transport result=%#v, error=%v", result, err)
|
||||
}
|
||||
if errors.Is(err, promptkit.ErrMalformedResponse) || strings.Contains(fmt.Sprintf("%#v", err), transportCause.Error()) {
|
||||
t.Fatalf("transport category or formatting = %v", err)
|
||||
}
|
||||
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
body string
|
||||
}{
|
||||
{name: "invalid framing", body: `{"choices":[]}{}`},
|
||||
{name: "missing choices", body: `{"choices":[]}`},
|
||||
{name: "missing content", body: `{"choices":[{"message":{}}]}`},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
engine := newBuiltInGenerationErrorEngine(t, http.StatusOK, tc.body)
|
||||
result, err := engine.Run(context.Background(), simpleGenerationRequest())
|
||||
if result != nil || !errors.Is(err, promptkit.ErrLLMGenerate) || !errors.Is(err, promptkit.ErrMalformedResponse) || errors.Is(err, promptkit.ErrTransport) {
|
||||
t.Fatalf("malformed result=%#v, error=%v", result, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuiltInGenerationPreservesContextIdentity(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
cause error
|
||||
}{
|
||||
{name: "canceled", cause: context.Canceled},
|
||||
{name: "deadline", cause: context.DeadlineExceeded},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
config := simpleConfig("message")
|
||||
config.HTTPClient = &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
||||
return nil, tc.cause
|
||||
})}
|
||||
engine, err := promptkit.NewEngine(config)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
result, err := engine.Run(context.Background(), simpleGenerationRequest())
|
||||
if result != nil || !errors.Is(err, tc.cause) || !errors.Is(err, promptkit.ErrLLMGenerate) || !errors.Is(err, promptkit.ErrTransport) {
|
||||
t.Fatalf("context result=%#v, error=%v", result, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestInjectedGenerationFailureCategories(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
client *fakeLLMClient
|
||||
malformed bool
|
||||
}{
|
||||
{name: "nil response", client: &fakeLLMClient{}, malformed: true},
|
||||
{name: "arbitrary error", client: &fakeLLMClient{err: errors.New("injected failure")}},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
engine := newSimpleEngine(t, "message", promptkit.WithLLMClient(tc.client))
|
||||
result, err := engine.Run(context.Background(), simpleGenerationRequest())
|
||||
if result != nil || !errors.Is(err, promptkit.ErrLLMGenerate) || errors.Is(err, promptkit.ErrTransport) || errors.Is(err, promptkit.ErrMalformedResponse) != tc.malformed {
|
||||
t.Fatalf("injected result=%#v, error=%v", result, err)
|
||||
}
|
||||
if tc.client.err != nil && !errors.Is(err, tc.client.err) {
|
||||
t.Fatalf("injected cause lost: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func assertGenerationError(t *testing.T, err error, statusCode int, code, providerType, message string) {
|
||||
t.Helper()
|
||||
|
||||
if !errors.Is(err, promptkit.ErrLLMGenerate) {
|
||||
t.Fatalf("errors.Is(%v, ErrLLMGenerate) = false", err)
|
||||
}
|
||||
var generationErr *promptkit.GenerationError
|
||||
if !errors.As(err, &generationErr) || generationErr == nil {
|
||||
t.Fatalf("error = %T, want *GenerationError", err)
|
||||
}
|
||||
if generationErr.StatusCode() != statusCode || generationErr.ProviderCode() != code || generationErr.ProviderType() != providerType || generationErr.ProviderMessage() != message {
|
||||
t.Fatalf("GenerationError = %#v", generationErr)
|
||||
}
|
||||
|
||||
wantFormatted := fmt.Sprintf("failed to generate output: provider returned HTTP status %d", statusCode)
|
||||
for _, rendered := range []string{fmt.Sprintf("%v", generationErr), fmt.Sprintf("%+v", generationErr), fmt.Sprintf("%#v", generationErr)} {
|
||||
if rendered != wantFormatted {
|
||||
t.Fatalf("formatted error = %q, want %q", rendered, wantFormatted)
|
||||
}
|
||||
for _, marker := range []string{code, providerType, message} {
|
||||
if marker != "" && strings.Contains(rendered, marker) {
|
||||
t.Fatalf("formatted error exposed provider marker %q: %q", marker, rendered)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func newBuiltInGenerationErrorEngine(t *testing.T, statusCode int, body string) *promptkit.Engine {
|
||||
t.Helper()
|
||||
|
||||
config := simpleConfig("message")
|
||||
noRetries := 0
|
||||
config.TransientRetries = &noRetries
|
||||
config.HTTPClient = &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
||||
return &http.Response{
|
||||
StatusCode: statusCode,
|
||||
ContentLength: int64(len(body)),
|
||||
Body: io.NopCloser(strings.NewReader(body)),
|
||||
}, nil
|
||||
})}
|
||||
engine, err := promptkit.NewEngine(config)
|
||||
if err != nil {
|
||||
t.Fatalf("NewEngine: %v", err)
|
||||
}
|
||||
return engine
|
||||
}
|
||||
|
||||
func simpleGenerationRequest() promptkit.RunRequest {
|
||||
return promptkit.RunRequest{PromptID: "prompt"}
|
||||
}
|
||||
32
generation_error_internal_test.go
Normal file
32
generation_error_internal_test.go
Normal file
@@ -0,0 +1,32 @@
|
||||
package promptkit
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestGenerationErrorNilAndZeroValue(t *testing.T) {
|
||||
var nilError *GenerationError
|
||||
zeroError := &GenerationError{}
|
||||
|
||||
for name, err := range map[string]*GenerationError{
|
||||
"nil": nilError,
|
||||
"zero": zeroError,
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
if err.StatusCode() != 0 || err.ProviderCode() != "" || err.ProviderType() != "" || err.ProviderMessage() != "" {
|
||||
t.Fatalf("accessors returned provider details: %#v", err)
|
||||
}
|
||||
if err.Error() != "failed to generate output" || err.GoString() != "failed to generate output" {
|
||||
t.Fatalf("redacted formatting = (%q, %q)", err.Error(), err.GoString())
|
||||
}
|
||||
if fmt.Sprintf("%v", err) != "failed to generate output" || fmt.Sprintf("%#v", err) != "failed to generate output" {
|
||||
t.Fatalf("formatted error = (%q, %q)", fmt.Sprintf("%v", err), fmt.Sprintf("%#v", err))
|
||||
}
|
||||
if !errors.Is(err, ErrLLMGenerate) {
|
||||
t.Fatalf("errors.Is(%v, ErrLLMGenerate) = false", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
9
go.mod
9
go.mod
@@ -3,8 +3,13 @@ module gitea.maximumdirect.net/eric/promptkit
|
||||
go 1.25.5
|
||||
|
||||
require (
|
||||
gitea.maximumdirect.net/eric/ConfigKit v0.5.0
|
||||
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v3 v3.20260930.0
|
||||
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v3 v3.20260930.0
|
||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
)
|
||||
|
||||
require golang.org/x/text v0.14.0 // indirect
|
||||
require (
|
||||
go.yaml.in/yaml/v4 v4.0.0-rc.6 // indirect
|
||||
golang.org/x/text v0.14.0 // indirect
|
||||
)
|
||||
|
||||
10
go.sum
10
go.sum
@@ -1,10 +1,16 @@
|
||||
gitea.maximumdirect.net/eric/ConfigKit v0.5.0 h1:2KwG10+kApWM3avOoYA+p768xcbHUp7tQ2rqizBNpyg=
|
||||
gitea.maximumdirect.net/eric/ConfigKit v0.5.0/go.mod h1:tYWpsrbLyu05r8+1MapWp5fN6ZrX9I8hU2xEPedYmYk=
|
||||
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v3 v3.20260930.0 h1:lzfCDbzJQRGzIlRPRprnAV9hVVL5zUQy80VMQ0kbYa0=
|
||||
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v3 v3.20260930.0/go.mod h1:HjS0f+3q0IdHCTEz+VVAEgIwIWJFAZUjuKdAtvtqKdg=
|
||||
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v3 v3.20260930.0 h1:h12ZTRldAmpLAomj0oQRGteFxHLN1LPE6XDl9zJNbm8=
|
||||
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v3 v3.20260930.0/go.mod h1:OEFBeKdjTsLzqJwLmXI2O5gG3r64FktcVJVb3mBrRfg=
|
||||
github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI=
|
||||
github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
|
||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ=
|
||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU=
|
||||
go.yaml.in/yaml/v4 v4.0.0-rc.6 h1:1h7H1ohdUh93/FyE4YaDa1Zh64K6VVbjF4K6WUxMtH4=
|
||||
go.yaml.in/yaml/v4 v4.0.0-rc.6/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0=
|
||||
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
|
||||
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
|
||||
@@ -5,69 +5,40 @@ package backend
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/jsonvalue"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/llm"
|
||||
)
|
||||
|
||||
const (
|
||||
// OpenRouterID is the reserved ID of Promptkit's built-in OpenRouter
|
||||
// backend.
|
||||
// OpenRouterID identifies the maintained OpenRouter backend.
|
||||
OpenRouterID = "openrouter"
|
||||
|
||||
openRouterEndpoint = "https://openrouter.ai/api/v1"
|
||||
openRouterAPIKeyEnv = "OPENROUTER_API_KEY"
|
||||
|
||||
openRouterConcurrencyLimit = 16
|
||||
defaultQueueCapacity = 1024
|
||||
// RakestrawHomeID identifies the maintained Rakestrawhome backend.
|
||||
RakestrawHomeID = "rakestrawhome"
|
||||
)
|
||||
|
||||
// ErrBackendNotFound identifies a registry lookup for an unknown backend ID.
|
||||
var ErrBackendNotFound = errors.New("backend not found")
|
||||
|
||||
var environmentVariableName = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`)
|
||||
|
||||
// Registry is an immutable collection of validated backend definitions.
|
||||
type Registry struct {
|
||||
backends map[string]domain.Backend
|
||||
}
|
||||
|
||||
// NewRegistry constructs a registry containing the built-in OpenRouter
|
||||
// definition followed by the supplied additions. Every ID must be unique.
|
||||
func NewRegistry(additions []domain.Backend) (*Registry, error) {
|
||||
registry := &Registry{
|
||||
backends: make(map[string]domain.Backend, len(additions)+1),
|
||||
}
|
||||
|
||||
definitions := make([]domain.Backend, 0, len(additions)+1)
|
||||
definitions = append(definitions, domain.Backend{
|
||||
ID: OpenRouterID,
|
||||
Endpoint: openRouterEndpoint,
|
||||
APIKeyEnv: openRouterAPIKeyEnv,
|
||||
ConcurrencyLimit: openRouterConcurrencyLimit,
|
||||
})
|
||||
definitions = append(definitions, additions...)
|
||||
|
||||
for _, definition := range definitions {
|
||||
definition.ID = strings.TrimSpace(definition.ID)
|
||||
if definition.ID == "" {
|
||||
return nil, errors.New("backend ID must not be blank")
|
||||
}
|
||||
if _, exists := registry.backends[definition.ID]; exists {
|
||||
return nil, fmt.Errorf("backend ID %q is already registered", definition.ID)
|
||||
}
|
||||
|
||||
normalized, err := normalizeBackend(definition)
|
||||
// NewEffectiveRegistry stores already-validated effective backend definitions
|
||||
// under their exact configuration keys. Configuration validation and
|
||||
// normalization precede this ownership boundary.
|
||||
func NewEffectiveRegistry(definitions map[string]domain.Backend) (*Registry, error) {
|
||||
registry := &Registry{backends: make(map[string]domain.Backend, len(definitions))}
|
||||
for id, definition := range definitions {
|
||||
params, err := jsonvalue.CopyMap(definition.ExtraParams)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, fmt.Errorf("backend %q extra parameters: %w", id, err)
|
||||
}
|
||||
registry.backends[normalized.ID] = normalized
|
||||
definition.ID = id
|
||||
definition.ExtraParams = params
|
||||
registry.backends[id] = definition
|
||||
}
|
||||
|
||||
return registry, nil
|
||||
}
|
||||
|
||||
@@ -106,79 +77,3 @@ func (r *Registry) CapacityPolicies() map[string]domain.BackendCapacityPolicy {
|
||||
}
|
||||
return policies
|
||||
}
|
||||
|
||||
func normalizeBackend(definition domain.Backend) (domain.Backend, error) {
|
||||
endpoint, err := domain.NormalizeOpenAICompatibleBaseEndpoint(definition.Endpoint)
|
||||
if err != nil {
|
||||
return domain.Backend{}, fmt.Errorf("backend %q endpoint: %w", definition.ID, err)
|
||||
}
|
||||
definition.Endpoint = endpoint
|
||||
|
||||
definition.APIKeyEnv = strings.TrimSpace(definition.APIKeyEnv)
|
||||
if definition.APIKeyEnv != "" && !environmentVariableName.MatchString(definition.APIKeyEnv) {
|
||||
return domain.Backend{}, fmt.Errorf(
|
||||
"backend %q api key environment variable %q is invalid",
|
||||
definition.ID,
|
||||
definition.APIKeyEnv,
|
||||
)
|
||||
}
|
||||
|
||||
if definition.ConcurrencyLimit < 0 {
|
||||
return domain.Backend{}, fmt.Errorf(
|
||||
"backend %q concurrency limit must not be negative",
|
||||
definition.ID,
|
||||
)
|
||||
}
|
||||
if definition.QueueCapacity < 0 {
|
||||
return domain.Backend{}, fmt.Errorf(
|
||||
"backend %q queue capacity must not be negative",
|
||||
definition.ID,
|
||||
)
|
||||
}
|
||||
if definition.ConcurrencyLimit == 0 {
|
||||
if definition.QueueCapacitySet {
|
||||
return domain.Backend{}, fmt.Errorf(
|
||||
"backend %q queue capacity requires a positive concurrency limit",
|
||||
definition.ID,
|
||||
)
|
||||
}
|
||||
definition.QueueCapacity = 0
|
||||
} else {
|
||||
if !definition.QueueCapacitySet {
|
||||
definition.QueueCapacity = defaultQueueCapacity
|
||||
definition.QueueCapacitySet = true
|
||||
}
|
||||
maxInt := int(^uint(0) >> 1)
|
||||
if definition.QueueCapacity > maxInt-definition.ConcurrencyLimit {
|
||||
return domain.Backend{}, fmt.Errorf(
|
||||
"backend %q total capacity overflows int",
|
||||
definition.ID,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
keys := make([]string, 0, len(definition.ExtraParams))
|
||||
for key := range definition.ExtraParams {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, key := range keys {
|
||||
if key == "" {
|
||||
return domain.Backend{}, fmt.Errorf("backend %q extra parameter key must not be empty", definition.ID)
|
||||
}
|
||||
if llm.IsReservedOpenAIChatRequestField(key) {
|
||||
return domain.Backend{}, fmt.Errorf(
|
||||
"backend %q extra parameter %q collides with a reserved request field",
|
||||
definition.ID,
|
||||
key,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
extraParams, err := jsonvalue.CopyMap(definition.ExtraParams)
|
||||
if err != nil {
|
||||
return domain.Backend{}, fmt.Errorf("backend %q extra parameters: %w", definition.ID, err)
|
||||
}
|
||||
definition.ExtraParams = extraParams
|
||||
return definition, nil
|
||||
}
|
||||
|
||||
@@ -2,367 +2,55 @@ package backend_test
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/backend"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
)
|
||||
|
||||
const validEndpoint = "https://backend.example/v1"
|
||||
|
||||
func TestRegistryIncludesExactOpenRouterDefinition(t *testing.T) {
|
||||
registry, err := backend.NewRegistry(nil)
|
||||
if err != nil {
|
||||
t.Fatalf("construct registry: %v", err)
|
||||
}
|
||||
|
||||
definition, err := registry.GetBackend(backend.OpenRouterID)
|
||||
if err != nil {
|
||||
t.Fatalf("look up OpenRouter: %v", err)
|
||||
}
|
||||
if definition.ID != "openrouter" ||
|
||||
definition.Endpoint != "https://openrouter.ai/api/v1" ||
|
||||
definition.APIKeyEnv != "OPENROUTER_API_KEY" ||
|
||||
definition.ConcurrencyLimit != 16 ||
|
||||
definition.QueueCapacity != 1024 ||
|
||||
!definition.QueueCapacitySet ||
|
||||
definition.ExtraParams != nil {
|
||||
t.Fatalf("unexpected OpenRouter definition: %#v", definition)
|
||||
}
|
||||
policies := registry.CapacityPolicies()
|
||||
if len(policies) != 1 ||
|
||||
policies["openrouter"] != (domain.BackendCapacityPolicy{
|
||||
ConcurrencyLimit: 16,
|
||||
QueueCapacity: 1024,
|
||||
}) {
|
||||
t.Fatalf("unexpected OpenRouter capacity policies: %#v", policies)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegistryNormalizesUniqueAdditionsAndIsolatesMutations(t *testing.T) {
|
||||
nested := map[string]int{"limit": 2}
|
||||
extraParams := map[string]any{
|
||||
"count": int64(7),
|
||||
"nested": nested,
|
||||
}
|
||||
registry, err := backend.NewRegistry([]domain.Backend{
|
||||
{
|
||||
ID: " custom ",
|
||||
Endpoint: " https://custom.example/openai/v1 ",
|
||||
APIKeyEnv: " CUSTOM_API_KEY ",
|
||||
ExtraParams: extraParams,
|
||||
ConcurrencyLimit: 3,
|
||||
QueueCapacity: 2,
|
||||
QueueCapacitySet: true,
|
||||
},
|
||||
{
|
||||
ID: "Custom",
|
||||
Endpoint: validEndpoint,
|
||||
func TestEffectiveRegistryOwnsDefinitionsAndCapacityPolicies(t *testing.T) {
|
||||
params := map[string]any{"nested": map[string]any{"enabled": true}}
|
||||
registry, err := backend.NewEffectiveRegistry(map[string]domain.Backend{
|
||||
" local.v1 ": {
|
||||
Endpoint: "https://backend.example/v1", ExtraParams: params,
|
||||
ConcurrencyLimit: 2, QueueCapacity: 3, QueueCapacitySet: true,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("construct registry: %v", err)
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
nested["limit"] = 99
|
||||
extraParams["added"] = true
|
||||
|
||||
got, err := registry.GetBackend("custom")
|
||||
if err != nil {
|
||||
t.Fatalf("look up custom backend: %v", err)
|
||||
params["nested"].(map[string]any)["enabled"] = false
|
||||
selected, err := registry.GetBackend(" local.v1 ")
|
||||
if err != nil || selected.ID != " local.v1 " || selected.ExtraParams["nested"].(map[string]any)["enabled"] != true {
|
||||
t.Fatalf("effective backend was not captured under its literal ID: %+v, %v", selected, err)
|
||||
}
|
||||
if got.ID != "custom" ||
|
||||
got.Endpoint != "https://custom.example/openai/v1" ||
|
||||
got.APIKeyEnv != "CUSTOM_API_KEY" ||
|
||||
got.ConcurrencyLimit != 3 ||
|
||||
got.QueueCapacity != 2 ||
|
||||
!got.QueueCapacitySet {
|
||||
t.Fatalf("unexpected normalized definition: %#v", got)
|
||||
selected.ExtraParams["nested"].(map[string]any)["enabled"] = false
|
||||
again, err := registry.GetBackend(" local.v1 ")
|
||||
if err != nil || again.ExtraParams["nested"].(map[string]any)["enabled"] != true {
|
||||
t.Fatalf("lookup exposed registry values: %+v, %v", again, err)
|
||||
}
|
||||
if count, ok := got.ExtraParams["count"].(int64); !ok || count != 7 {
|
||||
t.Fatalf("integer type or value changed: %#v", got.ExtraParams["count"])
|
||||
}
|
||||
gotNested, ok := got.ExtraParams["nested"].(map[string]int)
|
||||
if !ok || gotNested["limit"] != 2 {
|
||||
t.Fatalf("container type or value changed: %#v", got.ExtraParams["nested"])
|
||||
}
|
||||
if _, exists := got.ExtraParams["added"]; exists {
|
||||
t.Fatalf("registry retained caller map: %#v", got.ExtraParams)
|
||||
}
|
||||
|
||||
gotNested["limit"] = 100
|
||||
got.ExtraParams["added"] = true
|
||||
again, err := registry.GetBackend("custom")
|
||||
if err != nil {
|
||||
t.Fatalf("look up custom backend again: %v", err)
|
||||
}
|
||||
if again.ExtraParams["nested"].(map[string]int)["limit"] != 2 {
|
||||
t.Fatalf("lookup exposed registry nested map: %#v", again.ExtraParams)
|
||||
}
|
||||
if _, exists := again.ExtraParams["added"]; exists {
|
||||
t.Fatalf("lookup exposed registry map: %#v", again.ExtraParams)
|
||||
}
|
||||
|
||||
if _, err := registry.GetBackend("Custom"); err != nil {
|
||||
t.Fatalf("backend IDs should be case-sensitive: %v", err)
|
||||
}
|
||||
|
||||
policies := registry.CapacityPolicies()
|
||||
if len(policies) != 2 {
|
||||
t.Fatalf("unexpected capacity policy count: %#v", policies)
|
||||
}
|
||||
policies["custom"] = domain.BackendCapacityPolicy{}
|
||||
delete(policies, backend.OpenRouterID)
|
||||
againPolicies := registry.CapacityPolicies()
|
||||
if againPolicies["custom"] != (domain.BackendCapacityPolicy{
|
||||
ConcurrencyLimit: 3,
|
||||
QueueCapacity: 2,
|
||||
if !reflect.DeepEqual(policies, map[string]domain.BackendCapacityPolicy{
|
||||
" local.v1 ": {ConcurrencyLimit: 2, QueueCapacity: 3},
|
||||
}) {
|
||||
t.Fatalf("capacity policy map mutated registry state: %#v", againPolicies)
|
||||
t.Fatalf("capacity policy = %+v", policies)
|
||||
}
|
||||
if _, ok := againPolicies[backend.OpenRouterID]; !ok {
|
||||
t.Fatalf("capacity policy deletion mutated registry state: %#v", againPolicies)
|
||||
delete(policies, " local.v1 ")
|
||||
if len(registry.CapacityPolicies()) != 1 {
|
||||
t.Fatal("capacity policy map exposed registry state")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewRegistryNormalizesCapacityPolicy(t *testing.T) {
|
||||
maxInt := int(^uint(0) >> 1)
|
||||
tests := []struct {
|
||||
name string
|
||||
definition domain.Backend
|
||||
want domain.BackendCapacityPolicy
|
||||
wantSet bool
|
||||
wantError bool
|
||||
}{
|
||||
{
|
||||
name: "unlimited when omitted",
|
||||
definition: domain.Backend{},
|
||||
},
|
||||
{
|
||||
name: "default queue",
|
||||
definition: domain.Backend{
|
||||
ConcurrencyLimit: 2,
|
||||
},
|
||||
want: domain.BackendCapacityPolicy{
|
||||
ConcurrencyLimit: 2,
|
||||
QueueCapacity: 1024,
|
||||
},
|
||||
wantSet: true,
|
||||
},
|
||||
{
|
||||
name: "explicit zero queue",
|
||||
definition: domain.Backend{
|
||||
ConcurrencyLimit: 2,
|
||||
QueueCapacitySet: true,
|
||||
},
|
||||
want: domain.BackendCapacityPolicy{
|
||||
ConcurrencyLimit: 2,
|
||||
},
|
||||
wantSet: true,
|
||||
},
|
||||
{
|
||||
name: "negative concurrency limit",
|
||||
definition: domain.Backend{
|
||||
ConcurrencyLimit: -1,
|
||||
},
|
||||
wantError: true,
|
||||
},
|
||||
{
|
||||
name: "negative queue capacity",
|
||||
definition: domain.Backend{
|
||||
ConcurrencyLimit: 1,
|
||||
QueueCapacity: -1,
|
||||
QueueCapacitySet: true,
|
||||
},
|
||||
wantError: true,
|
||||
},
|
||||
{
|
||||
name: "queue without limit",
|
||||
definition: domain.Backend{
|
||||
QueueCapacitySet: true,
|
||||
},
|
||||
wantError: true,
|
||||
},
|
||||
{
|
||||
name: "total overflow",
|
||||
definition: domain.Backend{
|
||||
ConcurrencyLimit: maxInt,
|
||||
QueueCapacity: 1,
|
||||
QueueCapacitySet: true,
|
||||
},
|
||||
wantError: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
tc.definition.ID = "custom"
|
||||
tc.definition.Endpoint = validEndpoint
|
||||
registry, err := backend.NewRegistry([]domain.Backend{tc.definition})
|
||||
if tc.wantError {
|
||||
if err == nil {
|
||||
t.Fatal("expected invalid capacity policy error")
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("construct registry: %v", err)
|
||||
}
|
||||
|
||||
definition, err := registry.GetBackend("custom")
|
||||
if err != nil {
|
||||
t.Fatalf("look up custom backend: %v", err)
|
||||
}
|
||||
if definition.ConcurrencyLimit != tc.want.ConcurrencyLimit ||
|
||||
definition.QueueCapacity != tc.want.QueueCapacity ||
|
||||
definition.QueueCapacitySet != tc.wantSet {
|
||||
t.Fatalf("normalized capacity=(%d, %d, %t), want (%d, %d, %t)",
|
||||
definition.ConcurrencyLimit,
|
||||
definition.QueueCapacity,
|
||||
definition.QueueCapacitySet,
|
||||
tc.want.ConcurrencyLimit,
|
||||
tc.want.QueueCapacity,
|
||||
tc.wantSet,
|
||||
)
|
||||
}
|
||||
policies := registry.CapacityPolicies()
|
||||
got, ok := policies["custom"]
|
||||
if ok != tc.wantSet || got != tc.want {
|
||||
t.Fatalf("capacity policy=(%#v, %t), want (%#v, %t)", got, ok, tc.want, tc.wantSet)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewRegistryRejectsDuplicateIDs(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
additions []domain.Backend
|
||||
wantID string
|
||||
}{
|
||||
{
|
||||
name: "built-in collision after normalization",
|
||||
additions: []domain.Backend{{
|
||||
ID: " openrouter ",
|
||||
}},
|
||||
wantID: "openrouter",
|
||||
},
|
||||
{
|
||||
name: "consumer collision after normalization",
|
||||
additions: []domain.Backend{
|
||||
{ID: "custom", Endpoint: validEndpoint},
|
||||
{ID: " custom ", Endpoint: "https://other.example/v1"},
|
||||
},
|
||||
wantID: "custom",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
_, err := backend.NewRegistry(tc.additions)
|
||||
if err == nil {
|
||||
t.Fatal("expected duplicate ID error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), tc.wantID) {
|
||||
t.Fatalf("expected error to identify %q, got %v", tc.wantID, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewRegistryValidatesIDs(t *testing.T) {
|
||||
for _, id := range []string{"", " \t\n "} {
|
||||
t.Run(id, func(t *testing.T) {
|
||||
_, err := backend.NewRegistry([]domain.Backend{{
|
||||
ID: id,
|
||||
Endpoint: validEndpoint,
|
||||
}})
|
||||
if err == nil {
|
||||
t.Fatal("expected blank ID error")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewRegistryValidatesEndpoints(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
endpoint string
|
||||
}{
|
||||
{name: "blank", endpoint: ""},
|
||||
{name: "relative", endpoint: "/v1"},
|
||||
{name: "missing host", endpoint: "https:///v1"},
|
||||
{name: "unsupported scheme", endpoint: "ftp://backend.example/v1"},
|
||||
{name: "user information", endpoint: "https://user@backend.example/v1"},
|
||||
{name: "query", endpoint: "https://backend.example/v1?mode=chat"},
|
||||
{name: "empty query", endpoint: "https://backend.example/v1?"},
|
||||
{name: "fragment", endpoint: "https://backend.example/v1#chat"},
|
||||
{name: "empty fragment", endpoint: "https://backend.example/v1#"},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
_, err := backend.NewRegistry([]domain.Backend{{
|
||||
ID: "custom",
|
||||
Endpoint: tc.endpoint,
|
||||
}})
|
||||
if err == nil {
|
||||
t.Fatal("expected invalid endpoint error")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewRegistryValidatesEnvironmentVariableNames(t *testing.T) {
|
||||
for _, name := range []string{"1API_KEY", "API-KEY", "API KEY", "ÅPI_KEY"} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
_, err := backend.NewRegistry([]domain.Backend{{
|
||||
ID: "custom",
|
||||
Endpoint: validEndpoint,
|
||||
APIKeyEnv: name,
|
||||
}})
|
||||
if err == nil {
|
||||
t.Fatal("expected invalid environment-variable name error")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewRegistryRejectsInvalidAndReservedExtraParameters(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
extraParams map[string]any
|
||||
}{
|
||||
{name: "unsupported value", extraParams: map[string]any{"value": make(chan int)}},
|
||||
{name: "reserved key", extraParams: map[string]any{"model": "override"}},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
_, err := backend.NewRegistry([]domain.Backend{{
|
||||
ID: "custom",
|
||||
Endpoint: validEndpoint,
|
||||
ExtraParams: tc.extraParams,
|
||||
}})
|
||||
if err == nil {
|
||||
t.Fatal("expected invalid extra parameters error")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegistryLookupReportsNotFound(t *testing.T) {
|
||||
registry, err := backend.NewRegistry(nil)
|
||||
func TestEffectiveRegistryMissingBackend(t *testing.T) {
|
||||
registry, err := backend.NewEffectiveRegistry(nil)
|
||||
if err != nil {
|
||||
t.Fatalf("construct registry: %v", err)
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
_, err = registry.GetBackend("missing")
|
||||
if !errors.Is(err, backend.ErrBackendNotFound) {
|
||||
t.Fatalf("expected ErrBackendNotFound, got %v", err)
|
||||
if _, err := registry.GetBackend("missing"); !errors.Is(err, backend.ErrBackendNotFound) {
|
||||
t.Fatalf("missing backend error = %v", err)
|
||||
}
|
||||
if !strings.Contains(err.Error(), "missing") {
|
||||
t.Fatalf("expected error to identify backend, got %v", err)
|
||||
if _, err := registry.GetBackend("local.v1"); !errors.Is(err, backend.ErrBackendNotFound) {
|
||||
t.Fatalf("trimmed lookup unexpectedly matched: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -28,13 +28,13 @@ func (c *client) Generate(
|
||||
ctx context.Context,
|
||||
req domain.GenerateRequest,
|
||||
) (*domain.GenerateResponse, error) {
|
||||
pool := c.manager.getPool(req.Target.BackendID)
|
||||
if pool == nil {
|
||||
if c.manager.getPool(req.Target.BackendID) == nil {
|
||||
return c.next.Generate(ctx, req)
|
||||
}
|
||||
if err := pool.acquire(ctx); err != nil {
|
||||
release, err := c.manager.Acquire(ctx, req.Target.BackendID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer pool.releaseActive()
|
||||
defer release()
|
||||
return c.next.Generate(ctx, req)
|
||||
}
|
||||
|
||||
@@ -107,6 +107,26 @@ func (m *Manager) getPool(backendID string) *pool {
|
||||
return m.pools[backendID]
|
||||
}
|
||||
|
||||
// Limited reports whether a backend uses an active-generation pool.
|
||||
func (m *Manager) Limited(backendID string) bool { return m.getPool(backendID) != nil }
|
||||
|
||||
// Acquire obtains one active-generation permit for a backend. Its release
|
||||
// function is idempotent. Backends without a configured pool are unlimited.
|
||||
func (m *Manager) Acquire(ctx context.Context, backendID string) (func(), error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
pool := m.getPool(backendID)
|
||||
if pool == nil {
|
||||
return releaseNothing, nil
|
||||
}
|
||||
if err := pool.acquire(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var once sync.Once
|
||||
return func() { once.Do(pool.releaseActive) }, nil
|
||||
}
|
||||
|
||||
func (p *pool) acquire(ctx context.Context) error {
|
||||
p.mu.Lock()
|
||||
if err := ctx.Err(); err != nil {
|
||||
|
||||
@@ -60,6 +60,26 @@ func TestNewManagerRejectsInvalidPolicies(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestManagerAcquireReleaseIsIdempotent(t *testing.T) {
|
||||
manager, err := NewManager(map[string]domain.BackendCapacityPolicy{
|
||||
"limited": {ConcurrencyLimit: 1},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
release, err := manager.Acquire(context.Background(), "limited")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
release()
|
||||
release()
|
||||
other, err := manager.Acquire(context.Background(), "limited")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
other()
|
||||
}
|
||||
|
||||
func TestManagerAdmissionIsBoundedAndReleaseIsIdempotent(t *testing.T) {
|
||||
policies := map[string]domain.BackendCapacityPolicy{
|
||||
"limited": {
|
||||
|
||||
52
internal/catalog/catalog.go
Normal file
52
internal/catalog/catalog.go
Normal file
@@ -0,0 +1,52 @@
|
||||
// Package catalog selects the maintained configuration assets supplied with
|
||||
// PromptKit. ConfigKit owns their composition and effective validation.
|
||||
package catalog
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
|
||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
||||
openrouter "gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v3"
|
||||
rakestrawhome "gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v3"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/backend"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/filecatalog"
|
||||
)
|
||||
|
||||
// Underlays selects external catalogs followed by PromptKit-owned profiles.
|
||||
// Neither group is interpreted before the final effective configuration loads.
|
||||
func Underlays() ([]configkit.Underlay, error) {
|
||||
openrouterInputs, err := assetInputs("OpenRouter catalog", openrouter.FS(), openrouter.Root)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rakestrawhomeInputs, err := assetInputs("Rakestrawhome catalog", rakestrawhome.FS(), rakestrawhome.Root)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
catalogInputs := append(openrouterInputs, rakestrawhomeInputs...)
|
||||
owned := configkit.Input{Source: configkit.Mapping("PromptKit-owned profiles", map[string]any{
|
||||
"profiles": map[string]any{
|
||||
"promptkit-openrouter-standard": map[string]any{"extends": "gemini-flash-latest"},
|
||||
"promptkit-openrouter-auto": map[string]any{"backend": backend.OpenRouterID, "model": "openrouter/auto"},
|
||||
},
|
||||
})}
|
||||
return []configkit.Underlay{{Inputs: catalogInputs}, {Inputs: []configkit.Input{owned}}}, nil
|
||||
}
|
||||
|
||||
func assetInputs(label string, filesystem fs.FS, root string) ([]configkit.Input, error) {
|
||||
files, err := filecatalog.FindFSYAMLFiles(context.Background(), filesystem, root)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("discover %s configuration: %w", label, err)
|
||||
}
|
||||
namespace := configkit.NewFilesystem(label, filesystem)
|
||||
inputs := make([]configkit.Input, 0, len(files))
|
||||
for _, name := range files {
|
||||
inputs = append(inputs, configkit.Input{
|
||||
Source: namespace.File(name),
|
||||
Filesystem: &configkit.FilesystemContext{FS: &namespace, Root: root},
|
||||
})
|
||||
}
|
||||
return inputs, nil
|
||||
}
|
||||
68
internal/catalog/catalog_test.go
Normal file
68
internal/catalog/catalog_test.go
Normal file
@@ -0,0 +1,68 @@
|
||||
package catalog
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/configuration"
|
||||
)
|
||||
|
||||
func TestPublishedCatalogsResolveThroughEffectiveConfiguration(t *testing.T) {
|
||||
underlays, err := Underlays()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
state, err := configuration.Load(configkit.Input{Source: configkit.NoSource()}, underlays)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
openrouter, err := state.GetBackend("openrouter")
|
||||
if err != nil || openrouter.Endpoint != "https://openrouter.ai/api/v1" || openrouter.ConcurrencyLimit != 16 || openrouter.QueueCapacity != 1024 {
|
||||
t.Fatalf("OpenRouter backend: %+v, %v", openrouter, err)
|
||||
}
|
||||
rakestrawhome, err := state.GetBackend("rakestrawhome")
|
||||
if err != nil || rakestrawhome.Endpoint != "https://inference.ai.rakestrawhome.com/v1" || rakestrawhome.ConcurrencyLimit != 4 {
|
||||
t.Fatalf("Rakestrawhome backend: %+v, %v", rakestrawhome, err)
|
||||
}
|
||||
for id, want := range map[string]string{
|
||||
"promptkit-openrouter-standard": "~google/gemini-flash-latest",
|
||||
"promptkit-openrouter-auto": "openrouter/auto",
|
||||
"rakestrawhome-gemma-4-31b": "google/gemma-4-31b-it",
|
||||
} {
|
||||
profile, err := state.GetProfile(context.Background(), id)
|
||||
if err != nil || profile.ResolvedTarget.Model != want {
|
||||
t.Fatalf("profile %s: %+v, %v", id, profile, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestConsumerOverlayReplacesCatalogDefaultsBeforeValidation(t *testing.T) {
|
||||
underlays, err := Underlays()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
underlays = append(underlays, configkit.Underlay{Inputs: []configkit.Input{{Source: configkit.Mapping("operator", map[string]any{
|
||||
"backends": map[string]any{"openrouter": map[string]any{"endpoint": "invalid lower value"}},
|
||||
})}}})
|
||||
main := configkit.Input{Source: configkit.Mapping("application", map[string]any{
|
||||
"backends": map[string]any{"openrouter": map[string]any{"endpoint": "https://alternate.example/v1"}},
|
||||
"profiles": map[string]any{"gemini-flash-latest": map[string]any{"model": "consumer-model"}},
|
||||
})}
|
||||
state, err := configuration.Load(main, underlays)
|
||||
if err != nil {
|
||||
t.Fatalf("replaced lower-layer value failed final validation: %v", err)
|
||||
}
|
||||
profile, err := state.GetProfile(context.Background(), "promptkit-openrouter-standard")
|
||||
if err != nil || profile.ResolvedTarget.Model != "consumer-model" || profile.ResolvedTarget.Endpoint != "https://alternate.example/v1" {
|
||||
t.Fatalf("maintained profile did not observe final configuration: %+v, %v", profile, err)
|
||||
}
|
||||
_, err = configuration.Load(configkit.Input{Source: configkit.Mapping("invalid final", map[string]any{
|
||||
"backends": map[string]any{"openrouter": map[string]any{"endpoint": "invalid final value"}},
|
||||
})}, underlays[:2])
|
||||
var loadError *configkit.LoadError
|
||||
if !errors.As(err, &loadError) {
|
||||
t.Fatalf("invalid final backend did not produce a ConfigKit load error: %v", err)
|
||||
}
|
||||
}
|
||||
179
internal/configuration/benchmark_test.go
Normal file
179
internal/configuration/benchmark_test.go
Normal file
@@ -0,0 +1,179 @@
|
||||
package configuration
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
"testing/fstest"
|
||||
|
||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
)
|
||||
|
||||
const benchmarkDefinitionCount = 128
|
||||
|
||||
var (
|
||||
constructionResult *loaded
|
||||
selectedPrompt *domain.PromptDefinition
|
||||
selectedProfile *domain.ExecutionProfile
|
||||
)
|
||||
|
||||
type configurationShape struct {
|
||||
main configkit.Input
|
||||
underlays []configkit.Underlay
|
||||
inputEntries int
|
||||
effectiveEntries int
|
||||
tiers int
|
||||
}
|
||||
|
||||
func benchmarkPrompt(profileID string) map[string]any {
|
||||
return map[string]any{
|
||||
"default_profile": profileID,
|
||||
"messages": []any{map[string]any{"role": "user", "content": "Hello {{.name}}"}},
|
||||
"output": map[string]any{"format": "text", "validation_mode": "none"},
|
||||
}
|
||||
}
|
||||
|
||||
func independentDefinitions(count int) configurationShape {
|
||||
profiles := make(map[string]any, count)
|
||||
prompts := make(map[string]any, count)
|
||||
for index := range count {
|
||||
profileID := fmt.Sprintf("profile-%03d", index)
|
||||
promptID := fmt.Sprintf("prompt-%03d", index)
|
||||
profiles[profileID] = map[string]any{"endpoint": "https://example.test/v1", "model": profileID}
|
||||
prompts[promptID] = map[string]any{"1": benchmarkPrompt(profileID)}
|
||||
}
|
||||
return configurationShape{
|
||||
main: configkit.Input{Source: configkit.Mapping("independent definitions", map[string]any{"profiles": profiles, "prompts": prompts})},
|
||||
inputEntries: count * 2,
|
||||
effectiveEntries: count * 2,
|
||||
tiers: 1,
|
||||
}
|
||||
}
|
||||
|
||||
func fileBackedDefinitions(count int) configurationShape {
|
||||
profiles := make(map[string]any, count)
|
||||
prompts := make(map[string]any, count)
|
||||
for index := range count {
|
||||
profileID := fmt.Sprintf("profile-%03d", index)
|
||||
promptID := fmt.Sprintf("prompt-%03d", index)
|
||||
profiles[profileID] = map[string]any{"endpoint": "https://example.test/v1", "model": profileID}
|
||||
prompt := benchmarkPrompt(profileID)
|
||||
prompt["messages"] = []any{map[string]any{"role": "user", "content_file": "body.md"}}
|
||||
prompts[promptID] = map[string]any{"1": prompt}
|
||||
}
|
||||
namespace := configkit.NewFilesystem("benchmark content", fstest.MapFS{
|
||||
"assets/body.md": &fstest.MapFile{Data: []byte("Hello {{.name}}")},
|
||||
})
|
||||
return configurationShape{
|
||||
main: configkit.Input{
|
||||
Source: configkit.Mapping("file-backed definitions", map[string]any{"profiles": profiles, "prompts": prompts}),
|
||||
Filesystem: &configkit.FilesystemContext{FS: &namespace, Root: ".", BaseDir: "assets"},
|
||||
},
|
||||
inputEntries: count * 2, effectiveEntries: count * 2, tiers: 1,
|
||||
}
|
||||
}
|
||||
|
||||
func sharedBaseDefinitions(count int) configurationShape {
|
||||
profiles := make(map[string]any, count+1)
|
||||
profiles["base"] = map[string]any{
|
||||
"endpoint": "https://example.test/v1", "model": "shared-model",
|
||||
"extra_params": map[string]any{"provider": map[string]any{"mode": "shared", "limit": 4}},
|
||||
}
|
||||
for index := range count {
|
||||
profiles[fmt.Sprintf("child-%03d", index)] = map[string]any{"extends": "base", "temperature": float64(index%10) / 10}
|
||||
}
|
||||
return configurationShape{
|
||||
main: configkit.Input{Source: configkit.Mapping("inherited definitions", map[string]any{
|
||||
"profiles": profiles,
|
||||
"prompts": map[string]any{"shared": map[string]any{"1": benchmarkPrompt(fmt.Sprintf("child-%03d", count-1))}},
|
||||
})},
|
||||
inputEntries: count + 2,
|
||||
effectiveEntries: count + 2,
|
||||
tiers: 1,
|
||||
}
|
||||
}
|
||||
|
||||
func layeredDefinitions(count int) configurationShape {
|
||||
base := make(map[string]any, count)
|
||||
operator := make(map[string]any, count)
|
||||
programmatic := make(map[string]any, count)
|
||||
final := make(map[string]any, count)
|
||||
for index := range count {
|
||||
id := fmt.Sprintf("profile-%03d", index)
|
||||
base[id] = map[string]any{"endpoint": "https://example.test/v1", "model": "base-model"}
|
||||
operator[id] = map[string]any{"model": "operator-model"}
|
||||
programmatic[id] = map[string]any{"extra_params": map[string]any{"provider": map[string]any{"mode": "programmatic"}}}
|
||||
final[id] = map[string]any{"temperature": float64(index%10) / 10}
|
||||
}
|
||||
return configurationShape{
|
||||
underlays: []configkit.Underlay{
|
||||
{Inputs: []configkit.Input{{Source: configkit.Mapping("fallback", map[string]any{"profiles": base})}}},
|
||||
{Inputs: []configkit.Input{{Source: configkit.Mapping("operator", map[string]any{"profiles": operator})}}},
|
||||
{Inputs: []configkit.Input{{Source: configkit.Mapping("programmatic", map[string]any{"profiles": programmatic})}}},
|
||||
},
|
||||
main: configkit.Input{Source: configkit.Mapping("main", map[string]any{
|
||||
"profiles": final,
|
||||
"prompts": map[string]any{"layered": map[string]any{"1": benchmarkPrompt(fmt.Sprintf("profile-%03d", count-1))}},
|
||||
})},
|
||||
inputEntries: count*4 + 1,
|
||||
effectiveEntries: count + 1,
|
||||
tiers: 4,
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkConfigurationConstruction(b *testing.B) {
|
||||
for _, scenario := range []struct {
|
||||
name string
|
||||
shape configurationShape
|
||||
}{
|
||||
{name: "independent", shape: independentDefinitions(benchmarkDefinitionCount)},
|
||||
{name: "file_backed", shape: fileBackedDefinitions(64)},
|
||||
{name: "shared_base", shape: sharedBaseDefinitions(benchmarkDefinitionCount)},
|
||||
{name: "layered", shape: layeredDefinitions(benchmarkDefinitionCount)},
|
||||
} {
|
||||
b.Run(scenario.name, func(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
for b.Loop() {
|
||||
result, err := load(scenario.shape.main, scenario.shape.underlays)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
constructionResult = result
|
||||
}
|
||||
b.ReportMetric(float64(scenario.shape.inputEntries), "input-entries/op")
|
||||
b.ReportMetric(float64(scenario.shape.effectiveEntries), "effective-entries/op")
|
||||
b.ReportMetric(float64(scenario.shape.tiers), "tiers/op")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkConfigurationSelection(b *testing.B) {
|
||||
shape := independentDefinitions(benchmarkDefinitionCount)
|
||||
result, err := load(shape.main, nil)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
b.Run("prompt", func(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
for b.Loop() {
|
||||
definition, _, err := result.prompt("prompt-127", "1")
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
selectedPrompt = definition
|
||||
}
|
||||
b.ReportMetric(float64(shape.effectiveEntries), "effective-entries/op")
|
||||
})
|
||||
b.Run("profile", func(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
for b.Loop() {
|
||||
profile, err := result.GetProfile(context.Background(), "profile-127")
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
selectedProfile = profile
|
||||
}
|
||||
b.ReportMetric(float64(shape.effectiveEntries), "effective-entries/op")
|
||||
})
|
||||
}
|
||||
45
internal/configuration/inspection.go
Normal file
45
internal/configuration/inspection.go
Normal file
@@ -0,0 +1,45 @@
|
||||
package configuration
|
||||
|
||||
import (
|
||||
"io"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
||||
)
|
||||
|
||||
// Inspection is the read-only subset of the retained ConfigKit result used by
|
||||
// the facade. It does not expose typed configuration or resource contexts.
|
||||
type Inspection interface {
|
||||
Metadata() configkit.Metadata
|
||||
Sources() []configkit.SourceInfo
|
||||
Diagnostics() []configkit.Diagnostic
|
||||
Explain(configkit.Path) (configkit.Explanation, bool)
|
||||
WriteEffective(io.Writer) error
|
||||
WriteExplanation(io.Writer, configkit.Path) error
|
||||
}
|
||||
|
||||
// Inspection returns only the retained result's inspection operations.
|
||||
func (state *loaded) Inspection() Inspection {
|
||||
if state == nil {
|
||||
return nil
|
||||
}
|
||||
return state.result
|
||||
}
|
||||
|
||||
func secretProviderPath(path configkit.Path) bool {
|
||||
pointer := path.String()
|
||||
if pointer == "" {
|
||||
return false
|
||||
}
|
||||
parts := strings.Split(strings.TrimPrefix(pointer, "/"), "/")
|
||||
if len(parts) < 4 || (parts[0] != "backends" && parts[0] != "profiles") || parts[2] != "extra_params" {
|
||||
return false
|
||||
}
|
||||
key := strings.ToLower(strings.ReplaceAll(strings.ReplaceAll(parts[len(parts)-1], "~1", "/"), "~0", "~"))
|
||||
switch key {
|
||||
case "api_key", "apikey", "authorization", "credential", "credentials", "password", "secret", "token", "access_token":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
317
internal/configuration/load.go
Normal file
317
internal/configuration/load.go
Normal file
@@ -0,0 +1,317 @@
|
||||
// Package configuration adapts ConfigKit's effective configuration to
|
||||
// Promptkit's definition and execution boundaries.
|
||||
package configuration
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/backend"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/jsonvalue"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/llm"
|
||||
)
|
||||
|
||||
type document struct {
|
||||
Backends map[string]backendConfig `yaml:"backends"`
|
||||
Profiles map[string]profileConfig `yaml:"profiles"`
|
||||
Prompts map[string]map[string]promptConfig `yaml:"prompts"`
|
||||
}
|
||||
|
||||
type backendConfig struct {
|
||||
Endpoint *string `yaml:"endpoint"`
|
||||
APIKeyEnv *string `yaml:"api_key_env"`
|
||||
ConcurrencyLimit *int `yaml:"concurrency_limit"`
|
||||
QueueCapacity *int `yaml:"queue_capacity"`
|
||||
ExtraParams map[string]any `yaml:"extra_params"`
|
||||
}
|
||||
|
||||
type profileConfig struct {
|
||||
Backend *string `yaml:"backend"`
|
||||
Endpoint *string `yaml:"endpoint"`
|
||||
Model *string `yaml:"model"`
|
||||
Temperature *float64 `yaml:"temperature"`
|
||||
TopP *float64 `yaml:"top_p"`
|
||||
MaxTokens *int `yaml:"max_tokens"`
|
||||
TimeoutSeconds *int `yaml:"timeout_seconds"`
|
||||
ServiceTier *string `yaml:"service_tier"`
|
||||
ReasoningEffort *string `yaml:"reasoning_effort"`
|
||||
APIKeyEnv *string `yaml:"api_key_env"`
|
||||
APIKeyRequired *bool `yaml:"api_key_required"`
|
||||
ExtraParams map[string]any `yaml:"extra_params"`
|
||||
}
|
||||
|
||||
type promptConfig struct {
|
||||
DefaultProfile *string `yaml:"default_profile"`
|
||||
Description *string `yaml:"description"`
|
||||
SessionID *string `yaml:"session_id"`
|
||||
Inputs []promptInputConfig `yaml:"inputs"`
|
||||
Messages []promptMessageConfig `yaml:"messages"`
|
||||
Output *promptOutputConfig `yaml:"output"`
|
||||
}
|
||||
|
||||
type promptInputConfig struct {
|
||||
Name string `yaml:"name"`
|
||||
Required bool `yaml:"required"`
|
||||
ContentType string `yaml:"content_type"`
|
||||
Description string `yaml:"description"`
|
||||
}
|
||||
|
||||
type promptMessageConfig struct {
|
||||
Role *string `yaml:"role"`
|
||||
Content *string `yaml:"content"`
|
||||
ContentFile *string `yaml:"content_file"`
|
||||
CacheControl *cacheControlConfig `yaml:"cache_control"`
|
||||
}
|
||||
|
||||
type cacheControlConfig struct {
|
||||
Type *string `yaml:"type"`
|
||||
TTL *string `yaml:"ttl"`
|
||||
}
|
||||
|
||||
type promptOutputConfig struct {
|
||||
Format *string `yaml:"format"`
|
||||
ValidationMode *string `yaml:"validation_mode"`
|
||||
SchemaPath *string `yaml:"schema_path"`
|
||||
RepairAttempts int `yaml:"repair_attempts"`
|
||||
}
|
||||
|
||||
// fieldState distinguishes omission, an explicit value, and an explicit null.
|
||||
// Inherited values are explicit because ConfigKit has already expanded them.
|
||||
type fieldState uint8
|
||||
|
||||
const (
|
||||
fieldAbsent fieldState = iota
|
||||
fieldValue
|
||||
fieldNull
|
||||
)
|
||||
|
||||
type loaded struct {
|
||||
result *configkit.Result[document]
|
||||
backends map[string]map[string]fieldState
|
||||
profiles map[string]map[string]fieldState
|
||||
prompts map[string]map[string]*promptEntry
|
||||
ambiguous map[string]bool
|
||||
registry *backend.Registry
|
||||
indexedProfiles map[string]domain.ExecutionProfile
|
||||
}
|
||||
|
||||
var envName = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`)
|
||||
|
||||
var backendFields = []string{"endpoint", "api_key_env", "concurrency_limit", "queue_capacity", "extra_params"}
|
||||
var profileFields = []string{"backend", "endpoint", "model", "temperature", "top_p", "max_tokens", "timeout_seconds", "service_tier", "reasoning_effort", "api_key_env", "api_key_required", "extra_params"}
|
||||
|
||||
// Load captures one immutable effective configuration for internal consumers.
|
||||
func Load(main configkit.Input, underlays []configkit.Underlay) (*loaded, error) {
|
||||
return load(main, underlays)
|
||||
}
|
||||
|
||||
// load performs one ConfigKit lifecycle over the selected source tiers.
|
||||
func load(main configkit.Input, underlays []configkit.Underlay) (*loaded, error) {
|
||||
state := &loaded{
|
||||
backends: make(map[string]map[string]fieldState),
|
||||
profiles: make(map[string]map[string]fieldState),
|
||||
}
|
||||
result, err := configkit.LoadLayered(main, configkit.LayeredOptions[document]{
|
||||
Underlays: underlays,
|
||||
Definition: configkit.Definition[document]{
|
||||
Secret: secretProviderPath,
|
||||
Resolve: func(config *document, resolution *configkit.Resolution) error {
|
||||
captureFields(config.Backends, "backends", backendFields, resolution, state.backends)
|
||||
captureFields(config.Profiles, "profiles", profileFields, resolution, state.profiles)
|
||||
resolveBackends(config.Backends, resolution)
|
||||
resolveProfiles(config.Profiles, resolution)
|
||||
resolvePrompts(config.Prompts, resolution)
|
||||
return nil
|
||||
},
|
||||
Validate: validateEffective,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("load configuration: %w", err)
|
||||
}
|
||||
state.result = result
|
||||
if err := state.materializePrompts(); err != nil {
|
||||
return nil, fmt.Errorf("load configuration: %w", err)
|
||||
}
|
||||
if err := state.buildExecutionProfiles(); err != nil {
|
||||
return nil, fmt.Errorf("load configuration: %w", err)
|
||||
}
|
||||
return state, nil
|
||||
}
|
||||
|
||||
func captureFields[T any](entries map[string]T, collection string, fields []string, resolution *configkit.Resolution, destination map[string]map[string]fieldState) {
|
||||
for id := range entries {
|
||||
states := make(map[string]fieldState, len(fields))
|
||||
for _, field := range fields {
|
||||
info, exists := resolution.InputInfo(configkit.Keys(collection, id, field))
|
||||
switch {
|
||||
case !exists || !info.Explicit:
|
||||
states[field] = fieldAbsent
|
||||
case info.Null:
|
||||
states[field] = fieldNull
|
||||
default:
|
||||
states[field] = fieldValue
|
||||
}
|
||||
}
|
||||
destination[id] = states
|
||||
}
|
||||
}
|
||||
|
||||
func sortedKeys[T any](entries map[string]T) []string {
|
||||
keys := make([]string, 0, len(entries))
|
||||
for key := range entries {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
return keys
|
||||
}
|
||||
|
||||
func normalizeString(value **string, path configkit.Path, resolution *configkit.Resolution, normalized string) bool {
|
||||
if value != nil && *value != nil && **value != normalized {
|
||||
configkit.Normalize(resolution, path, value, &normalized, "normalize configuration value")
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func resolveBackends(backends map[string]backendConfig, resolution *configkit.Resolution) {
|
||||
for _, id := range sortedKeys(backends) {
|
||||
backend := backends[id]
|
||||
base := configkit.Keys("backends", id)
|
||||
if backend.Endpoint != nil {
|
||||
endpoint, err := domain.NormalizeOpenAICompatibleBaseEndpoint(*backend.Endpoint)
|
||||
if err != nil {
|
||||
resolution.Error(base.Key("endpoint"), err.Error())
|
||||
} else {
|
||||
normalizeString(&backend.Endpoint, base.Key("endpoint"), resolution, endpoint)
|
||||
}
|
||||
}
|
||||
if backend.APIKeyEnv != nil {
|
||||
normalizeString(&backend.APIKeyEnv, base.Key("api_key_env"), resolution, strings.TrimSpace(*backend.APIKeyEnv))
|
||||
}
|
||||
if backend.ConcurrencyLimit != nil && *backend.ConcurrencyLimit > 0 && backend.QueueCapacity == nil {
|
||||
capacity := 1024
|
||||
if resolution.IsExplicit(base.Key("queue_capacity")) {
|
||||
configkit.Normalize(resolution, base.Key("queue_capacity"), &backend.QueueCapacity, &capacity, "select default queue capacity")
|
||||
} else {
|
||||
configkit.Default(resolution, base.Key("queue_capacity"), &backend.QueueCapacity, &capacity, "select default queue capacity")
|
||||
}
|
||||
}
|
||||
backends[id] = backend
|
||||
}
|
||||
}
|
||||
|
||||
func resolveProfiles(profiles map[string]profileConfig, resolution *configkit.Resolution) {
|
||||
for _, id := range sortedKeys(profiles) {
|
||||
profile := profiles[id]
|
||||
base := configkit.Keys("profiles", id)
|
||||
if profile.Endpoint != nil && *profile.Endpoint != "" {
|
||||
endpoint, err := domain.NormalizeOpenAICompatibleBaseEndpoint(*profile.Endpoint)
|
||||
if err != nil {
|
||||
resolution.Error(base.Key("endpoint"), err.Error())
|
||||
} else {
|
||||
normalizeString(&profile.Endpoint, base.Key("endpoint"), resolution, endpoint)
|
||||
}
|
||||
}
|
||||
for _, field := range []struct {
|
||||
name string
|
||||
value **string
|
||||
}{
|
||||
{"service_tier", &profile.ServiceTier},
|
||||
{"reasoning_effort", &profile.ReasoningEffort},
|
||||
{"api_key_env", &profile.APIKeyEnv},
|
||||
} {
|
||||
if *field.value != nil {
|
||||
normalizeString(field.value, base.Key(field.name), resolution, strings.TrimSpace(**field.value))
|
||||
}
|
||||
}
|
||||
profiles[id] = profile
|
||||
}
|
||||
}
|
||||
|
||||
func validateEffective(config *document, validation *configkit.Validation) {
|
||||
for _, id := range sortedKeys(config.Backends) {
|
||||
backend := config.Backends[id]
|
||||
base := configkit.Keys("backends", id)
|
||||
if strings.TrimSpace(id) == "" {
|
||||
validation.Error(base, "backend ID must not be blank")
|
||||
}
|
||||
if backend.Endpoint == nil || *backend.Endpoint == "" {
|
||||
validation.Error(base.Key("endpoint"), "endpoint is required")
|
||||
}
|
||||
if backend.APIKeyEnv != nil && *backend.APIKeyEnv != "" && !envName.MatchString(*backend.APIKeyEnv) {
|
||||
validation.Error(base.Key("api_key_env"), "environment variable name is invalid")
|
||||
}
|
||||
if backend.ConcurrencyLimit != nil && *backend.ConcurrencyLimit < 0 {
|
||||
validation.Error(base.Key("concurrency_limit"), "concurrency limit must not be negative")
|
||||
}
|
||||
if backend.QueueCapacity != nil {
|
||||
if *backend.QueueCapacity < 0 {
|
||||
validation.Error(base.Key("queue_capacity"), "queue capacity must not be negative")
|
||||
}
|
||||
if backend.ConcurrencyLimit == nil || *backend.ConcurrencyLimit <= 0 {
|
||||
validation.Error(base.Key("queue_capacity"), "queue capacity requires a positive concurrency limit")
|
||||
} else if *backend.QueueCapacity > int(^uint(0)>>1)-*backend.ConcurrencyLimit {
|
||||
validation.Error(base.Key("queue_capacity"), "total capacity overflows int")
|
||||
}
|
||||
}
|
||||
validateExtraParams(backend.ExtraParams, base.Key("extra_params"), true, validation)
|
||||
}
|
||||
for _, id := range sortedKeys(config.Profiles) {
|
||||
profile := config.Profiles[id]
|
||||
base := configkit.Keys("profiles", id)
|
||||
if strings.TrimSpace(id) == "" {
|
||||
validation.Error(base, "profile ID must not be blank")
|
||||
}
|
||||
if profile.Backend != nil && *profile.Backend != "" {
|
||||
if _, exists := config.Backends[*profile.Backend]; !exists {
|
||||
validation.Error(base.Key("backend"), "backend is not defined")
|
||||
}
|
||||
}
|
||||
if profile.Model != nil && *profile.Model != "" && strings.TrimSpace(*profile.Model) == "" {
|
||||
validation.Error(base.Key("model"), "model must not be blank")
|
||||
}
|
||||
if profile.Temperature != nil {
|
||||
if err := domain.ValidateExecutionTargetSettings(domain.ExecutionTarget{Temperature: *profile.Temperature}); err != nil {
|
||||
validation.Error(base.Key("temperature"), err.Error())
|
||||
}
|
||||
}
|
||||
if profile.TopP != nil {
|
||||
if err := domain.ValidateExecutionTargetSettings(domain.ExecutionTarget{TopP: *profile.TopP}); err != nil {
|
||||
validation.Error(base.Key("top_p"), err.Error())
|
||||
}
|
||||
}
|
||||
if profile.MaxTokens != nil {
|
||||
if err := domain.ValidateExecutionTargetSettings(domain.ExecutionTarget{MaxTokens: *profile.MaxTokens}); err != nil {
|
||||
validation.Error(base.Key("max_tokens"), err.Error())
|
||||
}
|
||||
}
|
||||
if profile.TimeoutSeconds != nil {
|
||||
if err := domain.ValidateExecutionTargetSettings(domain.ExecutionTarget{TimeoutSeconds: *profile.TimeoutSeconds}); err != nil {
|
||||
validation.Error(base.Key("timeout_seconds"), err.Error())
|
||||
}
|
||||
}
|
||||
validateExtraParams(profile.ExtraParams, base.Key("extra_params"), false, validation)
|
||||
}
|
||||
}
|
||||
|
||||
func validateExtraParams(values map[string]any, path configkit.Path, backend bool, validation *configkit.Validation) {
|
||||
if values == nil {
|
||||
return
|
||||
}
|
||||
if _, err := jsonvalue.CopyMap(values); err != nil {
|
||||
validation.Error(path, "extra parameters must be a JSON-compatible map with nonempty keys")
|
||||
return
|
||||
}
|
||||
if backend {
|
||||
for key := range values {
|
||||
if llm.IsReservedOpenAIChatRequestField(key) {
|
||||
validation.Error(path.Key(key), "extra parameter conflicts with a request field")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
138
internal/configuration/load_test.go
Normal file
138
internal/configuration/load_test.go
Normal file
@@ -0,0 +1,138 @@
|
||||
package configuration
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
||||
)
|
||||
|
||||
func mapping(label string, values map[string]any) configkit.Input {
|
||||
return configkit.Input{Source: configkit.Mapping(label, values)}
|
||||
}
|
||||
|
||||
func TestLoadComposesLiteralKeysAndEffectivePresence(t *testing.T) {
|
||||
base := mapping("defaults", map[string]any{
|
||||
"backends": map[string]any{
|
||||
"local.v1": map[string]any{"endpoint": "http://localhost:8080/v1", "concurrency_limit": 2},
|
||||
},
|
||||
"profiles": map[string]any{
|
||||
"base/one": map[string]any{"backend": "local.v1", "temperature": 0.8, "top_p": 0.9, "api_key_required": true,
|
||||
"extra_params": map[string]any{"provider": map[string]any{"first": 1, "second": 2}}},
|
||||
"child.v1": map[string]any{"extends": "base/one"},
|
||||
},
|
||||
"prompts": map[string]any{"invoice/summary": map[string]any{"v1.0": map[string]any{
|
||||
"messages": []any{map[string]any{"role": "user", "content": "Summarize."}},
|
||||
"output": map[string]any{"format": "text", "validation_mode": "basic"},
|
||||
}}},
|
||||
})
|
||||
overlay := mapping("operator", map[string]any{
|
||||
"backends": map[string]any{"local.v1": map[string]any{"queue_capacity": 0}},
|
||||
"profiles": map[string]any{"child.v1": map[string]any{
|
||||
"temperature": 0, "top_p": nil, "api_key_required": false,
|
||||
"extra_params": map[string]any{"provider": map[string]any{"second": 3}},
|
||||
}},
|
||||
})
|
||||
loaded, err := load(configkit.Input{Source: configkit.NoSource()}, []configkit.Underlay{
|
||||
{Inputs: []configkit.Input{base}}, {Inputs: []configkit.Input{overlay}},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
profile := loaded.result.Config.Profiles["child.v1"]
|
||||
if profile.Temperature == nil || *profile.Temperature != 0 || profile.TopP != nil || profile.APIKeyRequired == nil || *profile.APIKeyRequired {
|
||||
t.Fatalf("effective values were not retained: %+v", profile)
|
||||
}
|
||||
if loaded.profiles["child.v1"]["temperature"] != fieldValue || loaded.profiles["child.v1"]["top_p"] != fieldNull || loaded.profiles["child.v1"]["api_key_required"] != fieldValue {
|
||||
t.Fatalf("effective presence was lost: %+v", loaded.profiles["child.v1"])
|
||||
}
|
||||
provider := profile.ExtraParams["provider"].(map[string]any)
|
||||
if provider["first"] != int64(1) || provider["second"] != int64(3) {
|
||||
t.Fatalf("recursive overlay failed: %+v", provider)
|
||||
}
|
||||
backend := loaded.result.Config.Backends["local.v1"]
|
||||
if backend.QueueCapacity == nil || *backend.QueueCapacity != 0 || loaded.backends["local.v1"]["queue_capacity"] != fieldValue {
|
||||
t.Fatalf("explicit zero queue capacity was lost: %+v", backend)
|
||||
}
|
||||
if loaded.result.Config.Prompts["invoice/summary"]["v1.0"].Messages[0].Content == nil {
|
||||
t.Fatal("literal prompt ID and version were not decoded")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadRecordsBackendNormalizationAndNullDefault(t *testing.T) {
|
||||
loaded, err := load(mapping("backend", map[string]any{"backends": map[string]any{
|
||||
"local": map[string]any{"endpoint": " HTTP://localhost:8080/v1 ", "api_key_env": " TOKEN_NAME ",
|
||||
"concurrency_limit": 2, "queue_capacity": nil},
|
||||
}}), nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
backend := loaded.result.Config.Backends["local"]
|
||||
if backend.Endpoint == nil || *backend.Endpoint != "http://localhost:8080/v1" ||
|
||||
backend.APIKeyEnv == nil || *backend.APIKeyEnv != "TOKEN_NAME" ||
|
||||
backend.QueueCapacity == nil || *backend.QueueCapacity != 1024 {
|
||||
t.Fatalf("backend normalization/default was not applied: %+v", backend)
|
||||
}
|
||||
if loaded.backends["local"]["queue_capacity"] != fieldNull {
|
||||
t.Fatalf("explicit null was not retained: %+v", loaded.backends["local"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadDefaultsAbsentQueueCapacity(t *testing.T) {
|
||||
loaded, err := load(mapping("backend", map[string]any{"backends": map[string]any{
|
||||
"local": map[string]any{"endpoint": "http://localhost:8080/v1", "concurrency_limit": 1},
|
||||
}}), nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
backend := loaded.result.Config.Backends["local"]
|
||||
if backend.QueueCapacity == nil || *backend.QueueCapacity != 1024 || loaded.backends["local"]["queue_capacity"] != fieldAbsent {
|
||||
t.Fatalf("absent queue capacity did not receive its default: %+v", backend)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadRejectsStrictGroupConflict(t *testing.T) {
|
||||
_, err := load(configkit.Input{Source: configkit.NoSource()}, []configkit.Underlay{{Inputs: []configkit.Input{
|
||||
mapping("first", map[string]any{"profiles": map[string]any{"shared": map[string]any{"model": "one"}}}),
|
||||
mapping("second", map[string]any{"profiles": map[string]any{"shared": map[string]any{"model": "one"}}}),
|
||||
}}})
|
||||
var loadError *configkit.LoadError
|
||||
if !errors.As(err, &loadError) {
|
||||
t.Fatalf("expected ConfigKit load error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadValidatesOnlyFinalDomainValues(t *testing.T) {
|
||||
underlay := mapping("defaults", map[string]any{
|
||||
"backends": map[string]any{"local": map[string]any{"endpoint": "not a URL"}},
|
||||
"profiles": map[string]any{"partial": map[string]any{"backend": "local"}},
|
||||
})
|
||||
main := mapping("operator", map[string]any{"backends": map[string]any{"local": map[string]any{"endpoint": "http://localhost:8080/v1"}}})
|
||||
loaded, err := load(main, []configkit.Underlay{{Inputs: []configkit.Input{underlay}}})
|
||||
if err != nil {
|
||||
t.Fatalf("replaced lower-layer domain value should not fail: %v", err)
|
||||
}
|
||||
if loaded.result.Config.Profiles["partial"].Model != nil {
|
||||
t.Fatal("unused partial profile unexpectedly gained a model")
|
||||
}
|
||||
|
||||
_, err = load(configkit.Input{Source: configkit.NoSource()}, []configkit.Underlay{{Inputs: []configkit.Input{
|
||||
mapping("invalid", map[string]any{"backends": map[string]any{"unused": map[string]any{"endpoint": "not a URL"}}}),
|
||||
}}})
|
||||
var loadError *configkit.LoadError
|
||||
if !errors.As(err, &loadError) {
|
||||
t.Fatalf("expected path-aware effective backend failure, got %v", err)
|
||||
}
|
||||
if diagnostics := loadError.Diagnostics(); len(diagnostics) != 1 || diagnostics[0].Path.String() != "/backends/unused/endpoint" {
|
||||
t.Fatalf("expected endpoint diagnostic, got %+v", diagnostics)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadKeepsSyntaxErrorsFromLowerTiers(t *testing.T) {
|
||||
_, err := load(mapping("operator", map[string]any{"backends": map[string]any{"local": map[string]any{"endpoint": "http://localhost:8080"}}}),
|
||||
[]configkit.Underlay{{Inputs: []configkit.Input{{Source: configkit.Bytes("broken", []byte("backends: ["))}}}})
|
||||
var loadError *configkit.LoadError
|
||||
if !errors.As(err, &loadError) {
|
||||
t.Fatalf("expected lower-tier parsing error, got %v", err)
|
||||
}
|
||||
}
|
||||
331
internal/configuration/prompts.go
Normal file
331
internal/configuration/prompts.go
Normal file
@@ -0,0 +1,331 @@
|
||||
package configuration
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
"text/template"
|
||||
|
||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/filecatalog"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/prompt"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/promptdef"
|
||||
)
|
||||
|
||||
var (
|
||||
errPromptNotFound = errors.New("prompt definition not found")
|
||||
errPromptAmbiguous = errors.New("prompt version is ambiguous")
|
||||
)
|
||||
|
||||
type promptEntry struct {
|
||||
definition domain.PromptDefinition
|
||||
inspection domain.PromptInspection
|
||||
hash string
|
||||
session *template.Template
|
||||
messages []*template.Template
|
||||
}
|
||||
|
||||
func resolvePrompts(prompts map[string]map[string]promptConfig, resolution *configkit.Resolution) {
|
||||
for _, id := range sortedKeys(prompts) {
|
||||
for _, version := range sortedKeys(prompts[id]) {
|
||||
definition := prompts[id][version]
|
||||
base := configkit.Keys("prompts", id, version)
|
||||
for _, field := range []struct {
|
||||
name string
|
||||
value **string
|
||||
}{
|
||||
{"description", &definition.Description},
|
||||
{"session_id", &definition.SessionID},
|
||||
} {
|
||||
if *field.value != nil {
|
||||
normalizeString(field.value, base.Key(field.name), resolution, strings.TrimSpace(**field.value))
|
||||
}
|
||||
}
|
||||
inputsChanged := false
|
||||
for index := range definition.Inputs {
|
||||
input := &definition.Inputs[index]
|
||||
inputPath := base.Key("inputs").Index(index)
|
||||
for _, field := range []struct {
|
||||
name string
|
||||
value *string
|
||||
}{
|
||||
{"name", &input.Name},
|
||||
{"content_type", &input.ContentType},
|
||||
{"description", &input.Description},
|
||||
} {
|
||||
normalized := strings.TrimSpace(*field.value)
|
||||
if normalized != *field.value {
|
||||
configkit.Normalize(resolution, inputPath.Key(field.name), field.value, normalized, "normalize input metadata")
|
||||
inputsChanged = true
|
||||
}
|
||||
}
|
||||
}
|
||||
if inputsChanged {
|
||||
resolution.Record(base.Key("inputs"), configkit.ChangeNormalization, "normalize input metadata")
|
||||
}
|
||||
// Normalize message metadata on execution-owned copies below. ConfigKit
|
||||
// records sequence mutations as whole-value replacements, which would
|
||||
// discard the effective source contexts of unchanged content_file fields.
|
||||
if definition.Output != nil && definition.Output.SchemaPath != nil {
|
||||
normalizeString(&definition.Output.SchemaPath, base.Key("output").Key("schema_path"), resolution, strings.TrimSpace(*definition.Output.SchemaPath))
|
||||
}
|
||||
prompts[id][version] = definition
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (state *loaded) materializePrompts() error {
|
||||
state.prompts = make(map[string]map[string]*promptEntry, len(state.result.Config.Prompts))
|
||||
state.ambiguous = make(map[string]bool)
|
||||
var metadata *configkit.Metadata
|
||||
lookupOrigin := func(path configkit.Path) (configkit.FieldInfo, bool) {
|
||||
if metadata == nil {
|
||||
captured := state.result.Metadata()
|
||||
metadata = &captured
|
||||
}
|
||||
return metadata.Lookup(path)
|
||||
}
|
||||
for _, id := range sortedKeys(state.result.Config.Prompts) {
|
||||
if strings.TrimSpace(id) == "" {
|
||||
return fmt.Errorf("%s: prompt ID must not be blank", configkit.Keys("prompts", id))
|
||||
}
|
||||
versions := state.result.Config.Prompts[id]
|
||||
if versions == nil {
|
||||
return fmt.Errorf("%s: prompt collection must be a mapping", configkit.Keys("prompts", id))
|
||||
}
|
||||
state.prompts[id] = make(map[string]*promptEntry, len(versions))
|
||||
state.ambiguous[id] = len(versions) > 1
|
||||
for _, version := range sortedKeys(versions) {
|
||||
if strings.TrimSpace(version) == "" {
|
||||
return fmt.Errorf("%s: prompt version must not be blank", configkit.Keys("prompts", id, version))
|
||||
}
|
||||
entry, err := state.materializePrompt(id, version, versions[version], lookupOrigin)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
state.prompts[id][version] = entry
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (state *loaded) materializePrompt(id, version string, raw promptConfig, lookupOrigin func(configkit.Path) (configkit.FieldInfo, bool)) (*promptEntry, error) {
|
||||
base := configkit.Keys("prompts", id, version)
|
||||
if len(raw.Messages) == 0 {
|
||||
return nil, fmt.Errorf("%s: at least one message is required", base.Key("messages"))
|
||||
}
|
||||
definition := domain.PromptDefinition{ID: id, Version: version}
|
||||
if raw.DefaultProfile != nil {
|
||||
if strings.TrimSpace(*raw.DefaultProfile) == "" {
|
||||
return nil, fmt.Errorf("%s: default profile must not be blank", base.Key("default_profile"))
|
||||
}
|
||||
definition.DefaultProfile = *raw.DefaultProfile
|
||||
}
|
||||
if raw.Description != nil {
|
||||
definition.Description = *raw.Description
|
||||
}
|
||||
if raw.SessionID != nil {
|
||||
definition.SessionID = *raw.SessionID
|
||||
}
|
||||
seenInputs := make(map[string]bool, len(raw.Inputs))
|
||||
definition.Inputs = make([]domain.PromptInput, 0, len(raw.Inputs))
|
||||
for index, input := range raw.Inputs {
|
||||
path := base.Key("inputs").Index(index)
|
||||
if input.Name == "" {
|
||||
return nil, fmt.Errorf("%s: input name is required", path.Key("name"))
|
||||
}
|
||||
if seenInputs[input.Name] {
|
||||
return nil, fmt.Errorf("%s: input name is duplicated", path.Key("name"))
|
||||
}
|
||||
seenInputs[input.Name] = true
|
||||
definition.Inputs = append(definition.Inputs, domain.PromptInput{
|
||||
Name: input.Name, Required: input.Required, ContentType: input.ContentType, Description: input.Description,
|
||||
})
|
||||
}
|
||||
entry := &promptEntry{messages: make([]*template.Template, 0, len(raw.Messages))}
|
||||
definition.Templates = make([]domain.PromptMessageTemplate, 0, len(raw.Messages))
|
||||
for index, message := range raw.Messages {
|
||||
path := base.Key("messages").Index(index)
|
||||
if message.Role == nil {
|
||||
return nil, fmt.Errorf("%s: role is required", path.Key("role"))
|
||||
}
|
||||
role, err := domain.NormalizeMessageRole(*message.Role)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s: %w", path.Key("role"), err)
|
||||
}
|
||||
if (message.Content != nil) == (message.ContentFile != nil) {
|
||||
return nil, fmt.Errorf("%s: exactly one of content or content_file is required", path)
|
||||
}
|
||||
content := ""
|
||||
if message.Content != nil {
|
||||
content = *message.Content
|
||||
} else {
|
||||
var err error
|
||||
content, err = state.readContent(path.Key("content_file"), *message.ContentFile, lookupOrigin)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
var control *domain.CacheControl
|
||||
if message.CacheControl != nil {
|
||||
if message.CacheControl.Type == nil {
|
||||
return nil, fmt.Errorf("%s: type is required", path.Key("cache_control").Key("type"))
|
||||
}
|
||||
control = &domain.CacheControl{Type: domain.CacheControlType(*message.CacheControl.Type)}
|
||||
if message.CacheControl.TTL != nil {
|
||||
control.TTL = *message.CacheControl.TTL
|
||||
}
|
||||
var err error
|
||||
control, err = domain.NormalizeCacheControl(control)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s: %w", path.Key("cache_control"), err)
|
||||
}
|
||||
}
|
||||
compiled, err := prompt.CompileTemplate(fmt.Sprintf("msg_%d", index), content)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s: invalid template syntax: %w", path, err)
|
||||
}
|
||||
entry.messages = append(entry.messages, compiled)
|
||||
definition.Templates = append(definition.Templates, domain.PromptMessageTemplate{
|
||||
Role: role, Content: content, CacheControl: control,
|
||||
})
|
||||
}
|
||||
if raw.Output == nil {
|
||||
return nil, fmt.Errorf("%s: output is required", base.Key("output"))
|
||||
}
|
||||
if raw.Output.Format == nil || raw.Output.ValidationMode == nil {
|
||||
return nil, fmt.Errorf("%s: format and validation_mode are required", base.Key("output"))
|
||||
}
|
||||
contract := domain.OutputContract{
|
||||
Format: domain.OutputFormat(*raw.Output.Format),
|
||||
ValidationMode: domain.ValidationMode(*raw.Output.ValidationMode),
|
||||
RepairAttempts: raw.Output.RepairAttempts,
|
||||
}
|
||||
if raw.Output.SchemaPath != nil {
|
||||
contract.SchemaPath = *raw.Output.SchemaPath
|
||||
}
|
||||
if err := domain.ValidateOutputContract(contract); err != nil {
|
||||
return nil, fmt.Errorf("%s: invalid output contract: %w", base.Key("output"), err)
|
||||
}
|
||||
definition.OutputFormat = contract.Format
|
||||
definition.Validation = contract
|
||||
if raw.SessionID != nil {
|
||||
var err error
|
||||
entry.session, err = prompt.CompileTemplate("session_id", definition.SessionID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s: invalid template syntax: %w", base.Key("session_id"), err)
|
||||
}
|
||||
}
|
||||
hash, err := promptdef.HashDefinition(&definition)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s: hash prompt definition: %w", base, err)
|
||||
}
|
||||
entry.definition = definition
|
||||
entry.hash = hash
|
||||
entry.inspection = domain.PromptInspection{
|
||||
PromptID: id, PromptVersion: version, PromptHash: hash,
|
||||
DefaultProfileID: definition.DefaultProfile, Inputs: append([]domain.PromptInput(nil), definition.Inputs...),
|
||||
OutputContract: contract,
|
||||
}
|
||||
return entry, nil
|
||||
}
|
||||
|
||||
func (state *loaded) readContent(fieldPath configkit.Path, name string, lookupOrigin func(configkit.Path) (configkit.FieldInfo, bool)) (string, error) {
|
||||
if strings.TrimSpace(name) == "" {
|
||||
return "", fmt.Errorf("%s: content_file path must not be blank", fieldPath)
|
||||
}
|
||||
field, found := lookupOrigin(fieldPath)
|
||||
if !found || len(field.Origin.Locations) != 1 {
|
||||
return "", fmt.Errorf("%s: content_file needs one contributing source with a filesystem context", fieldPath)
|
||||
}
|
||||
sourceID := field.Origin.Locations[0].SourceID
|
||||
context, found := state.result.SourceContext(sourceID)
|
||||
if !found || context.FS == nil {
|
||||
return "", fmt.Errorf("%s: content_file source %s has no filesystem context", fieldPath, sourceID)
|
||||
}
|
||||
resolved, _, err := filecatalog.ResolveFSPath(".", context.BaseDir, name)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("%s: content_file path is outside its source root or invalid: %w", fieldPath, err)
|
||||
}
|
||||
file, err := context.FS.Open(resolved)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("%s: open content_file from source %s: %w", fieldPath, sourceID, err)
|
||||
}
|
||||
content, readErr := io.ReadAll(file)
|
||||
closeErr := file.Close()
|
||||
if readErr != nil {
|
||||
return "", fmt.Errorf("%s: read content_file from source %s: %w", fieldPath, sourceID, readErr)
|
||||
}
|
||||
if closeErr != nil {
|
||||
return "", fmt.Errorf("%s: close content_file from source %s: %w", fieldPath, sourceID, closeErr)
|
||||
}
|
||||
return string(content), nil
|
||||
}
|
||||
|
||||
func (state *loaded) prompt(id, version string) (*domain.PromptDefinition, domain.PromptInspection, error) {
|
||||
if state == nil || strings.TrimSpace(id) == "" || (version != "" && strings.TrimSpace(version) == "") {
|
||||
return nil, domain.PromptInspection{}, errPromptNotFound
|
||||
}
|
||||
versions := state.prompts[id]
|
||||
if version == "" {
|
||||
if state.ambiguous[id] {
|
||||
return nil, domain.PromptInspection{}, errPromptAmbiguous
|
||||
}
|
||||
for selected := range versions {
|
||||
version = selected
|
||||
}
|
||||
}
|
||||
entry := versions[version]
|
||||
if entry == nil {
|
||||
return nil, domain.PromptInspection{}, errPromptNotFound
|
||||
}
|
||||
definition := entry.definition
|
||||
definition.Inputs = append([]domain.PromptInput(nil), definition.Inputs...)
|
||||
definition.Templates = append([]domain.PromptMessageTemplate(nil), definition.Templates...)
|
||||
for index := range definition.Templates {
|
||||
if control := definition.Templates[index].CacheControl; control != nil {
|
||||
copy := *control
|
||||
definition.Templates[index].CacheControl = ©
|
||||
}
|
||||
}
|
||||
inspection := entry.inspection
|
||||
inspection.Inputs = append([]domain.PromptInput(nil), inspection.Inputs...)
|
||||
return &definition, inspection, nil
|
||||
}
|
||||
|
||||
// GetPromptSnapshot returns an independently owned captured definition and its
|
||||
// construction-time hash. Lookup retains literal ID and version keys.
|
||||
func (state *loaded) GetPromptSnapshot(ctx context.Context, id, version string) (*domain.PromptDefinition, string, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
definition, inspection, err := state.prompt(id, version)
|
||||
if errors.Is(err, errPromptNotFound) {
|
||||
return nil, "", promptdef.ErrPromptDefinitionNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
return definition, inspection.PromptHash, nil
|
||||
}
|
||||
|
||||
// GetPromptDefinition implements the runner's prompt lookup boundary.
|
||||
func (state *loaded) GetPromptDefinition(ctx context.Context, id, version string) (*domain.PromptDefinition, error) {
|
||||
definition, _, err := state.GetPromptSnapshot(ctx, id, version)
|
||||
return definition, err
|
||||
}
|
||||
|
||||
// GetCompiledTemplates returns construction-time parses for one exact prompt.
|
||||
func (state *loaded) GetCompiledTemplates(id, version string) (prompt.CompiledTemplates, bool) {
|
||||
if state == nil {
|
||||
return prompt.CompiledTemplates{}, false
|
||||
}
|
||||
entry := state.prompts[id][version]
|
||||
if entry == nil {
|
||||
return prompt.CompiledTemplates{}, false
|
||||
}
|
||||
return prompt.CompiledTemplates{Session: entry.session, Messages: entry.messages}, true
|
||||
}
|
||||
206
internal/configuration/prompts_test.go
Normal file
206
internal/configuration/prompts_test.go
Normal file
@@ -0,0 +1,206 @@
|
||||
package configuration
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"testing/fstest"
|
||||
|
||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
||||
)
|
||||
|
||||
func TestLoadCapturesInheritedImportedContent(t *testing.T) {
|
||||
files := fstest.MapFS{
|
||||
"app/main.yaml": {Data: []byte("configkit:\n imports:\n - path: ../catalog/base.yaml\nprompts:\n child:\n extends: base\n")},
|
||||
"catalog/base.yaml": {Data: []byte("prompts:\n base:\n v1:\n messages:\n - role: ' USER '\n cache_control:\n type: ' ephemeral '\n ttl: ' 1h '\n content_file: ./body.md\n output:\n format: text\n validation_mode: basic\n")},
|
||||
"catalog/body.md": {Data: []byte("Captured {{.name}}")},
|
||||
}
|
||||
namespace := configkit.NewFilesystem("embedded", files)
|
||||
state, err := load(configkit.Input{
|
||||
Source: namespace.File("app/main.yaml"), Imports: true,
|
||||
Filesystem: &configkit.FilesystemContext{FS: &namespace, Root: "."},
|
||||
}, nil)
|
||||
if err != nil {
|
||||
var loadError *configkit.LoadError
|
||||
if errors.As(err, &loadError) {
|
||||
t.Fatalf("%v: %+v", err, loadError.Diagnostics())
|
||||
}
|
||||
t.Fatal(err)
|
||||
}
|
||||
definition, inspection, err := state.prompt("child", "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if definition.Templates[0].Content != "Captured {{.name}}" || inspection.PromptHash == "" {
|
||||
t.Fatalf("inherited file content was not captured: %+v, %+v", definition, inspection)
|
||||
}
|
||||
message := definition.Templates[0]
|
||||
if message.Role != "user" || message.CacheControl == nil || message.CacheControl.Type != "ephemeral" || message.CacheControl.TTL != "1h" {
|
||||
t.Fatalf("message metadata was not normalized: %+v", message)
|
||||
}
|
||||
field, ok := state.result.Metadata().Lookup(configkit.Keys("prompts", "child", "v1", "messages").Index(0).Key("content_file"))
|
||||
if !ok || len(field.Origin.Locations) != 1 {
|
||||
t.Fatalf("content origin was lost during normalization: %+v", field)
|
||||
}
|
||||
var originPath string
|
||||
for _, source := range state.result.Sources() {
|
||||
if source.ID == field.Origin.Locations[0].SourceID {
|
||||
originPath = source.Path
|
||||
}
|
||||
}
|
||||
if originPath != "catalog/base.yaml" {
|
||||
t.Fatalf("content origin = %q, want catalog/base.yaml", originPath)
|
||||
}
|
||||
files["catalog/body.md"].Data = []byte("Changed")
|
||||
again, againInspection, err := state.prompt("child", "v1")
|
||||
if err != nil || again.Templates[0].Content != "Captured {{.name}}" || againInspection.PromptHash != inspection.PromptHash {
|
||||
t.Fatalf("captured definition changed after source mutation: %+v, %+v, %v", again, againInspection, err)
|
||||
}
|
||||
definition.Templates[0].Content = "caller mutation"
|
||||
third, _, err := state.prompt("child", "v1")
|
||||
if err != nil || third.Templates[0].Content != "Captured {{.name}}" {
|
||||
t.Fatalf("caller mutation escaped into index: %+v, %v", third, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadUsesExplicitMappingContextAndClosesContent(t *testing.T) {
|
||||
filesystem := &countingFS{FS: fstest.MapFS{
|
||||
"assets/body.md": {Data: []byte("From mapping")},
|
||||
}}
|
||||
namespace := configkit.NewFilesystem("mapping-context", filesystem)
|
||||
source := configkit.Mapping("application", promptWithFile("body.md"))
|
||||
state, err := load(configkit.Input{
|
||||
Source: source,
|
||||
Filesystem: &configkit.FilesystemContext{FS: &namespace, Root: ".", BaseDir: "assets"},
|
||||
}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
definition, _, err := state.prompt("example", "v1")
|
||||
if err != nil || definition.Templates[0].Content != "From mapping" || filesystem.contentOpened != 1 || filesystem.contentClosed != 1 {
|
||||
t.Fatalf("mapping context or handle ownership failed: %+v, opens=%d closes=%d, %v", definition, filesystem.contentOpened, filesystem.contentClosed, err)
|
||||
}
|
||||
_, err = load(mapping("contextless", promptWithFile("body.md")), nil)
|
||||
if err == nil || !strings.Contains(err.Error(), "/prompts/example/v1/messages/0/content_file") {
|
||||
t.Fatalf("expected semantic path for contextless resource, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadCapturesNativeContentFile(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
definitionPath := filepath.Join(root, "definition.yaml")
|
||||
contentPath := filepath.Join(root, "body.md")
|
||||
if err := os.WriteFile(definitionPath, []byte("prompts:\n example:\n v1:\n messages:\n - role: user\n content_file: body.md\n output:\n format: text\n validation_mode: basic\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(contentPath, []byte("Native body"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
state, err := load(configkit.Input{Source: configkit.File(definitionPath), Filesystem: &configkit.FilesystemContext{Root: root}}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(contentPath, []byte("Modified"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
definition, _, err := state.prompt("example", "v1")
|
||||
if err != nil || definition.Templates[0].Content != "Native body" {
|
||||
t.Fatalf("native content was not captured: %+v, %v", definition, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadRejectsMalformedStaticSyntaxAndAmbiguousSelection(t *testing.T) {
|
||||
_, err := load(mapping("bad-template", map[string]any{"prompts": map[string]any{"example": map[string]any{
|
||||
"v1": map[string]any{"messages": []any{map[string]any{"role": "user", "content": "{{if}}"}},
|
||||
"output": map[string]any{"format": "text", "validation_mode": "basic"}},
|
||||
}}}), nil)
|
||||
if err == nil || !strings.Contains(err.Error(), "invalid template syntax") {
|
||||
t.Fatalf("expected construction-time syntax error, got %v", err)
|
||||
}
|
||||
state, err := load(mapping("versions", map[string]any{"prompts": map[string]any{"example": map[string]any{
|
||||
"v1": map[string]any{"messages": []any{map[string]any{"role": "user", "content": "one"}},
|
||||
"output": map[string]any{"format": "text", "validation_mode": "basic"}},
|
||||
"v2": map[string]any{"messages": []any{map[string]any{"role": "user", "content": "two"}},
|
||||
"output": map[string]any{"format": "text", "validation_mode": "basic"}},
|
||||
}}}), nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, _, err := state.prompt("example", ""); !errors.Is(err, errPromptAmbiguous) {
|
||||
t.Fatalf("expected ambiguous version selection, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadAcceptsEmptyInlineContentAndRejectsInvalidPromptFields(t *testing.T) {
|
||||
state, err := load(mapping("empty-content", map[string]any{"prompts": map[string]any{" exact.id ": map[string]any{"v1": map[string]any{
|
||||
"inputs": []any{map[string]any{"name": " note ", "required": true}},
|
||||
"messages": []any{map[string]any{"role": " USER ", "content": "", "content_file": nil,
|
||||
"cache_control": map[string]any{"type": " ephemeral ", "ttl": "1h "}}},
|
||||
"output": map[string]any{"format": "text", "validation_mode": "basic"},
|
||||
}}}}), nil)
|
||||
if err != nil {
|
||||
var loadError *configkit.LoadError
|
||||
if errors.As(err, &loadError) {
|
||||
t.Fatalf("%v: %+v", err, loadError.Diagnostics())
|
||||
}
|
||||
t.Fatal(err)
|
||||
}
|
||||
definition, _, err := state.prompt(" exact.id ", "v1")
|
||||
if err != nil || definition.ID != " exact.id " || definition.Templates[0].Content != "" || definition.Templates[0].Role != "user" ||
|
||||
definition.Inputs[0].Name != "note" || definition.Templates[0].CacheControl.TTL != "1h" {
|
||||
t.Fatalf("explicit empty content or literal ID was changed: %+v, %v", definition, err)
|
||||
}
|
||||
if _, _, err := state.prompt("exact.id", "v1"); !errors.Is(err, errPromptNotFound) {
|
||||
t.Fatalf("trimmed prompt identity unexpectedly matched: %v", err)
|
||||
}
|
||||
_, err = load(mapping("obsolete-field", map[string]any{"prompts": map[string]any{"example": map[string]any{"v1": map[string]any{
|
||||
"id": "example", "messages": []any{map[string]any{"role": "user", "content": "ok"}},
|
||||
"output": map[string]any{"format": "text", "validation_mode": "basic"},
|
||||
}}}}), nil)
|
||||
var loadError *configkit.LoadError
|
||||
if !errors.As(err, &loadError) {
|
||||
t.Fatalf("obsolete identity field should fail strict decoding: %v", err)
|
||||
}
|
||||
_, err = load(mapping("null-prompt", map[string]any{"prompts": map[string]any{"example": nil}}), nil)
|
||||
if err == nil {
|
||||
t.Fatal("null prompt collection was accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func promptWithFile(name string) map[string]any {
|
||||
return map[string]any{"prompts": map[string]any{"example": map[string]any{"v1": map[string]any{
|
||||
"messages": []any{map[string]any{"role": "user", "content_file": name}},
|
||||
"output": map[string]any{"format": "text", "validation_mode": "basic"},
|
||||
}}}}
|
||||
}
|
||||
|
||||
type countingFS struct {
|
||||
fs.FS
|
||||
contentOpened int
|
||||
contentClosed int
|
||||
}
|
||||
|
||||
func (filesystem *countingFS) Open(name string) (fs.File, error) {
|
||||
file, err := filesystem.FS.Open(name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if name == "assets/body.md" {
|
||||
filesystem.contentOpened++
|
||||
return &countingFile{File: file, owner: filesystem}, nil
|
||||
}
|
||||
return file, nil
|
||||
}
|
||||
|
||||
type countingFile struct {
|
||||
fs.File
|
||||
owner *countingFS
|
||||
}
|
||||
|
||||
func (file *countingFile) Close() error {
|
||||
file.owner.contentClosed++
|
||||
return file.File.Close()
|
||||
}
|
||||
187
internal/configuration/targets.go
Normal file
187
internal/configuration/targets.go
Normal file
@@ -0,0 +1,187 @@
|
||||
package configuration
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/backend"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/defaults"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/jsonvalue"
|
||||
profilepkg "gitea.maximumdirect.net/eric/promptkit/internal/profile"
|
||||
)
|
||||
|
||||
func (state *loaded) buildExecutionProfiles() error {
|
||||
definitions := make(map[string]domain.Backend, len(state.result.Config.Backends))
|
||||
for _, id := range sortedKeys(state.result.Config.Backends) {
|
||||
config := state.result.Config.Backends[id]
|
||||
params, err := jsonvalue.CopyMap(config.ExtraParams)
|
||||
if err != nil {
|
||||
return fmt.Errorf("backend %q extra parameters: %w", id, err)
|
||||
}
|
||||
definition := domain.Backend{ID: id, ExtraParams: params}
|
||||
if config.Endpoint != nil {
|
||||
definition.Endpoint = *config.Endpoint
|
||||
}
|
||||
if config.APIKeyEnv != nil {
|
||||
definition.APIKeyEnv = *config.APIKeyEnv
|
||||
}
|
||||
if config.ConcurrencyLimit != nil {
|
||||
definition.ConcurrencyLimit = *config.ConcurrencyLimit
|
||||
}
|
||||
if config.QueueCapacity != nil {
|
||||
definition.QueueCapacity = *config.QueueCapacity
|
||||
definition.QueueCapacitySet = definition.ConcurrencyLimit > 0
|
||||
}
|
||||
definitions[id] = definition
|
||||
}
|
||||
registry, err := backend.NewEffectiveRegistry(definitions)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
state.registry = registry
|
||||
state.indexedProfiles = make(map[string]domain.ExecutionProfile, len(state.result.Config.Profiles))
|
||||
for _, id := range sortedKeys(state.result.Config.Profiles) {
|
||||
config := state.result.Config.Profiles[id]
|
||||
fields := state.profiles[id]
|
||||
target := defaults.ExecutionTargetDefault()
|
||||
if config.Backend != nil && *config.Backend != "" {
|
||||
selected, err := registry.GetBackend(*config.Backend)
|
||||
if err != nil {
|
||||
return fmt.Errorf("profile %q backend: %w", id, err)
|
||||
}
|
||||
target.BackendID = selected.ID
|
||||
target.Endpoint = selected.Endpoint
|
||||
target.APIKeyEnv = selected.APIKeyEnv
|
||||
target.ExtraParams = selected.ExtraParams
|
||||
}
|
||||
presence := domain.ExecutionTargetPresence{}
|
||||
if fields["endpoint"] != fieldAbsent {
|
||||
target.Endpoint = valueOrZero(config.Endpoint)
|
||||
}
|
||||
if fields["model"] != fieldAbsent {
|
||||
target.Model = valueOrZero(config.Model)
|
||||
}
|
||||
if fields["temperature"] != fieldAbsent {
|
||||
target.Temperature = valueOrZero(config.Temperature)
|
||||
presence.Temperature = fields["temperature"] == fieldValue
|
||||
}
|
||||
if fields["top_p"] != fieldAbsent {
|
||||
target.TopP = valueOrZero(config.TopP)
|
||||
presence.TopP = fields["top_p"] == fieldValue
|
||||
}
|
||||
if fields["max_tokens"] != fieldAbsent {
|
||||
target.MaxTokens = valueOrZero(config.MaxTokens)
|
||||
presence.MaxTokens = fields["max_tokens"] == fieldValue
|
||||
}
|
||||
if fields["timeout_seconds"] == fieldValue {
|
||||
target.TimeoutSeconds = valueOrZero(config.TimeoutSeconds)
|
||||
presence.TimeoutSeconds = true
|
||||
}
|
||||
if fields["service_tier"] != fieldAbsent {
|
||||
target.ServiceTier = valueOrZero(config.ServiceTier)
|
||||
}
|
||||
if fields["reasoning_effort"] != fieldAbsent {
|
||||
target.ReasoningEffort = valueOrZero(config.ReasoningEffort)
|
||||
}
|
||||
if fields["api_key_env"] != fieldAbsent {
|
||||
target.APIKeyEnv = valueOrZero(config.APIKeyEnv)
|
||||
}
|
||||
if fields["api_key_required"] != fieldAbsent {
|
||||
target.APIKeyRequired = valueOrZero(config.APIKeyRequired)
|
||||
}
|
||||
if target.APIKeyRequired && fields["api_key_env"] == fieldAbsent {
|
||||
target.APIKeyEnv = ""
|
||||
}
|
||||
switch fields["extra_params"] {
|
||||
case fieldNull:
|
||||
target.ExtraParams = nil
|
||||
case fieldValue:
|
||||
merged, err := overlayProviderMap(target.ExtraParams, config.ExtraParams)
|
||||
if err != nil {
|
||||
return fmt.Errorf("profile %q extra parameters: %w", id, err)
|
||||
}
|
||||
target.ExtraParams = merged
|
||||
}
|
||||
profile := domain.ExecutionProfile{ID: id, ResolvedTarget: &target, TargetPresence: presence}
|
||||
state.indexedProfiles[id] = profile
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func valueOrZero[T any](value *T) T {
|
||||
if value == nil {
|
||||
var zero T
|
||||
return zero
|
||||
}
|
||||
return *value
|
||||
}
|
||||
|
||||
// overlayProviderMap applies only the backend-to-profile JSON-object default
|
||||
// merge. ConfigKit has already resolved source layering and inheritance.
|
||||
func overlayProviderMap(lower, upper map[string]any) (map[string]any, error) {
|
||||
merged, err := jsonvalue.CopyMap(lower)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if merged == nil {
|
||||
merged = make(map[string]any)
|
||||
}
|
||||
for key, value := range upper {
|
||||
if child, ok := value.(map[string]any); ok {
|
||||
if existing, ok := merged[key].(map[string]any); ok {
|
||||
combined, err := overlayProviderMap(existing, child)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
merged[key] = combined
|
||||
continue
|
||||
}
|
||||
}
|
||||
copied, err := jsonvalue.Copy(value)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
merged[key] = copied
|
||||
}
|
||||
return merged, nil
|
||||
}
|
||||
|
||||
// GetProfile returns an independently owned indexed profile for the internal
|
||||
// runner's lookup boundary.
|
||||
func (state *loaded) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if state == nil {
|
||||
return nil, fmt.Errorf("%w: %q", profilepkg.ErrProfileNotFound, id)
|
||||
}
|
||||
profile, found := state.indexedProfiles[id]
|
||||
if !found {
|
||||
return nil, fmt.Errorf("%w: %q", profilepkg.ErrProfileNotFound, id)
|
||||
}
|
||||
target := *profile.ResolvedTarget
|
||||
params, err := jsonvalue.CopyMap(target.ExtraParams)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
target.ExtraParams = params
|
||||
profile.ResolvedTarget = &target
|
||||
return &profile, nil
|
||||
}
|
||||
|
||||
// GetBackend returns an independently owned effective backend definition.
|
||||
func (state *loaded) GetBackend(id string) (domain.Backend, error) {
|
||||
if state == nil {
|
||||
return domain.Backend{}, fmt.Errorf("backend registry is not configured")
|
||||
}
|
||||
return state.registry.GetBackend(id)
|
||||
}
|
||||
|
||||
// CapacityPolicies returns the selected backend admission policies.
|
||||
func (state *loaded) CapacityPolicies() map[string]domain.BackendCapacityPolicy {
|
||||
if state == nil {
|
||||
return nil
|
||||
}
|
||||
return state.registry.CapacityPolicies()
|
||||
}
|
||||
152
internal/configuration/targets_test.go
Normal file
152
internal/configuration/targets_test.go
Normal file
@@ -0,0 +1,152 @@
|
||||
package configuration
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/defaults"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/llm"
|
||||
)
|
||||
|
||||
func TestLoadBuildsEffectiveTargetsAfterInheritance(t *testing.T) {
|
||||
state, err := load(mapping("effective-targets", map[string]any{
|
||||
"backends": map[string]any{
|
||||
"first": map[string]any{"endpoint": "https://first.example/v1", "api_key_env": "FIRST_KEY",
|
||||
"extra_params": map[string]any{"source": "first", "nested": map[string]any{"old": true}}},
|
||||
"second": map[string]any{"endpoint": "https://second.example/v1", "api_key_env": "SECOND_KEY",
|
||||
"concurrency_limit": 2, "queue_capacity": 0,
|
||||
"extra_params": map[string]any{"source": "second", "nested": map[string]any{"backend": true}}},
|
||||
},
|
||||
"profiles": map[string]any{
|
||||
"base": map[string]any{"backend": "first", "model": "model", "temperature": 0.8,
|
||||
"top_p": 0.9, "max_tokens": 100, "timeout_seconds": 30, "api_key_required": true,
|
||||
"extra_params": map[string]any{"nested": map[string]any{"profile": true}}},
|
||||
"child": map[string]any{"extends": "base", "backend": "second", "temperature": 0.0,
|
||||
"top_p": nil, "max_tokens": 0, "timeout_seconds": nil, "api_key_env": nil,
|
||||
"api_key_required": false,
|
||||
"extra_params": map[string]any{"nested": map[string]any{"child": nil}}},
|
||||
"direct-key": map[string]any{"backend": "second", "api_key_required": true},
|
||||
"named-key": map[string]any{"backend": "second", "api_key_required": true, "api_key_env": "CUSTOM_KEY"},
|
||||
"no-provider-map": map[string]any{"backend": "second", "extra_params": nil},
|
||||
"no-timeout": map[string]any{"backend": "second", "timeout_seconds": 0},
|
||||
"partial": map[string]any{},
|
||||
"endpoint-only": map[string]any{"endpoint": "https://standalone.example/v1"},
|
||||
"cleared-endpoint": map[string]any{"extends": "base", "endpoint": nil},
|
||||
},
|
||||
}), nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
selected, err := state.registry.GetBackend("second")
|
||||
if err != nil || selected.QueueCapacity != 0 || !selected.QueueCapacitySet {
|
||||
t.Fatalf("backend capacity was not preserved: %+v, %v", selected, err)
|
||||
}
|
||||
child, err := state.GetProfile(context.Background(), "child")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := state.GetProfile(context.Background(), " child "); err == nil {
|
||||
t.Fatal("profile lookup silently changed a literal ID")
|
||||
}
|
||||
target := child.ResolvedTarget
|
||||
if target.BackendID != "second" || target.Endpoint != "https://second.example/v1" || target.Model != "model" ||
|
||||
target.Temperature != 0 || target.TopP != 0 || target.MaxTokens != 0 ||
|
||||
target.TimeoutSeconds != defaults.ExecutionDefaultTimeoutSeconds || target.APIKeyEnv != "" || target.APIKeyRequired {
|
||||
t.Fatalf("effective child target is wrong: %+v", target)
|
||||
}
|
||||
if !child.TargetPresence.Temperature || !child.TargetPresence.MaxTokens || child.TargetPresence.TopP || child.TargetPresence.TimeoutSeconds {
|
||||
t.Fatalf("numeric presence was not retained: %+v", child.TargetPresence)
|
||||
}
|
||||
wantParams := map[string]any{"source": "second", "nested": map[string]any{
|
||||
"backend": true, "profile": true, "child": nil,
|
||||
}}
|
||||
if !reflect.DeepEqual(target.ExtraParams, wantParams) {
|
||||
t.Fatalf("backend/profile provider map overlay failed: %+v", target.ExtraParams)
|
||||
}
|
||||
if _, exists := target.ExtraParams["nested"].(map[string]any)["old"]; exists {
|
||||
t.Fatal("previous backend derived parameter leaked into child")
|
||||
}
|
||||
for id, wantEnv := range map[string]string{"direct-key": "", "named-key": "CUSTOM_KEY"} {
|
||||
profile, err := state.GetProfile(context.Background(), id)
|
||||
if err != nil || profile.ResolvedTarget.APIKeyEnv != wantEnv {
|
||||
t.Fatalf("credential source for %s: %+v, %v", id, profile, err)
|
||||
}
|
||||
}
|
||||
noMap, _ := state.GetProfile(context.Background(), "no-provider-map")
|
||||
if noMap.ResolvedTarget.ExtraParams != nil {
|
||||
t.Fatalf("explicit null provider map did not suppress backend defaults: %+v", noMap.ResolvedTarget.ExtraParams)
|
||||
}
|
||||
noTimeout, _ := state.GetProfile(context.Background(), "no-timeout")
|
||||
if noTimeout.ResolvedTarget.TimeoutSeconds != 0 || !noTimeout.TargetPresence.TimeoutSeconds {
|
||||
t.Fatalf("explicit zero timeout was lost: %+v", noTimeout)
|
||||
}
|
||||
partial, _ := state.GetProfile(context.Background(), "partial")
|
||||
if partial.ResolvedTarget.Endpoint != "" || partial.ResolvedTarget.Model != "" {
|
||||
t.Fatalf("partial profile gained a runnable target: %+v", partial.ResolvedTarget)
|
||||
}
|
||||
endpointOnly, _ := state.GetProfile(context.Background(), "endpoint-only")
|
||||
if endpointOnly.ResolvedTarget.Endpoint != "https://standalone.example/v1" ||
|
||||
endpointOnly.ResolvedTarget.BackendID != "" || endpointOnly.ResolvedTarget.Model != "" {
|
||||
t.Fatalf("endpoint-only profile was not retained: %+v", endpointOnly.ResolvedTarget)
|
||||
}
|
||||
clearedEndpoint, _ := state.GetProfile(context.Background(), "cleared-endpoint")
|
||||
if clearedEndpoint.ResolvedTarget.Endpoint != "" || clearedEndpoint.ResolvedTarget.BackendID != "first" {
|
||||
t.Fatalf("explicit null recovered backend endpoint: %+v", clearedEndpoint.ResolvedTarget)
|
||||
}
|
||||
child.ResolvedTarget.ExtraParams["source"] = "mutated"
|
||||
again, _ := state.GetProfile(context.Background(), "child")
|
||||
if again.ResolvedTarget.ExtraParams["source"] != "second" {
|
||||
t.Fatal("profile lookup exposed snapshot ownership")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadedZeroControlsReachBuiltInRequest(t *testing.T) {
|
||||
payloads := make(chan map[string]any, 1)
|
||||
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
||||
var payload map[string]any
|
||||
if err := json.NewDecoder(request.Body).Decode(&payload); err != nil {
|
||||
writer.WriteHeader(http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
payloads <- payload
|
||||
writer.Header().Set("Content-Type", "application/json")
|
||||
_, _ = writer.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
state, err := load(mapping("wire-controls", map[string]any{
|
||||
"backends": map[string]any{"local": map[string]any{"endpoint": server.URL + "/v1"}},
|
||||
"profiles": map[string]any{"direct": map[string]any{
|
||||
"backend": "local", "model": "example", "temperature": 0.0, "max_tokens": 0, "top_p": nil,
|
||||
}},
|
||||
}), nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
profile, err := state.GetProfile(context.Background(), "direct")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
client, err := llm.NewOpenAICompatibleClient(llm.OpenAICompatibleConfig{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = client.Generate(context.Background(), domain.GenerateRequest{
|
||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hello"}}},
|
||||
Target: *profile.ResolvedTarget, TargetPresence: profile.TargetPresence,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
payload := <-payloads
|
||||
if payload["temperature"] != float64(0) || payload["max_tokens"] != float64(0) {
|
||||
t.Fatalf("explicit zeros were omitted from outbound request: %+v", payload)
|
||||
}
|
||||
if _, exists := payload["top_p"]; exists {
|
||||
t.Fatalf("explicit null top_p was sent: %+v", payload)
|
||||
}
|
||||
}
|
||||
@@ -60,15 +60,18 @@ type CacheControl struct {
|
||||
|
||||
// RunRequest represents a request to generate a single artifact.
|
||||
type RunRequest struct {
|
||||
PromptID string
|
||||
PromptVersion string
|
||||
ProfileID string
|
||||
SessionID string
|
||||
APIKey string `json:"-" yaml:"-"`
|
||||
Inputs map[string]ArtifactRef
|
||||
Vars map[string]string
|
||||
Execution *ExecutionTargetOverride
|
||||
Validation *OutputContract
|
||||
PromptID string
|
||||
PromptVersion string
|
||||
ProfileID string
|
||||
SessionID string
|
||||
APIKey string `json:"-" yaml:"-"`
|
||||
Inputs map[string]ArtifactRef
|
||||
Vars map[string]string
|
||||
Execution *ExecutionTargetOverride
|
||||
Validation *OutputContract
|
||||
AppendedMessages []RenderedMessage
|
||||
TransientRetries *int
|
||||
Observer ExecutionObserver `json:"-"`
|
||||
}
|
||||
|
||||
// RunResult represents the complete result of a prompt execution run.
|
||||
@@ -189,21 +192,11 @@ type BackendCapacityPolicy struct {
|
||||
QueueCapacity int
|
||||
}
|
||||
|
||||
// ExecutionProfile describes how and where to execute a model.
|
||||
// ExecutionProfile holds a captured configuration target for execution.
|
||||
type ExecutionProfile struct {
|
||||
ID string `yaml:"id"`
|
||||
BackendID string `yaml:"backend"`
|
||||
Endpoint string `yaml:"endpoint"`
|
||||
Model string `yaml:"model"`
|
||||
Temperature float64 `yaml:"temperature"`
|
||||
MaxTokens int `yaml:"max_tokens"`
|
||||
TopP float64 `yaml:"top_p"`
|
||||
TimeoutSeconds int `yaml:"timeout_seconds"`
|
||||
ServiceTier string `yaml:"service_tier"`
|
||||
ReasoningEffort string `yaml:"reasoning_effort"`
|
||||
APIKeyEnv string `yaml:"api_key_env"`
|
||||
APIKeyRequired bool `yaml:"-" json:"-"`
|
||||
ExtraParams map[string]any `yaml:"extra_params"`
|
||||
ID string
|
||||
ResolvedTarget *ExecutionTarget
|
||||
TargetPresence ExecutionTargetPresence
|
||||
}
|
||||
|
||||
// ExecutionTargetOverride represents per-request runtime setting overrides.
|
||||
@@ -220,8 +213,9 @@ type ExecutionTargetOverride struct {
|
||||
ExtraParams map[string]any `json:"extra_params,omitempty"`
|
||||
}
|
||||
|
||||
// ExecutionTargetPresence tracks which effective runtime fields came from an
|
||||
// explicit request override even when the resolved value is a zero value.
|
||||
// ExecutionTargetPresence tracks effective numeric values that must be sent
|
||||
// even when they are zero. Values may come from configuration or a request
|
||||
// override; explicit configuration null suppresses the corresponding bit.
|
||||
type ExecutionTargetPresence struct {
|
||||
Temperature bool
|
||||
MaxTokens bool
|
||||
@@ -280,6 +274,8 @@ type GenerateRequest struct {
|
||||
Target ExecutionTarget
|
||||
TargetPresence ExecutionTargetPresence
|
||||
StructuredOutput *StructuredOutputSpec
|
||||
TransientRetries uint64
|
||||
Generation uint64
|
||||
}
|
||||
|
||||
// StructuredOutputType indicates which provider-level output mode is requested.
|
||||
@@ -304,8 +300,18 @@ type StructuredOutputJSONSpec struct {
|
||||
|
||||
// GenerateResponse is the response received from the LLM client.
|
||||
type GenerateResponse struct {
|
||||
Content string
|
||||
Usage TokenUsage
|
||||
Content string
|
||||
Usage TokenUsage
|
||||
UsagePresence *TokenUsagePresence
|
||||
}
|
||||
|
||||
// TokenUsagePresence records which token fields a provider reported.
|
||||
type TokenUsagePresence struct {
|
||||
PromptTokens bool
|
||||
CompletionTokens bool
|
||||
TotalTokens bool
|
||||
CachedTokens bool
|
||||
CacheWriteTokens bool
|
||||
}
|
||||
|
||||
// TokenUsage tracks token consumption.
|
||||
|
||||
82
internal/domain/message.go
Normal file
82
internal/domain/message.go
Normal file
@@ -0,0 +1,82 @@
|
||||
package domain
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
const (
|
||||
RoleDeveloper = "developer"
|
||||
RoleSystem = "system"
|
||||
RoleUser = "user"
|
||||
RoleAssistant = "assistant"
|
||||
)
|
||||
|
||||
// NormalizeMessageRole validates and canonicalizes a provider-bound chat role.
|
||||
func NormalizeMessageRole(role string) (string, error) {
|
||||
if !utf8.ValidString(role) {
|
||||
return "", errors.New("message role must be valid UTF-8")
|
||||
}
|
||||
|
||||
normalized := strings.ToLower(strings.TrimSpace(role))
|
||||
switch normalized {
|
||||
case RoleDeveloper, RoleSystem, RoleUser, RoleAssistant:
|
||||
return normalized, nil
|
||||
default:
|
||||
return "", errors.New("message role must be developer, system, user, or assistant")
|
||||
}
|
||||
}
|
||||
|
||||
// NormalizeCacheControl validates, canonicalizes, and copies cache metadata.
|
||||
func NormalizeCacheControl(control *CacheControl) (*CacheControl, error) {
|
||||
if control == nil {
|
||||
return nil, nil
|
||||
}
|
||||
if !utf8.ValidString(string(control.Type)) {
|
||||
return nil, errors.New("cache control type must be valid UTF-8")
|
||||
}
|
||||
if !utf8.ValidString(control.TTL) {
|
||||
return nil, errors.New("cache control ttl must be valid UTF-8")
|
||||
}
|
||||
|
||||
cacheType := strings.TrimSpace(string(control.Type))
|
||||
if cacheType == "" {
|
||||
return nil, errors.New("cache control type is required")
|
||||
}
|
||||
if CacheControlType(cacheType) != CacheControlEphemeral {
|
||||
return nil, errors.New("unsupported type")
|
||||
}
|
||||
|
||||
ttl := strings.TrimSpace(control.TTL)
|
||||
if ttl != "" && ttl != "1h" {
|
||||
return nil, errors.New("unsupported ttl")
|
||||
}
|
||||
|
||||
return &CacheControl{Type: CacheControlType(cacheType), TTL: ttl}, nil
|
||||
}
|
||||
|
||||
// CloneRenderedMessages returns a deep copy of rendered messages.
|
||||
func CloneRenderedMessages(messages []RenderedMessage) []RenderedMessage {
|
||||
cloned := make([]RenderedMessage, len(messages))
|
||||
copyRenderedMessages(cloned, messages)
|
||||
return cloned
|
||||
}
|
||||
|
||||
// ConcatRenderedMessages returns an independently owned concatenation of messages.
|
||||
func ConcatRenderedMessages(prefix, suffix []RenderedMessage) []RenderedMessage {
|
||||
messages := make([]RenderedMessage, len(prefix)+len(suffix))
|
||||
copyRenderedMessages(messages, prefix)
|
||||
copyRenderedMessages(messages[len(prefix):], suffix)
|
||||
return messages
|
||||
}
|
||||
|
||||
func copyRenderedMessages(destination, source []RenderedMessage) {
|
||||
for index, message := range source {
|
||||
destination[index] = message
|
||||
if message.CacheControl != nil {
|
||||
cacheControl := *message.CacheControl
|
||||
destination[index].CacheControl = &cacheControl
|
||||
}
|
||||
}
|
||||
}
|
||||
121
internal/domain/message_test.go
Normal file
121
internal/domain/message_test.go
Normal file
@@ -0,0 +1,121 @@
|
||||
package domain
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestNormalizeMessageRole(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input string
|
||||
want string
|
||||
wantErr bool
|
||||
}{
|
||||
{name: "developer", input: RoleDeveloper, want: RoleDeveloper},
|
||||
{name: "system", input: RoleSystem, want: RoleSystem},
|
||||
{name: "user", input: RoleUser, want: RoleUser},
|
||||
{name: "assistant", input: RoleAssistant, want: RoleAssistant},
|
||||
{name: "surrounding whitespace and mixed case", input: " \u2003UsEr\u2003 ", want: RoleUser},
|
||||
{name: "blank", input: " \t\n ", wantErr: true},
|
||||
{name: "tool", input: "tool", wantErr: true},
|
||||
{name: "function", input: "function", wantErr: true},
|
||||
{name: "custom", input: "custom-role", wantErr: true},
|
||||
{name: "invalid UTF-8", input: string([]byte{0xff}), wantErr: true},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
got, err := NormalizeMessageRole(test.input)
|
||||
if test.wantErr {
|
||||
if err == nil {
|
||||
t.Fatal("expected an error")
|
||||
}
|
||||
if strings.Contains(err.Error(), test.input) {
|
||||
t.Fatalf("error exposed the input: %v", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("NormalizeMessageRole() error = %v", err)
|
||||
}
|
||||
if got != test.want {
|
||||
t.Fatalf("NormalizeMessageRole() = %q, want %q", got, test.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeCacheControl(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input *CacheControl
|
||||
want *CacheControl
|
||||
wantErr bool
|
||||
}{
|
||||
{name: "nil", input: nil, want: nil},
|
||||
{
|
||||
name: "canonical values",
|
||||
input: &CacheControl{Type: CacheControlEphemeral, TTL: "1h"},
|
||||
want: &CacheControl{Type: CacheControlEphemeral, TTL: "1h"},
|
||||
},
|
||||
{
|
||||
name: "trims values",
|
||||
input: &CacheControl{Type: " ephemeral ", TTL: " 1h\t"},
|
||||
want: &CacheControl{Type: CacheControlEphemeral, TTL: "1h"},
|
||||
},
|
||||
{name: "empty type", input: &CacheControl{}, wantErr: true},
|
||||
{name: "unsupported type", input: &CacheControl{Type: "persistent"}, wantErr: true},
|
||||
{name: "unsupported ttl", input: &CacheControl{Type: CacheControlEphemeral, TTL: "5m"}, wantErr: true},
|
||||
{name: "invalid type UTF-8", input: &CacheControl{Type: CacheControlType(string([]byte{0xff}))}, wantErr: true},
|
||||
{name: "invalid ttl UTF-8", input: &CacheControl{Type: CacheControlEphemeral, TTL: string([]byte{0xff})}, wantErr: true},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
got, err := NormalizeCacheControl(test.input)
|
||||
if test.wantErr {
|
||||
if err == nil {
|
||||
t.Fatal("expected an error")
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("NormalizeCacheControl() error = %v", err)
|
||||
}
|
||||
if got == nil || test.want == nil {
|
||||
if got != test.want {
|
||||
t.Fatalf("NormalizeCacheControl() = %#v, want %#v", got, test.want)
|
||||
}
|
||||
return
|
||||
}
|
||||
if *got != *test.want {
|
||||
t.Fatalf("NormalizeCacheControl() = %#v, want %#v", got, test.want)
|
||||
}
|
||||
if got == test.input {
|
||||
t.Fatal("normalized cache control aliases its input")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenderedMessageCloning(t *testing.T) {
|
||||
prefix := []RenderedMessage{{Role: RoleSystem, Content: "prefix", CacheControl: &CacheControl{Type: CacheControlEphemeral, TTL: "1h"}}}
|
||||
suffix := []RenderedMessage{{Role: RoleUser, Content: " suffix "}, {Role: RoleAssistant, Content: "", CacheControl: &CacheControl{Type: CacheControlEphemeral}}}
|
||||
|
||||
cloned := CloneRenderedMessages(prefix)
|
||||
combined := ConcatRenderedMessages(prefix, suffix)
|
||||
if len(combined) != 3 || combined[0].Content != "prefix" || combined[1].Content != " suffix " || combined[2].Content != "" {
|
||||
t.Fatalf("unexpected combined messages: %#v", combined)
|
||||
}
|
||||
if cloned[0].CacheControl == prefix[0].CacheControl || combined[0].CacheControl == prefix[0].CacheControl || combined[2].CacheControl == suffix[1].CacheControl {
|
||||
t.Fatal("cloned cache controls alias their inputs")
|
||||
}
|
||||
|
||||
prefix[0].Content = "changed"
|
||||
prefix[0].CacheControl.TTL = ""
|
||||
suffix[1].CacheControl.Type = "changed"
|
||||
if cloned[0].Content != "prefix" || cloned[0].CacheControl.TTL != "1h" || combined[0].Content != "prefix" || combined[0].CacheControl.TTL != "1h" || combined[2].CacheControl.Type != CacheControlEphemeral {
|
||||
t.Fatalf("cloned messages changed with their inputs: cloned=%#v combined=%#v", cloned, combined)
|
||||
}
|
||||
}
|
||||
90
internal/domain/observation.go
Normal file
90
internal/domain/observation.go
Normal file
@@ -0,0 +1,90 @@
|
||||
package domain
|
||||
|
||||
import "time"
|
||||
|
||||
// ExecutionObserver receives one immutable metadata event synchronously.
|
||||
type ExecutionObserver func(ExecutionEvent)
|
||||
|
||||
type ExecutionEventKind string
|
||||
|
||||
const (
|
||||
ExecutionStarted ExecutionEventKind = "started"
|
||||
ExecutionAttempt ExecutionEventKind = "attempt"
|
||||
ExecutionRetryDelay ExecutionEventKind = "retry_delay"
|
||||
ExecutionFinished ExecutionEventKind = "finished"
|
||||
)
|
||||
|
||||
type ExecutionOutcome string
|
||||
|
||||
const (
|
||||
ExecutionOutcomeSuccess ExecutionOutcome = "success"
|
||||
ExecutionOutcomeInvalidOutput ExecutionOutcome = "invalid_output"
|
||||
ExecutionOutcomeError ExecutionOutcome = "error"
|
||||
)
|
||||
|
||||
type ExecutionFailureKind string
|
||||
|
||||
const (
|
||||
ExecutionFailureNone ExecutionFailureKind = ""
|
||||
ExecutionFailureCanceled ExecutionFailureKind = "canceled"
|
||||
ExecutionFailureDeadline ExecutionFailureKind = "deadline"
|
||||
ExecutionFailureTransport ExecutionFailureKind = "transport"
|
||||
ExecutionFailureProviderStatus ExecutionFailureKind = "provider_status"
|
||||
ExecutionFailureMalformedResponse ExecutionFailureKind = "malformed_response"
|
||||
ExecutionFailureCapacity ExecutionFailureKind = "capacity"
|
||||
ExecutionFailureInvalidConfig ExecutionFailureKind = "invalid_config"
|
||||
ExecutionFailureInvalidRequest ExecutionFailureKind = "invalid_request"
|
||||
ExecutionFailurePromptNotFound ExecutionFailureKind = "prompt_not_found"
|
||||
ExecutionFailureProfileNotFound ExecutionFailureKind = "profile_not_found"
|
||||
ExecutionFailurePromptLoad ExecutionFailureKind = "prompt_load"
|
||||
ExecutionFailureProfileLoad ExecutionFailureKind = "profile_load"
|
||||
ExecutionFailureArtifactLoad ExecutionFailureKind = "artifact_load"
|
||||
ExecutionFailurePromptRender ExecutionFailureKind = "prompt_render"
|
||||
ExecutionFailureValidation ExecutionFailureKind = "validation"
|
||||
ExecutionFailureGeneration ExecutionFailureKind = "generation"
|
||||
ExecutionFailureOther ExecutionFailureKind = "other"
|
||||
)
|
||||
|
||||
// ObservedTokenCount distinguishes omitted usage from reported zero.
|
||||
type ObservedTokenCount struct {
|
||||
Value int
|
||||
Known bool
|
||||
Complete bool
|
||||
}
|
||||
|
||||
// ObservedTokenUsage reports presence-aware token counts.
|
||||
type ObservedTokenUsage struct {
|
||||
PromptTokens ObservedTokenCount
|
||||
CompletionTokens ObservedTokenCount
|
||||
TotalTokens ObservedTokenCount
|
||||
CachedTokens ObservedTokenCount
|
||||
CacheWriteTokens ObservedTokenCount
|
||||
}
|
||||
|
||||
// ExecutionEvent contains metadata only; it carries no mutable containers or
|
||||
// error values. Nonterminal events leave aggregate counters at zero.
|
||||
type ExecutionEvent struct {
|
||||
Kind ExecutionEventKind
|
||||
RunID string
|
||||
Time time.Time
|
||||
PromptID string
|
||||
PromptVersion string
|
||||
ProfileID string
|
||||
BackendID string
|
||||
Model string
|
||||
Generation uint64
|
||||
Attempt uint64
|
||||
Dispatched bool
|
||||
CapacityWait time.Duration
|
||||
GenerationDuration time.Duration
|
||||
RetryDelay time.Duration
|
||||
Duration time.Duration
|
||||
Outcome ExecutionOutcome
|
||||
Failure ExecutionFailureKind
|
||||
StatusCode int
|
||||
RetriesExhausted bool
|
||||
Usage ObservedTokenUsage
|
||||
Attempts uint64
|
||||
TransientRetries uint64
|
||||
RepairCalls uint64
|
||||
}
|
||||
@@ -6,6 +6,8 @@ import (
|
||||
"strings"
|
||||
)
|
||||
|
||||
const maxOutputRepairAttempts = 3
|
||||
|
||||
// ValidateOutputContract validates source-neutral output-contract invariants.
|
||||
func ValidateOutputContract(contract OutputContract) error {
|
||||
switch contract.Format {
|
||||
@@ -26,5 +28,11 @@ func ValidateOutputContract(contract OutputContract) error {
|
||||
if contract.RepairAttempts < 0 {
|
||||
return errors.New("repair_attempts must be greater than or equal to 0")
|
||||
}
|
||||
if contract.RepairAttempts > maxOutputRepairAttempts {
|
||||
return fmt.Errorf("repair_attempts must be less than or equal to %d", maxOutputRepairAttempts)
|
||||
}
|
||||
if contract.ValidationMode == ValidationNone && contract.RepairAttempts > 0 {
|
||||
return errors.New("repair_attempts requires basic, json, or json_schema validation")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -30,9 +30,28 @@ func TestValidateOutputContract(t *testing.T) {
|
||||
}},
|
||||
{name: "empty validation mode", change: func(c *OutputContract) { c.ValidationMode = "" }, wantErr: "validation mode"},
|
||||
{name: "unsupported validation mode", change: func(c *OutputContract) { c.ValidationMode = ValidationMode("unknown") }, wantErr: "validation mode"},
|
||||
{name: "negative repair attempts", change: func(c *OutputContract) { c.RepairAttempts = -1 }, wantErr: "repair_attempts"},
|
||||
{name: "negative repair attempts", change: func(c *OutputContract) {
|
||||
c.ValidationMode = ValidationBasic
|
||||
c.RepairAttempts = -1
|
||||
}, wantErr: "repair_attempts"},
|
||||
{name: "zero repair attempts", change: func(c *OutputContract) { c.RepairAttempts = 0 }},
|
||||
{name: "positive repair attempts", change: func(c *OutputContract) { c.RepairAttempts = 1 }},
|
||||
{name: "one repair attempt", change: func(c *OutputContract) {
|
||||
c.ValidationMode = ValidationBasic
|
||||
c.RepairAttempts = 1
|
||||
}},
|
||||
{name: "maximum repair attempts", change: func(c *OutputContract) {
|
||||
c.ValidationMode = ValidationJSON
|
||||
c.RepairAttempts = 3
|
||||
}},
|
||||
{name: "too many repair attempts", change: func(c *OutputContract) {
|
||||
c.ValidationMode = ValidationJSONSchema
|
||||
c.SchemaPath = "schema.json"
|
||||
c.RepairAttempts = 4
|
||||
}, wantErr: "repair_attempts"},
|
||||
{name: "none validation with repair attempts", change: func(c *OutputContract) {
|
||||
c.ValidationMode = ValidationNone
|
||||
c.RepairAttempts = 1
|
||||
}, wantErr: "repair_attempts"},
|
||||
{name: "json schema empty path", change: func(c *OutputContract) {
|
||||
c.ValidationMode = ValidationJSONSchema
|
||||
c.SchemaPath = ""
|
||||
|
||||
15
internal/domain/retries.go
Normal file
15
internal/domain/retries.go
Normal file
@@ -0,0 +1,15 @@
|
||||
package domain
|
||||
|
||||
import "fmt"
|
||||
|
||||
// ResolveTransientRetries selects a nonnegative retry budget. A nil selection
|
||||
// inherits the supplied default.
|
||||
func ResolveTransientRetries(defaultBudget uint64, selection *int) (uint64, error) {
|
||||
if selection == nil {
|
||||
return defaultBudget, nil
|
||||
}
|
||||
if *selection < 0 {
|
||||
return 0, fmt.Errorf("transient retries must not be negative")
|
||||
}
|
||||
return uint64(*selection), nil
|
||||
}
|
||||
195
internal/engine/engine.go
Normal file
195
internal/engine/engine.go
Normal file
@@ -0,0 +1,195 @@
|
||||
// Package engine assembles Promptkit's internal execution components.
|
||||
package engine
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/artifact"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/capacity"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/catalog"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/configuration"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/defaults"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/generation"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/llm"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/prompt"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/usecase"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/validate"
|
||||
)
|
||||
|
||||
// Config carries construction settings selected by the public facade.
|
||||
type Config struct {
|
||||
Sources ConfigurationSources
|
||||
SchemaDir string
|
||||
Timeout time.Duration
|
||||
HTTPClient *http.Client
|
||||
TransientRetries *int
|
||||
}
|
||||
|
||||
// ConfigurationSources identifies caller-selected ConfigKit layers.
|
||||
type ConfigurationSources struct {
|
||||
Underlays []configkit.Underlay
|
||||
Main *configkit.Input
|
||||
}
|
||||
|
||||
// Options collects construction choices from the public facade.
|
||||
type Options struct {
|
||||
llmClient llm.Client
|
||||
artifactReader artifact.Reader
|
||||
validator validate.Validator
|
||||
validatorSource bool
|
||||
artifactSource bool
|
||||
}
|
||||
|
||||
// SetLLMClient selects the client collaborator for one engine.
|
||||
func (o *Options) SetLLMClient(client llm.Client) { o.llmClient = client }
|
||||
|
||||
// SetArtifactReader selects the artifact collaborator for one engine.
|
||||
func (o *Options) SetArtifactReader(reader artifact.Reader) {
|
||||
o.artifactReader = reader
|
||||
o.artifactSource = true
|
||||
}
|
||||
|
||||
// SetSchemaFS selects a schema source rooted in fsys.
|
||||
func (o *Options) SetSchemaFS(fsys fs.FS, root string) error {
|
||||
if err := validateSourceFS(fsys, root); err != nil {
|
||||
return err
|
||||
}
|
||||
o.validator = validate.NewFSValidator(fsys, root)
|
||||
o.validatorSource = true
|
||||
return nil
|
||||
}
|
||||
|
||||
// SetSchemaFile selects a single schema file.
|
||||
func (o *Options) SetSchemaFile(path string) error {
|
||||
fsys, root, _, err := fileSource(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
o.validator = validate.NewFSValidator(fsys, root)
|
||||
o.validatorSource = true
|
||||
return nil
|
||||
}
|
||||
|
||||
// Engine forwards domain operations to the single assembled use-case runner.
|
||||
type Engine struct {
|
||||
runner *usecase.Runner
|
||||
inspection configuration.Inspection
|
||||
}
|
||||
|
||||
// New assembles an engine from facade-selected configuration and collaborators.
|
||||
func New(cfg Config, options Options) (*Engine, error) {
|
||||
defaultRetries, err := domain.ResolveTransientRetries(2, cfg.TransientRetries)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
underlays, err := catalog.Underlays()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("select maintained configuration: %w", err)
|
||||
}
|
||||
underlays = append(underlays, cfg.Sources.Underlays...)
|
||||
main := configkit.Input{Source: configkit.NoSource()}
|
||||
if cfg.Sources.Main != nil {
|
||||
main = *cfg.Sources.Main
|
||||
}
|
||||
snapshot, err := configuration.Load(main, underlays)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
manager, err := capacity.NewManager(snapshot.CapacityPolicies())
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to construct backend capacity manager: %w", err)
|
||||
}
|
||||
|
||||
validator := options.validator
|
||||
if !options.validatorSource {
|
||||
schemaDir := cfg.SchemaDir
|
||||
if strings.TrimSpace(schemaDir) == "" {
|
||||
schemaDir = defaults.SchemaDirDefault
|
||||
}
|
||||
validator = validate.NewStandardValidator(schemaDir)
|
||||
}
|
||||
var client llm.Client
|
||||
if options.llmClient == nil {
|
||||
builtIn, newErr := llm.NewOpenAICompatibleClient(llm.OpenAICompatibleConfig{Timeout: cfg.Timeout, HTTPClient: cfg.HTTPClient})
|
||||
if newErr != nil {
|
||||
return nil, newErr
|
||||
}
|
||||
client = generation.NewBuiltIn(manager, builtIn)
|
||||
} else {
|
||||
client = generation.NewInjected(manager, options.llmClient)
|
||||
}
|
||||
reader := options.artifactReader
|
||||
if !options.artifactSource {
|
||||
reader = artifact.NewCompositeReader()
|
||||
}
|
||||
|
||||
return &Engine{
|
||||
runner: usecase.NewRunnerWithRepairer(snapshot, snapshot, reader, prompt.NewCompiledRenderer(snapshot), client, validator, usecase.NewDefaultOutputRepairer(client), manager, defaultRetries),
|
||||
inspection: snapshot.Inspection(),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// InspectConfiguration returns read-only operations over the captured result.
|
||||
func (e *Engine) InspectConfiguration() configuration.Inspection {
|
||||
if e == nil {
|
||||
return nil
|
||||
}
|
||||
return e.inspection
|
||||
}
|
||||
|
||||
func validateSourceFS(fsys fs.FS, root string) error {
|
||||
if fsys == nil {
|
||||
return fmt.Errorf("source filesystem is nil")
|
||||
}
|
||||
if strings.TrimSpace(root) == "" {
|
||||
return fmt.Errorf("source root is required")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func fileSource(name string) (fs.FS, string, string, error) {
|
||||
if strings.TrimSpace(name) == "" {
|
||||
return nil, "", "", fmt.Errorf("source path is required")
|
||||
}
|
||||
directory := filepath.Dir(name)
|
||||
base := filepath.Base(name)
|
||||
if base == "." || base == string(filepath.Separator) {
|
||||
return nil, "", "", fmt.Errorf("source path must name a file")
|
||||
}
|
||||
info, err := os.Stat(name)
|
||||
if err != nil {
|
||||
return nil, "", "", fmt.Errorf("failed to access source file %q: %w", name, err)
|
||||
}
|
||||
if info.IsDir() {
|
||||
return nil, "", "", fmt.Errorf("source path %q must be a file", name)
|
||||
}
|
||||
return os.DirFS(directory), filepath.ToSlash(base), directory, nil
|
||||
}
|
||||
|
||||
func (e *Engine) InspectPrompt(ctx context.Context, id, version string) (*domain.PromptInspection, error) {
|
||||
return e.runner.InspectPrompt(ctx, id, version)
|
||||
}
|
||||
func (e *Engine) InspectProfile(ctx context.Context, id string) (*domain.ProfileInspection, error) {
|
||||
return e.runner.InspectProfile(ctx, id)
|
||||
}
|
||||
func (e *Engine) Prepare(ctx context.Context, request domain.RunRequest) (*domain.PreparedRun, error) {
|
||||
return e.runner.Prepare(ctx, request)
|
||||
}
|
||||
func (e *Engine) PrepareExecution(ctx context.Context, request domain.RunRequest) (*usecase.PreparedExecution, error) {
|
||||
return e.runner.PrepareExecution(ctx, request)
|
||||
}
|
||||
func (e *Engine) Run(ctx context.Context, request domain.RunRequest) (*domain.RunResult, error) {
|
||||
return e.runner.Run(ctx, request)
|
||||
}
|
||||
func (e *Engine) RunPrepared(ctx context.Context, prepared *usecase.PreparedExecution) (*domain.RunResult, error) {
|
||||
return e.runner.RunPrepared(ctx, prepared)
|
||||
}
|
||||
294
internal/generation/coordinator.go
Normal file
294
internal/generation/coordinator.go
Normal file
@@ -0,0 +1,294 @@
|
||||
// Package generation coordinates model attempts, backend permits, and bounded
|
||||
// built-in HTTP retries for one generation request.
|
||||
package generation
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math/rand/v2"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/capacity"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/llm"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/observation"
|
||||
)
|
||||
|
||||
const (
|
||||
initialBackoff = 500 * time.Millisecond
|
||||
maximumBackoff = 8 * time.Second
|
||||
)
|
||||
|
||||
// Coordinator implements the internal model client using the existing backend
|
||||
// capacity manager. Exactly one of builtIn and injected is selected.
|
||||
type Coordinator struct {
|
||||
manager *capacity.Manager
|
||||
builtIn *llm.OpenAICompatibleClient
|
||||
injected llm.Client
|
||||
now func() time.Time
|
||||
jitter func(time.Duration) time.Duration
|
||||
wait func(context.Context, time.Duration) error
|
||||
}
|
||||
|
||||
// NewBuiltIn selects the built-in HTTP retry path.
|
||||
func NewBuiltIn(manager *capacity.Manager, client *llm.OpenAICompatibleClient) *Coordinator {
|
||||
return &Coordinator{manager: manager, builtIn: client, now: time.Now, jitter: randomJitter, wait: waitContext}
|
||||
}
|
||||
|
||||
// NewInjected selects one opaque client dispatch with no automatic retry.
|
||||
func NewInjected(manager *capacity.Manager, client llm.Client) *Coordinator {
|
||||
return &Coordinator{manager: manager, injected: client, now: time.Now, jitter: randomJitter, wait: waitContext}
|
||||
}
|
||||
|
||||
// ExhaustedError retains the final eligible status failure and the number of
|
||||
// attempts actually dispatched.
|
||||
type ExhaustedError struct {
|
||||
attempts uint64
|
||||
cause error
|
||||
}
|
||||
|
||||
func (e *ExhaustedError) Error() string {
|
||||
if e == nil {
|
||||
return "transient model retries exhausted"
|
||||
}
|
||||
return fmt.Sprintf("transient model retries exhausted after %d attempts", e.attempts)
|
||||
}
|
||||
|
||||
func (e *ExhaustedError) Unwrap() error {
|
||||
if e == nil {
|
||||
return nil
|
||||
}
|
||||
return e.cause
|
||||
}
|
||||
|
||||
func (e *ExhaustedError) Attempts() uint64 {
|
||||
if e == nil {
|
||||
return 0
|
||||
}
|
||||
return e.attempts
|
||||
}
|
||||
|
||||
// Generate executes either one opaque client call or a frozen built-in HTTP
|
||||
// sequence with one permit per actual dispatch.
|
||||
func (c *Coordinator) Generate(ctx context.Context, req domain.GenerateRequest) (*domain.GenerateResponse, error) {
|
||||
if c.injected != nil {
|
||||
return c.generateInjected(ctx, req)
|
||||
}
|
||||
prepared, err := c.builtIn.PrepareGeneration(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var attempts, retriesUsed uint64
|
||||
recorder := observation.FromContext(ctx)
|
||||
generation := req.Generation
|
||||
if generation == 0 {
|
||||
generation = 1
|
||||
}
|
||||
for {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ordinal := attempts + 1
|
||||
outcome, dispatched, capacityWait, generationDuration := c.execute(ctx, req.Target.BackendID, prepared)
|
||||
if dispatched {
|
||||
attempts++
|
||||
}
|
||||
if recorder != nil {
|
||||
recorder.Attempt(observation.AttemptData{Generation: generation, Attempt: ordinal,
|
||||
Dispatched: dispatched, CapacityWait: capacityWait, GenerationDuration: generationDuration,
|
||||
Outcome: attemptOutcome(outcome.Err), Failure: attemptFailure(outcome.Err),
|
||||
StatusCode: outcome.StatusCode, Usage: outcome.Usage, UsagePresence: outcome.UsagePresence})
|
||||
}
|
||||
if outcome.Err == nil {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return outcome.Response, nil
|
||||
}
|
||||
if !dispatched || !eligibleStatus(outcome) {
|
||||
return nil, outcome.Err
|
||||
}
|
||||
if retriesUsed == req.TransientRetries {
|
||||
if retriesUsed == 0 {
|
||||
return nil, outcome.Err
|
||||
}
|
||||
return nil, &ExhaustedError{attempts: attempts, cause: outcome.Err}
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
retryOrdinal := retriesUsed + 1
|
||||
delay := retryDelay(c.backoff(retryOrdinal), outcome.RetryNotBefore, c.now())
|
||||
waitStart := c.now()
|
||||
waitErr := c.wait(ctx, delay)
|
||||
if recorder != nil {
|
||||
recorder.Delay(observation.DelayData{Generation: generation, Attempt: attempts,
|
||||
Elapsed: elapsed(waitStart, c.now()), Failure: attemptFailure(waitErr)})
|
||||
}
|
||||
if waitErr != nil {
|
||||
return nil, waitErr
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
retriesUsed++
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Coordinator) generateInjected(ctx context.Context, req domain.GenerateRequest) (*domain.GenerateResponse, error) {
|
||||
recorder := observation.FromContext(ctx)
|
||||
generation := req.Generation
|
||||
if generation == 0 {
|
||||
generation = 1
|
||||
}
|
||||
start := c.now()
|
||||
release, err := c.manager.Acquire(ctx, req.Target.BackendID)
|
||||
capacityWait := c.capacityWait(req.Target.BackendID, start)
|
||||
if err != nil {
|
||||
if recorder != nil {
|
||||
recorder.Attempt(observation.AttemptData{Generation: generation, Attempt: 1, CapacityWait: capacityWait, Outcome: domain.ExecutionOutcomeError, Failure: attemptFailure(err)})
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
defer release()
|
||||
if err := ctx.Err(); err != nil {
|
||||
release()
|
||||
if recorder != nil {
|
||||
recorder.Attempt(observation.AttemptData{Generation: generation, Attempt: 1, CapacityWait: capacityWait, Outcome: domain.ExecutionOutcomeError, Failure: attemptFailure(err)})
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
dispatchStart := c.now()
|
||||
response, err := c.injected.Generate(ctx, req)
|
||||
duration := elapsed(dispatchStart, c.now())
|
||||
if err == nil && response == nil {
|
||||
err = llm.ErrMalformedResponse
|
||||
}
|
||||
release()
|
||||
if recorder != nil {
|
||||
data := observation.AttemptData{Generation: generation, Attempt: 1, Dispatched: true, CapacityWait: capacityWait, GenerationDuration: duration,
|
||||
Outcome: attemptOutcome(err), Failure: attemptFailure(err)}
|
||||
if err == nil {
|
||||
data.Usage = response.Usage
|
||||
data.UsagePresence = response.UsagePresence
|
||||
}
|
||||
recorder.Attempt(data)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return response, nil
|
||||
}
|
||||
|
||||
func (c *Coordinator) execute(ctx context.Context, backendID string, prepared *llm.PreparedGeneration) (outcome llm.AttemptOutcome, dispatched bool, capacityWait, generationDuration time.Duration) {
|
||||
start := c.now()
|
||||
release, err := c.manager.Acquire(ctx, backendID)
|
||||
capacityWait = c.capacityWait(backendID, start)
|
||||
if err != nil {
|
||||
return llm.AttemptOutcome{Err: err}, false, capacityWait, 0
|
||||
}
|
||||
defer release()
|
||||
if err := ctx.Err(); err != nil {
|
||||
return llm.AttemptOutcome{Err: err}, false, capacityWait, 0
|
||||
}
|
||||
dispatchStart := c.now()
|
||||
outcome = prepared.Execute(ctx)
|
||||
return outcome, true, capacityWait, elapsed(dispatchStart, c.now())
|
||||
}
|
||||
|
||||
func (c *Coordinator) capacityWait(backendID string, start time.Time) time.Duration {
|
||||
if !c.manager.Limited(backendID) {
|
||||
return 0
|
||||
}
|
||||
return elapsed(start, c.now())
|
||||
}
|
||||
|
||||
func elapsed(start, end time.Time) time.Duration {
|
||||
if d := end.Sub(start); d > 0 {
|
||||
return d
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func attemptOutcome(err error) domain.ExecutionOutcome {
|
||||
if err == nil {
|
||||
return domain.ExecutionOutcomeSuccess
|
||||
}
|
||||
return domain.ExecutionOutcomeError
|
||||
}
|
||||
|
||||
func attemptFailure(err error) domain.ExecutionFailureKind {
|
||||
if err == nil {
|
||||
return domain.ExecutionFailureNone
|
||||
}
|
||||
if errors.Is(err, context.Canceled) {
|
||||
return domain.ExecutionFailureCanceled
|
||||
}
|
||||
if errors.Is(err, context.DeadlineExceeded) {
|
||||
return domain.ExecutionFailureDeadline
|
||||
}
|
||||
if errors.Is(err, llm.ErrUnexpectedStatus) {
|
||||
return domain.ExecutionFailureProviderStatus
|
||||
}
|
||||
if errors.Is(err, llm.ErrTransport) {
|
||||
return domain.ExecutionFailureTransport
|
||||
}
|
||||
if errors.Is(err, llm.ErrMalformedResponse) {
|
||||
return domain.ExecutionFailureMalformedResponse
|
||||
}
|
||||
if errors.Is(err, llm.ErrInvalidRequest) {
|
||||
return domain.ExecutionFailureInvalidRequest
|
||||
}
|
||||
return domain.ExecutionFailureGeneration
|
||||
}
|
||||
|
||||
func eligibleStatus(outcome llm.AttemptOutcome) bool {
|
||||
if !errors.Is(outcome.Err, llm.ErrUnexpectedStatus) {
|
||||
return false
|
||||
}
|
||||
switch outcome.StatusCode {
|
||||
case 429, 500, 502, 503, 504:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Coordinator) backoff(ordinal uint64) time.Duration {
|
||||
base := initialBackoff
|
||||
for step := uint64(1); step < ordinal && base < maximumBackoff; step++ {
|
||||
base *= 2
|
||||
}
|
||||
half := base / 2
|
||||
return half + c.jitter(half)
|
||||
}
|
||||
|
||||
func retryDelay(local time.Duration, notBefore, now time.Time) time.Duration {
|
||||
if providerDelay := notBefore.Sub(now); providerDelay > local {
|
||||
return providerDelay
|
||||
}
|
||||
return local
|
||||
}
|
||||
|
||||
func randomJitter(max time.Duration) time.Duration {
|
||||
return time.Duration(rand.Int64N(int64(max) + 1))
|
||||
}
|
||||
|
||||
func waitContext(ctx context.Context, delay time.Duration) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
timer := time.NewTimer(delay)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-timer.C:
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
278
internal/generation/coordinator_test.go
Normal file
278
internal/generation/coordinator_test.go
Normal file
@@ -0,0 +1,278 @@
|
||||
package generation
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"math"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/capacity"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/llm"
|
||||
)
|
||||
|
||||
type transportFunc func(*http.Request) (*http.Response, error)
|
||||
|
||||
func (f transportFunc) RoundTrip(req *http.Request) (*http.Response, error) { return f(req) }
|
||||
|
||||
func response(status int, body string) *http.Response {
|
||||
return &http.Response{StatusCode: status, Body: io.NopCloser(strings.NewReader(body)), Header: make(http.Header)}
|
||||
}
|
||||
|
||||
func testRequest(retries uint64) domain.GenerateRequest {
|
||||
return domain.GenerateRequest{
|
||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hello"}}},
|
||||
Target: domain.ExecutionTarget{BackendID: "limited", Endpoint: "https://example.invalid/v1", Model: "model"},
|
||||
TransientRetries: retries,
|
||||
}
|
||||
}
|
||||
|
||||
func testCoordinator(t *testing.T, transport transportFunc) *Coordinator {
|
||||
t.Helper()
|
||||
manager, err := capacity.NewManager(map[string]domain.BackendCapacityPolicy{
|
||||
"limited": {ConcurrencyLimit: 1, QueueCapacity: 1},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
client, err := llm.NewOpenAICompatibleClient(llm.OpenAICompatibleConfig{HTTPClient: &http.Client{Transport: transport}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
coordinator := NewBuiltIn(manager, client)
|
||||
coordinator.jitter = func(time.Duration) time.Duration { return 0 }
|
||||
coordinator.wait = func(ctx context.Context, _ time.Duration) error { return ctx.Err() }
|
||||
return coordinator
|
||||
}
|
||||
|
||||
func TestBuiltInRetryEligibilityAndExhaustion(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
status int
|
||||
eligible bool
|
||||
}{
|
||||
{status: 429, eligible: true}, {status: 500, eligible: true},
|
||||
{status: 502, eligible: true}, {status: 503, eligible: true},
|
||||
{status: 504, eligible: true}, {status: 400}, {status: 501},
|
||||
} {
|
||||
t.Run(http.StatusText(tc.status), func(t *testing.T) {
|
||||
calls := 0
|
||||
coordinator := testCoordinator(t, func(*http.Request) (*http.Response, error) {
|
||||
calls++
|
||||
if calls == 1 {
|
||||
return response(tc.status, `{"error":{}}`), nil
|
||||
}
|
||||
return response(200, `{"choices":[{"message":{"content":"ok"}}]}`), nil
|
||||
})
|
||||
result, err := coordinator.Generate(context.Background(), testRequest(1))
|
||||
if tc.eligible {
|
||||
if err != nil || result == nil || result.Content != "ok" || calls != 2 {
|
||||
t.Fatalf("result=%+v, error=%v, calls=%d", result, err, calls)
|
||||
}
|
||||
} else if result != nil || err == nil || calls != 1 {
|
||||
t.Fatalf("result=%+v, error=%v, calls=%d", result, err, calls)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
for _, budget := range []uint64{0, 2} {
|
||||
calls := 0
|
||||
coordinator := testCoordinator(t, func(*http.Request) (*http.Response, error) {
|
||||
calls++
|
||||
return response(503, `{"error":{}}`), nil
|
||||
})
|
||||
_, err := coordinator.Generate(context.Background(), testRequest(budget))
|
||||
var exhausted *ExhaustedError
|
||||
if !errors.Is(err, llm.ErrUnexpectedStatus) || calls != int(budget+1) || errors.As(err, &exhausted) != (budget > 0) {
|
||||
t.Fatalf("budget=%d calls=%d error=%v", budget, calls, err)
|
||||
}
|
||||
if budget > 0 && exhausted.Attempts() != budget+1 {
|
||||
t.Fatalf("attempts=%d", exhausted.Attempts())
|
||||
}
|
||||
}
|
||||
|
||||
calls := 0
|
||||
coordinator := testCoordinator(t, func(*http.Request) (*http.Response, error) {
|
||||
calls++
|
||||
if calls == 1 {
|
||||
return response(503, `{"error":{}}`), nil
|
||||
}
|
||||
return response(400, `{"error":{}}`), nil
|
||||
})
|
||||
_, err := coordinator.Generate(context.Background(), testRequest(3))
|
||||
var exhausted *ExhaustedError
|
||||
if calls != 2 || !errors.Is(err, llm.ErrUnexpectedStatus) || errors.As(err, &exhausted) {
|
||||
t.Fatalf("later permanent failure: calls=%d, error=%v", calls, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuiltInRetryWaitHonorsContextAndProviderMinimum(t *testing.T) {
|
||||
when := time.Date(2026, 9, 23, 12, 0, 0, 0, time.UTC)
|
||||
coordinator := testCoordinator(t, func(*http.Request) (*http.Response, error) { return response(503, `{"error":{}}`), nil })
|
||||
if got := coordinator.backoff(1); got != 250*time.Millisecond {
|
||||
t.Fatalf("first backoff=%v", got)
|
||||
}
|
||||
if got := coordinator.backoff(math.MaxUint64); got != 4*time.Second {
|
||||
t.Fatalf("capped backoff=%v", got)
|
||||
}
|
||||
coordinator.jitter = func(max time.Duration) time.Duration { return max }
|
||||
if got := coordinator.backoff(math.MaxUint64); got != 8*time.Second {
|
||||
t.Fatalf("maximum equal jitter=%v", got)
|
||||
}
|
||||
if got := retryDelay(500*time.Millisecond, when.Add(3*time.Second), when); got != 3*time.Second {
|
||||
t.Fatalf("provider minimum=%v", got)
|
||||
}
|
||||
if got := retryDelay(500*time.Millisecond, when.Add(-time.Second), when); got != 500*time.Millisecond {
|
||||
t.Fatalf("past provider minimum=%v", got)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
calls := 0
|
||||
coordinator = testCoordinator(t, func(*http.Request) (*http.Response, error) {
|
||||
calls++
|
||||
return response(503, `{"error":{}}`), nil
|
||||
})
|
||||
coordinator.wait = func(ctx context.Context, delay time.Duration) error {
|
||||
if delay < 250*time.Millisecond {
|
||||
t.Fatalf("delay=%v", delay)
|
||||
}
|
||||
cancel()
|
||||
return ctx.Err()
|
||||
}
|
||||
_, err := coordinator.Generate(ctx, testRequest(2))
|
||||
if !errors.Is(err, context.Canceled) || calls != 1 {
|
||||
t.Fatalf("canceled wait: error=%v calls=%d", err, calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuiltInRetryUsesSeparateAttemptDeadlines(t *testing.T) {
|
||||
var contexts []context.Context
|
||||
coordinator := testCoordinator(t, func(req *http.Request) (*http.Response, error) {
|
||||
contexts = append(contexts, req.Context())
|
||||
if len(contexts) == 1 {
|
||||
return response(503, `{"error":{}}`), nil
|
||||
}
|
||||
return response(200, `{"choices":[{"message":{"content":"ok"}}]}`), nil
|
||||
})
|
||||
req := testRequest(1)
|
||||
req.Target.TimeoutSeconds = 1
|
||||
if _, err := coordinator.Generate(context.Background(), req); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(contexts) != 2 || contexts[0] == contexts[1] {
|
||||
t.Fatalf("attempt contexts=%d", len(contexts))
|
||||
}
|
||||
for _, ctx := range contexts {
|
||||
if _, ok := ctx.Deadline(); !ok || !errors.Is(ctx.Err(), context.Canceled) {
|
||||
t.Fatalf("attempt deadline/error=%t/%v", ok, ctx.Err())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCanceledCapacityWaitDispatchesNothing(t *testing.T) {
|
||||
calls := 0
|
||||
coordinator := testCoordinator(t, func(*http.Request) (*http.Response, error) {
|
||||
calls++
|
||||
return response(200, `{"choices":[{"message":{"content":"ok"}}]}`), nil
|
||||
})
|
||||
release, err := coordinator.manager.Acquire(context.Background(), "limited")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer release()
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
result := make(chan error, 1)
|
||||
go func() { _, err := coordinator.Generate(ctx, testRequest(2)); result <- err }()
|
||||
cancel()
|
||||
if err := <-result; !errors.Is(err, context.Canceled) || calls != 0 {
|
||||
t.Fatalf("capacity cancellation: error=%v calls=%d", err, calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuiltInRetryFreesPermitAndFreezesCredential(t *testing.T) {
|
||||
const envName = "PROMPTKIT_GENERATION_RETRY_TEST_KEY"
|
||||
t.Setenv(envName, "first-key")
|
||||
var calls atomic.Int64
|
||||
var firstAuth, retryAuth string
|
||||
coordinator := testCoordinator(t, func(req *http.Request) (*http.Response, error) {
|
||||
switch calls.Add(1) {
|
||||
case 1:
|
||||
firstAuth = req.Header.Get("Authorization")
|
||||
return response(503, `{"error":{}}`), nil
|
||||
case 2:
|
||||
return response(200, `{"choices":[{"message":{"content":"other"}}]}`), nil
|
||||
default:
|
||||
retryAuth = req.Header.Get("Authorization")
|
||||
return response(200, `{"choices":[{"message":{"content":"retry"}}]}`), nil
|
||||
}
|
||||
})
|
||||
entered := make(chan struct{})
|
||||
resume := make(chan struct{})
|
||||
coordinator.wait = func(ctx context.Context, _ time.Duration) error {
|
||||
close(entered)
|
||||
select {
|
||||
case <-resume:
|
||||
return ctx.Err()
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
req := testRequest(1)
|
||||
req.Target.APIKeyEnv = envName
|
||||
first := make(chan error, 1)
|
||||
go func() { _, err := coordinator.Generate(context.Background(), req); first <- err }()
|
||||
<-entered
|
||||
t.Setenv(envName, "later-key")
|
||||
other, err := coordinator.Generate(context.Background(), testRequest(0))
|
||||
if err != nil || other == nil || other.Content != "other" {
|
||||
t.Fatalf("other call while backoff blocked = %+v, %v", other, err)
|
||||
}
|
||||
close(resume)
|
||||
if err := <-first; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if calls.Load() != 3 || firstAuth != "Bearer first-key" || retryAuth != firstAuth {
|
||||
t.Fatalf("calls=%d, first auth=%q, retry auth=%q", calls.Load(), firstAuth, retryAuth)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInjectedClientDispatchesOnceAndReleasesAfterPanic(t *testing.T) {
|
||||
manager, err := capacity.NewManager(map[string]domain.BackendCapacityPolicy{"limited": {ConcurrencyLimit: 1}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
client := &injectedClient{panicFirst: true}
|
||||
coordinator := NewInjected(manager, client)
|
||||
func() {
|
||||
defer func() {
|
||||
if recover() == nil {
|
||||
t.Fatal("expected injected panic")
|
||||
}
|
||||
}()
|
||||
_, _ = coordinator.Generate(context.Background(), testRequest(5))
|
||||
}()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
result, err := coordinator.Generate(ctx, testRequest(5))
|
||||
if err != nil || result == nil || client.calls != 2 {
|
||||
t.Fatalf("after panic result=%+v error=%v calls=%d", result, err, client.calls)
|
||||
}
|
||||
}
|
||||
|
||||
type injectedClient struct {
|
||||
calls int
|
||||
panicFirst bool
|
||||
}
|
||||
|
||||
func (c *injectedClient) Generate(context.Context, domain.GenerateRequest) (*domain.GenerateResponse, error) {
|
||||
c.calls++
|
||||
if c.panicFirst && c.calls == 1 {
|
||||
panic("injected panic")
|
||||
}
|
||||
return &domain.GenerateResponse{Content: "ok"}, nil
|
||||
}
|
||||
201
internal/generation/observation_test.go
Normal file
201
internal/generation/observation_test.go
Normal file
@@ -0,0 +1,201 @@
|
||||
package generation
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/capacity"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/observation"
|
||||
)
|
||||
|
||||
func recordEvents(t *testing.T, ctx context.Context, events *[]domain.ExecutionEvent) context.Context {
|
||||
t.Helper()
|
||||
recorder, err := observation.New(func(event domain.ExecutionEvent) { *events = append(*events, event) })
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return observation.WithRecorder(ctx, recorder)
|
||||
}
|
||||
|
||||
func TestObservedRetryAndExhaustion(t *testing.T) {
|
||||
for _, status := range []int{200, 503} {
|
||||
calls := 0
|
||||
coordinator := testCoordinator(t, func(*http.Request) (*http.Response, error) {
|
||||
calls++
|
||||
if calls == 1 || status == 503 {
|
||||
return response(503, `{"error":{}}`), nil
|
||||
}
|
||||
return response(200, `{"choices":[{"message":{"content":"ok"}}],"usage":{"prompt_tokens":0,"completion_tokens":2}}`), nil
|
||||
})
|
||||
var events []domain.ExecutionEvent
|
||||
ctx := recordEvents(t, context.Background(), &events)
|
||||
_, err := coordinator.Generate(ctx, testRequest(1))
|
||||
if status == 200 && err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if status == 503 {
|
||||
var exhausted *ExhaustedError
|
||||
if !errors.As(err, &exhausted) {
|
||||
t.Fatalf("error=%v", err)
|
||||
}
|
||||
}
|
||||
if len(events) != 3 || events[0].Kind != domain.ExecutionAttempt || events[1].Kind != domain.ExecutionRetryDelay || events[2].Kind != domain.ExecutionAttempt {
|
||||
t.Fatalf("events=%+v", events)
|
||||
}
|
||||
if events[0].Attempt != 1 || events[1].Attempt != 1 || events[2].Attempt != 2 || !events[0].Dispatched || !events[2].Dispatched {
|
||||
t.Fatalf("indexes=%+v", events)
|
||||
}
|
||||
if events[0].Failure != domain.ExecutionFailureProviderStatus || events[1].RetryDelay < 0 {
|
||||
t.Fatalf("first=%+v delay=%+v", events[0], events[1])
|
||||
}
|
||||
if status == 200 && (!events[2].Usage.PromptTokens.Known || events[2].Usage.PromptTokens.Value != 0 || events[2].Usage.TotalTokens.Known) {
|
||||
t.Fatalf("usage=%+v", events[2].Usage)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestObservedInterruptedWaitAndCanceledCapacity(t *testing.T) {
|
||||
coordinator := testCoordinator(t, func(*http.Request) (*http.Response, error) { return response(503, `{"error":{}}`), nil })
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
var events []domain.ExecutionEvent
|
||||
ctx = recordEvents(t, ctx, &events)
|
||||
coordinator.wait = func(ctx context.Context, _ time.Duration) error { cancel(); return ctx.Err() }
|
||||
_, err := coordinator.Generate(ctx, testRequest(1))
|
||||
if !errors.Is(err, context.Canceled) || len(events) != 2 || events[1].Kind != domain.ExecutionRetryDelay || events[1].Failure != domain.ExecutionFailureCanceled {
|
||||
t.Fatalf("error=%v events=%+v", err, events)
|
||||
}
|
||||
|
||||
coordinator = testCoordinator(t, func(*http.Request) (*http.Response, error) { t.Fatal("unexpected dispatch"); return nil, nil })
|
||||
release, err := coordinator.manager.Acquire(context.Background(), "limited")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer release()
|
||||
ctx, cancel = context.WithCancel(context.Background())
|
||||
events = nil
|
||||
ctx = recordEvents(t, ctx, &events)
|
||||
entered := make(chan struct{})
|
||||
ctx = ¬ifyingContext{Context: ctx, entered: entered}
|
||||
done := make(chan error, 1)
|
||||
go func() { _, err := coordinator.Generate(ctx, testRequest(0)); done <- err }()
|
||||
<-entered
|
||||
cancel()
|
||||
if err := <-done; !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("error=%v", err)
|
||||
}
|
||||
if len(events) != 1 || events[0].Dispatched || events[0].Failure != domain.ExecutionFailureCanceled {
|
||||
t.Fatalf("events=%+v", events)
|
||||
}
|
||||
}
|
||||
|
||||
type notifyingContext struct {
|
||||
context.Context
|
||||
entered chan struct{}
|
||||
}
|
||||
|
||||
func (c *notifyingContext) Done() <-chan struct{} {
|
||||
select {
|
||||
case <-c.entered:
|
||||
default:
|
||||
close(c.entered)
|
||||
}
|
||||
return c.Context.Done()
|
||||
}
|
||||
|
||||
func TestObservedCallbackReleasesPermitAndCanCancelRetry(t *testing.T) {
|
||||
coordinator := testCoordinator(t, func(*http.Request) (*http.Response, error) { return response(503, `{"error":{}}`), nil })
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
var events []domain.ExecutionEvent
|
||||
recorder, err := observation.New(func(event domain.ExecutionEvent) {
|
||||
events = append(events, event)
|
||||
if event.Kind != domain.ExecutionAttempt {
|
||||
return
|
||||
}
|
||||
permitCtx, stop := context.WithTimeout(context.Background(), time.Second)
|
||||
defer stop()
|
||||
release, err := coordinator.manager.Acquire(permitCtx, "limited")
|
||||
if err != nil {
|
||||
t.Errorf("callback could not acquire permit: %v", err)
|
||||
} else {
|
||||
release()
|
||||
}
|
||||
cancel()
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = coordinator.Generate(observation.WithRecorder(ctx, recorder), testRequest(2))
|
||||
if !errors.Is(err, context.Canceled) || len(events) != 1 {
|
||||
t.Fatalf("error=%v events=%+v", err, events)
|
||||
}
|
||||
}
|
||||
|
||||
type fixedClient struct {
|
||||
response *domain.GenerateResponse
|
||||
err error
|
||||
}
|
||||
|
||||
func (c fixedClient) Generate(context.Context, domain.GenerateRequest) (*domain.GenerateResponse, error) {
|
||||
return c.response, c.err
|
||||
}
|
||||
|
||||
func TestObservedInjectedUsageAndRepairGeneration(t *testing.T) {
|
||||
manager, err := capacity.NewManager(nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var events []domain.ExecutionEvent
|
||||
ctx := recordEvents(t, context.Background(), &events)
|
||||
legacy := NewInjected(manager, fixedClient{response: &domain.GenerateResponse{Usage: domain.TokenUsage{PromptTokens: 3}}})
|
||||
if _, err := legacy.Generate(ctx, testRequest(4)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
presence := &domain.TokenUsagePresence{CompletionTokens: true}
|
||||
repair := NewInjected(manager, fixedClient{response: &domain.GenerateResponse{UsagePresence: presence}})
|
||||
req := testRequest(4)
|
||||
req.Generation = 2
|
||||
if _, err := repair.Generate(ctx, req); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(events) != 2 || !events[0].Usage.PromptTokens.Known || events[0].Usage.CompletionTokens.Known || !events[1].Usage.CompletionTokens.Known || events[1].Usage.CompletionTokens.Value != 0 || events[1].Usage.PromptTokens.Known {
|
||||
t.Fatalf("events=%+v", events)
|
||||
}
|
||||
if events[1].Generation != 2 || events[1].Attempt != 1 {
|
||||
t.Fatalf("repair indexes=%+v", events[1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestObservedSuccessfulGenerationBeforeRepairFailure(t *testing.T) {
|
||||
manager, err := capacity.NewManager(nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var events []domain.ExecutionEvent
|
||||
recorder, err := observation.New(func(event domain.ExecutionEvent) { events = append(events, event) })
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx := observation.WithRecorder(context.Background(), recorder)
|
||||
initial := NewInjected(manager, fixedClient{response: &domain.GenerateResponse{Usage: domain.TokenUsage{PromptTokens: 5}}})
|
||||
if _, err := initial.Generate(ctx, testRequest(0)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
repair := NewInjected(manager, fixedClient{err: errors.New("repair unavailable")})
|
||||
req := testRequest(0)
|
||||
req.Generation = 2
|
||||
if _, err := repair.Generate(ctx, req); err == nil {
|
||||
t.Fatal("expected repair failure")
|
||||
}
|
||||
recorder.Finish(observation.FinishData{Outcome: domain.ExecutionOutcomeError, Failure: domain.ExecutionFailureGeneration})
|
||||
if len(events) != 3 || events[0].Outcome != domain.ExecutionOutcomeSuccess || events[1].Outcome != domain.ExecutionOutcomeError || events[2].Attempts != 2 || events[2].RepairCalls != 1 {
|
||||
t.Fatalf("events=%+v", events)
|
||||
}
|
||||
if got := events[2].Usage.PromptTokens; got.Value != 5 || !got.Known || got.Complete {
|
||||
t.Fatalf("prompt usage=%+v", got)
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user