72 Commits

Author SHA1 Message Date
5ccfa4a345 Plan the codebase audit remediation 2026-08-11 20:10:14 +00:00
14e03f19d0 Consolidate and close the codebase audit 2026-08-11 17:21:29 +00:00
7c562a9374 Document cross-cutting architecture audit findings 2026-08-11 17:10:59 +00:00
ef97d85ac9 Document repository-wide test strategy audit 2026-08-11 17:00:59 +00:00
805e48c873 Document capacity scheduling audit results 2026-08-11 16:51:28 +00:00
e9e126dcba Document OpenAI transport audit findings 2026-08-11 16:42:55 +00:00
32e7a3557c Document prepared execution lifecycle audit 2026-08-11 16:31:05 +00:00
1d1b04e2e0 Record ordinary execution and repair audit findings 2026-08-11 16:22:09 +00:00
9748897751 Document inspection and target resolution audit findings 2026-08-11 16:09:50 +00:00
9d020039d5 Record output validation audit findings 2026-08-11 15:59:46 +00:00
5247ce0b73 Record artifact loading and prompt rendering audit findings 2026-08-11 15:41:57 +00:00
c434aa1dae Record profile source audit findings 2026-08-11 15:29:23 +00:00
ac9b3f3d80 Record prompt source audit findings 2026-08-11 15:18:17 +00:00
4f12a89a1b Record backend registry and defaults audit findings 2026-08-11 15:05:15 +00:00
df31e7f58e Record domain and JSON value audit findings 2026-08-11 14:54:56 +00:00
0678d242b9 Record engine operation audit findings 2026-08-11 14:43:20 +00:00
3b4ea21208 Record engine construction audit findings 2026-08-11 14:36:33 +00:00
1430e85147 Record configuration and adapter audit findings 2026-08-11 14:28:07 +00:00
34d7a19da5 Record public value and error audit findings 2026-08-11 14:17:55 +00:00
ebf1602635 Prepare the codebase audit plan 2026-08-11 14:05:30 +00:00
31f2ce3a09 Document Promptkit v0.5.0 2026-08-01 13:18:46 +00:00
fd06e4ca6b Clean up roadmap and fallback profile guidance 2026-08-01 13:16:26 +00:00
e63b8de1e9 Complete application fallback profile implementation 2026-08-01 12:39:01 +00:00
9354d2b373 Add application fallback profile sources 2026-08-01 12:37:01 +00:00
01ca5430bd Move profile composition to the engine facade 2026-08-01 12:31:50 +00:00
ae2179d103 Complete optional parameter omission 2026-08-01 02:43:02 +00:00
a248433d0f Omit unset optional request parameters 2026-08-01 02:41:44 +00:00
bd6cffc9d0 Prepare documentation for Promptkit v0.4.0 2026-07-30 23:48:25 +00:00
e40c4f182b Document structured capacity errors 2026-07-30 23:26:44 +00:00
7428e50c2c Expose structured capacity errors 2026-07-30 23:24:04 +00:00
63c67a4520 Add internal capacity error identity 2026-07-30 23:21:27 +00:00
25a7052a3d Clarify prompt input requirement documentation 2026-07-30 22:10:33 +00:00
fc3255967e Document prompt inspection API 2026-07-30 21:06:33 +00:00
e920168b30 Expose prompt inspection through the engine 2026-07-30 21:03:47 +00:00
272b6a4bc1 Add internal prompt inspection 2026-07-30 21:00:05 +00:00
dde48a31fc Document profile inspection API 2026-07-30 19:57:07 +00:00
242eace4a7 Expose profile inspection through the engine 2026-07-30 19:52:00 +00:00
0bf5f88136 Add internal profile inspection resolution 2026-07-30 19:48:09 +00:00
369ab5392d Add feature roadmap and implementation plan for profile inspection API 2026-07-30 19:42:51 +00:00
2ba0146e5d Tighten prepared execution credential handling 2026-07-30 19:06:17 +00:00
6112c2af0c Document prepared execution workflow 2026-07-30 18:27:03 +00:00
f5e12c00f5 Expose prepared execution handles 2026-07-30 18:20:35 +00:00
49fe402dd2 Add prepared execution lifecycle to the runner 2026-07-30 18:10:28 +00:00
c301eb8d55 Add frozen validation preparation plans 2026-07-30 17:59:45 +00:00
c13e9710d9 Add feature roadmap and implementation plan for downstream consumer wishlist items 2026-07-30 17:54:44 +00:00
87b5ec3d75 Organize downstream feature requests in the future roadmap 2026-07-30 17:13:26 +00:00
cb4028a637 Add feature roadmaps with wishlists from downstream consumers 2026-07-30 16:59:37 +00:00
5a1bff4529 Prepare the local backend convenience release 2026-07-30 04:01:19 +00:00
805a7f965d Document local backend configuration paths 2026-07-30 03:40:52 +00:00
147f5e5ff5 Add local backend convenience constructor 2026-07-30 03:38:23 +00:00
e361c97bb5 Document the v0.2.0 upgrade path 2026-07-29 23:22:38 +00:00
be67707582 Keep capacity cancellation tests independent of implementation details 2026-07-29 22:41:12 +00:00
e61ab700c7 Document backend capacity management 2026-07-29 21:16:49 +00:00
d2c4051dd0 Wire backend capacity into engine runtime 2026-07-29 21:11:08 +00:00
861da355d8 Add early backend run admission 2026-07-29 21:01:37 +00:00
a752f88166 Add backend capacity manager 2026-07-29 20:55:46 +00:00
238fa90bfa Add backend capacity policy configuration 2026-07-29 20:48:10 +00:00
ffe6d261a9 Add roadmap and implementation plan for backend concurrency scaling 2026-07-29 20:42:45 +00:00
dc39562ff7 Keep session IDs on repair requests and retire completed plans 2026-07-29 20:25:00 +00:00
0a839aa16d Document per-run session and reasoning overrides 2026-07-29 19:47:39 +00:00
f6ee18f6b3 Add direct per-run session overrides 2026-07-29 19:43:26 +00:00
eb8ab215e8 Add tri-state reasoning overrides 2026-07-29 19:36:48 +00:00
f89cb94ed2 Address backend implementation review findings 2026-07-29 19:08:05 +00:00
359b7313f4 Harden backend contracts and documentation 2026-07-29 17:22:55 +00:00
ae210b3c26 Add engine-scoped backend registration 2026-07-29 17:15:51 +00:00
810f80e7c9 Resolve profiles through built-in backends 2026-07-29 17:10:34 +00:00
8d00354c59 Add immutable backend registry foundation 2026-07-29 16:59:43 +00:00
d0010689f3 Prepare the backend registry implementation roadmap 2026-07-29 16:50:36 +00:00
b462153483 Document consumer safety and offline execution 2026-07-29 14:13:52 +00:00
086cf0fc86 Make GoDoc the public API contract 2026-07-29 14:06:54 +00:00
c1cecb1ee8 Resolve public documentation contract questions 2026-07-29 13:53:33 +00:00
bcb327f643 Document the published Promptkit release process 2026-07-28 19:31:32 +00:00
96 changed files with 17409 additions and 1008 deletions

View File

@@ -31,4 +31,23 @@ Contributors should start with the [development guide](docs/development.md).
The [architecture policy](docs/policy/architecture.md) defines the library
boundary and constraints that framework work must preserve.
## Release Guidance
Consumers upgrading from `v0.4.0` to `v0.5.0` should read the
[v0.5.0 changelog and migration guide](docs/releases/v0.5.0.md).
Consumers upgrading from `v0.3.0` to `v0.4.0` should read the
[v0.4.0 changelog and adoption guide](docs/releases/v0.4.0.md).
Consumers upgrading from `v0.2.0` to `v0.3.0` should read the
[v0.3.0 changelog](docs/releases/v0.3.0.md).
Consumers moving from `v0.1.0` to `v0.2.0` should read the
[v0.2.0 changelog and migration guide](docs/releases/v0.2.0.md).
## Related Project
[Scriptorium](https://gitea.maximumdirect.net/eric/scriptorium) is the CLI and
HTTP application built on Promptkit.
Promptkit is licensed under the [GNU General Public License version 3](LICENSE).

99
backends.go Normal file
View File

@@ -0,0 +1,99 @@
package promptkit
import (
"gitea.maximumdirect.net/eric/promptkit/internal/backend"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
)
// BackendOpenRouter is the reserved ID of Promptkit's built-in OpenRouter
// backend.
const BackendOpenRouter = backend.OpenRouterID
// BackendLocal is the case-sensitive conventional ID used by [LocalBackend].
// It is not a built-in or reserved backend and must be registered with
// [WithBackend].
const BackendLocal = "local"
// Backend configures one engine-scoped OpenAI-compatible backend.
//
// Backend has no stable JSON representation. Use keyed literals so additions
// to this configuration value do not break source compatibility.
type Backend struct {
// ID is the stable, case-sensitive registry key. NewEngine trims it and
// requires a non-blank value. BackendOpenRouter is reserved.
ID string
// Endpoint is the OpenAI-compatible base endpoint. NewEngine trims it and
// requires an absolute HTTP or HTTPS URL with a host and without user
// information, a query string, or a fragment. Paths are allowed.
Endpoint string
// APIKeyEnv optionally names the environment variable containing the API
// key. NewEngine trims it and requires the portable form
// [A-Za-z_][A-Za-z0-9_]*. Store only the name, never a credential value.
APIKeyEnv string
// ExtraParams contains backend-wide request defaults. Values must be
// JSON-compatible, finite, acyclic, and keyed by non-empty strings. Keys
// must not be model, session_id, messages, temperature, max_tokens, top_p,
// service_tier, reasoning_effort, or response_format. An empty map supplies
// no defaults. NewEngine deeply copies the map.
ExtraParams map[string]any
// ConcurrencyLimit is the maximum number of simultaneous model-generation
// calls allowed for this backend within one Engine. Zero leaves the backend
// unlimited. A negative value makes NewEngine fail with ErrInvalidConfig.
ConcurrencyLimit int
// QueueCapacity controls how many additional Run or RunPrepared calls may
// be admitted beyond ConcurrencyLimit. Nil uses 1024 when ConcurrencyLimit
// is positive; a pointer uses its exact value, including zero. The pointed-to
// value must be non-negative, and QueueCapacity must be nil when
// ConcurrencyLimit is zero. Their sum must fit in an int. WithBackend copies
// the value and does not retain the pointer.
QueueCapacity *int
}
// LocalBackend returns a caller-owned Backend for a conventional local
// OpenAI-compatible endpoint. It sets ID to BackendLocal and copies endpoint
// and concurrencyLimit into Endpoint and ConcurrencyLimit without
// normalization or validation. APIKeyEnv, ExtraParams, and QueueCapacity keep
// their zero values.
//
// LocalBackend does not read environment variables, register the value, or
// mutate engine or package state. Supply the returned value through
// [WithBackend]; [NewEngine] then applies the ordinary backend validation and
// concurrency semantics, including default queue capacity for a positive
// limit, unlimited behavior for zero, and ErrInvalidConfig for a negative
// limit.
func LocalBackend(endpoint string, concurrencyLimit int) Backend {
return Backend{
ID: BackendLocal,
Endpoint: endpoint,
ConcurrencyLimit: concurrencyLimit,
}
}
// WithBackend adds one Backend registration to the constructed Engine.
//
// Registrations accumulate in option order. Every normalized ID must be unique
// across consumer registrations and built-ins; a duplicate or invalid
// definition makes NewEngine fail with ErrInvalidConfig. In particular,
// BackendOpenRouter cannot be replaced. The immutable registration is scoped
// to the resulting Engine and cannot be enumerated, replaced, removed, or
// mutated after construction. WithBackend does not install package-global
// state.
func WithBackend(backend Backend) Option {
queueCapacity := 0
queueCapacitySet := backend.QueueCapacity != nil
if queueCapacitySet {
queueCapacity = *backend.QueueCapacity
}
return optionFunc(func(options *engineOptions) error {
options.backends = append(options.backends, domain.Backend{
ID: backend.ID,
Endpoint: backend.Endpoint,
APIKeyEnv: backend.APIKeyEnv,
ExtraParams: backend.ExtraParams,
ConcurrencyLimit: backend.ConcurrencyLimit,
QueueCapacity: queueCapacity,
QueueCapacitySet: queueCapacitySet,
})
return nil
})
}

409
capacity_contract_test.go Normal file
View File

@@ -0,0 +1,409 @@
package promptkit_test
import (
"context"
"errors"
"sync"
"testing"
"time"
"gitea.maximumdirect.net/eric/promptkit"
)
func TestEngineLimitsInjectedClientConcurrency(t *testing.T) {
release := make(chan struct{})
client := newCapacityGateClient(release, 8)
engine := newBackendCapacityEngine(t, client, 2, capacityInt(4), nil)
results := make(chan capacityRunResult, 6)
for i := 0; i < 6; i++ {
go runCapacityRequest(engine, context.Background(), promptkit.RunRequest{
PromptID: "prompt",
Execution: &promptkit.ExecutionTargetOverride{
Endpoint: "http://request.example/v1",
},
}, results)
}
first := awaitCapacityRequest(t, client.started)
second := awaitCapacityRequest(t, client.started)
if first.Target.BackendID != "limited" || second.Target.BackendID != "limited" {
t.Fatalf("endpoint override changed backend pool: first=%q second=%q",
first.Target.BackendID, second.Target.BackendID)
}
if active, peak, _ := client.snapshot(); active != 2 || peak != 2 {
t.Fatalf("client concurrency before release=(active=%d peak=%d), want 2", active, peak)
}
close(release)
for i := 0; i < 6; i++ {
outcome := awaitCapacityRun(t, results)
if outcome.err != nil || outcome.result == nil {
t.Fatalf("run outcome=(%+v, %v), want success", outcome.result, outcome.err)
}
}
if _, peak, calls := client.snapshot(); peak > 2 || calls != 6 {
t.Fatalf("client observations=(peak=%d calls=%d), want peak <= 2 and 6 calls", peak, calls)
}
}
func TestEngineRejectsRunBeforeCompletionWhenAdmissionIsFull(t *testing.T) {
artifactRelease := make(chan struct{})
reader := &capacityArtifactReader{
entered: make(chan struct{}, 2),
release: artifactRelease,
}
client := newCapacityGateClient(closedCapacityChannel(), 2)
engine := newBackendCapacityEngine(t, client, 1, capacityInt(0), reader)
firstResult := make(chan capacityRunResult, 1)
go runCapacityRequest(engine, context.Background(), capacityInputRequest("http://first.example/v1"), firstResult)
awaitCapacitySignal(t, reader.entered, "first artifact read")
canceledContext, cancel := context.WithCancel(context.Background())
cancel()
result, err := engine.Run(canceledContext, capacityInputRequest("http://canceled.example/v1"))
if result != nil || !errors.Is(err, context.Canceled) {
t.Fatalf("canceled capacity admission=(%+v, %v), want context cancellation", result, err)
}
var canceledCapacityErr *promptkit.CapacityError
if errors.Is(err, promptkit.ErrCapacityExceeded) || errors.As(err, &canceledCapacityErr) {
t.Fatalf("canceled admission exposed capacity rejection: %v", err)
}
result, err = engine.Run(context.Background(), capacityInputRequest("http://second.example/v1"))
if result != nil {
t.Fatalf("capacity rejection returned partial result: %+v", result)
}
if !errors.Is(err, promptkit.ErrCapacityExceeded) {
t.Fatalf("capacity rejection=%v, want ErrCapacityExceeded", err)
}
if errors.Is(err, promptkit.ErrInvalidRequest) || errors.Is(err, promptkit.ErrLLMGenerate) {
t.Fatalf("capacity rejection had an unrelated category: %v", err)
}
var capacityErr *promptkit.CapacityError
if !errors.As(err, &capacityErr) || capacityErr == nil {
t.Fatalf("capacity rejection=%v, want CapacityError", err)
}
if capacityErr.BackendID != "limited" {
t.Fatalf("capacity backend ID=%q, want limited", capacityErr.BackendID)
}
capacityErr.BackendID = "changed"
result, err = engine.Run(context.Background(), capacityInputRequest("http://third.example/v1"))
var subsequentCapacityErr *promptkit.CapacityError
if result != nil || !errors.As(err, &subsequentCapacityErr) ||
subsequentCapacityErr == nil || subsequentCapacityErr.BackendID != "limited" {
t.Fatalf("subsequent capacity rejection=(%+v, %v), want independent limited CapacityError", result, err)
}
if calls := reader.callCount(); calls != 1 {
t.Fatalf("artifact calls=%d, want only the admitted run", calls)
}
if _, _, calls := client.snapshot(); calls != 0 {
t.Fatalf("client calls=%d before admitted run was released, want 0", calls)
}
close(artifactRelease)
outcome := awaitCapacityRun(t, firstResult)
if outcome.err != nil || outcome.result == nil {
t.Fatalf("first run outcome=(%+v, %v), want success", outcome.result, outcome.err)
}
}
func TestBackendCapacityIsIndependentBetweenEngines(t *testing.T) {
firstRelease := make(chan struct{})
firstClient := newCapacityGateClient(firstRelease, 1)
firstEngine := newBackendCapacityEngine(t, firstClient, 1, capacityInt(0), nil)
secondClient := newCapacityGateClient(closedCapacityChannel(), 1)
secondEngine := newBackendCapacityEngine(t, secondClient, 1, capacityInt(0), nil)
firstResult := make(chan capacityRunResult, 1)
go runCapacityRequest(firstEngine, context.Background(), promptkit.RunRequest{PromptID: "prompt"}, firstResult)
awaitCapacityRequest(t, firstClient.started)
result, err := secondEngine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
if err != nil || result == nil {
t.Fatalf("second engine run=(%+v, %v), want independent success", result, err)
}
if _, _, calls := secondClient.snapshot(); calls != 1 {
t.Fatalf("second engine client calls=%d, want 1", calls)
}
close(firstRelease)
outcome := awaitCapacityRun(t, firstResult)
if outcome.err != nil || outcome.result == nil {
t.Fatalf("first engine run=(%+v, %v), want success", outcome.result, outcome.err)
}
}
func TestUnlimitedBackendsRetainInjectedClientConcurrency(t *testing.T) {
tests := []struct {
name string
configure func(*testing.T, promptkit.LLMClient) *promptkit.Engine
}{
{
name: "custom backend",
configure: func(t *testing.T, client promptkit.LLMClient) *promptkit.Engine {
return newBackendCapacityEngine(t, client, 0, nil, nil)
},
},
{
name: "endpoint-only profile",
configure: func(t *testing.T, client promptkit.LLMClient) *promptkit.Engine {
engine, err := promptkit.NewEngine(promptkit.Config{},
promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."),
promptkit.WithProfiles(promptkit.Profile{
ID: "profile", Endpoint: "http://endpoint.example/v1", Model: "model",
}),
promptkit.WithLLMClient(client),
)
if err != nil {
t.Fatalf("construct endpoint-only engine: %v", err)
}
return engine
},
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
release := make(chan struct{})
client := newCapacityGateClient(release, 2)
engine := tc.configure(t, client)
results := make(chan capacityRunResult, 2)
for i := 0; i < 2; i++ {
go runCapacityRequest(
engine,
context.Background(),
promptkit.RunRequest{PromptID: "prompt"},
results,
)
}
awaitCapacityRequest(t, client.started)
awaitCapacityRequest(t, client.started)
if active, peak, _ := client.snapshot(); active != 2 || peak != 2 {
t.Fatalf("unlimited concurrency=(active=%d peak=%d), want 2", active, peak)
}
close(release)
for i := 0; i < 2; i++ {
outcome := awaitCapacityRun(t, results)
if outcome.err != nil || outcome.result == nil {
t.Fatalf("run outcome=(%+v, %v), want success", outcome.result, outcome.err)
}
}
})
}
}
func TestCapacityExceededSentinelContract(t *testing.T) {
if promptkit.ErrCapacityExceeded == nil {
t.Fatal("ErrCapacityExceeded is nil")
}
var nilCapacityErr *promptkit.CapacityError
zeroCapacityErr := &promptkit.CapacityError{}
populatedCapacityErr := &promptkit.CapacityError{BackendID: "limited"}
for _, capacityErr := range []error{nilCapacityErr, zeroCapacityErr} {
if !errors.Is(capacityErr, promptkit.ErrCapacityExceeded) {
t.Fatalf("capacity error=%v, want ErrCapacityExceeded", capacityErr)
}
}
var discoveredCapacityErr *promptkit.CapacityError
if !errors.As(populatedCapacityErr, &discoveredCapacityErr) || discoveredCapacityErr != populatedCapacityErr {
t.Fatalf("populated capacity error is not discoverable: %v", populatedCapacityErr)
}
for _, unrelated := range []error{
promptkit.ErrInvalidConfig,
promptkit.ErrInvalidRequest,
promptkit.ErrLLMGenerate,
promptkit.ErrValidation,
} {
if errors.Is(promptkit.ErrCapacityExceeded, unrelated) ||
errors.Is(unrelated, promptkit.ErrCapacityExceeded) ||
errors.Is(populatedCapacityErr, unrelated) {
t.Fatalf("ErrCapacityExceeded aliases unrelated sentinel %v", unrelated)
}
}
}
type capacityRunResult struct {
result *promptkit.RunResult
err error
}
func runCapacityRequest(
engine *promptkit.Engine,
ctx context.Context,
request promptkit.RunRequest,
results chan<- capacityRunResult,
) {
result, err := engine.Run(ctx, request)
results <- capacityRunResult{result: result, err: err}
}
func newBackendCapacityEngine(
t *testing.T,
client promptkit.LLMClient,
limit int,
queueCapacity *int,
reader promptkit.ArtifactReader,
) *promptkit.Engine {
t.Helper()
promptFS := contractPromptFS("prompt", "profile", "message")
if reader != nil {
promptFS = contractInputPromptFS()
}
options := []promptkit.Option{
promptkit.WithPromptFS(promptFS, "."),
promptkit.WithBackend(promptkit.Backend{
ID: "limited",
Endpoint: "http://backend.example/v1",
ConcurrencyLimit: limit,
QueueCapacity: queueCapacity,
}),
promptkit.WithProfiles(promptkit.Profile{
ID: "profile", BackendID: "limited", Model: "model",
}),
promptkit.WithLLMClient(client),
}
if reader != nil {
options = append(options, promptkit.WithArtifactReader(reader))
}
engine, err := promptkit.NewEngine(promptkit.Config{}, options...)
if err != nil {
t.Fatalf("construct capacity engine: %v", err)
}
return engine
}
func capacityInputRequest(endpoint string) promptkit.RunRequest {
return promptkit.RunRequest{
PromptID: "input-prompt",
Inputs: map[string]promptkit.ArtifactRef{
"input": promptkit.Inline("input"),
},
Execution: &promptkit.ExecutionTargetOverride{Endpoint: endpoint},
}
}
type capacityGateClient struct {
mu sync.Mutex
active int
peak int
calls int
started chan promptkit.GenerateRequest
release <-chan struct{}
}
func newCapacityGateClient(release <-chan struct{}, buffer int) *capacityGateClient {
return &capacityGateClient{
started: make(chan promptkit.GenerateRequest, buffer),
release: release,
}
}
func (c *capacityGateClient) Generate(
ctx context.Context,
request promptkit.GenerateRequest,
) (*promptkit.GenerateResponse, error) {
c.mu.Lock()
c.calls++
c.active++
if c.active > c.peak {
c.peak = c.active
}
c.mu.Unlock()
defer func() {
c.mu.Lock()
c.active--
c.mu.Unlock()
}()
c.started <- request
select {
case <-c.release:
return &promptkit.GenerateResponse{Content: "ok"}, nil
case <-ctx.Done():
return nil, ctx.Err()
}
}
func (c *capacityGateClient) snapshot() (active, peak, calls int) {
c.mu.Lock()
defer c.mu.Unlock()
return c.active, c.peak, c.calls
}
type capacityArtifactReader struct {
mu sync.Mutex
calls int
entered chan struct{}
release <-chan struct{}
}
func (r *capacityArtifactReader) Read(
ctx context.Context,
_ promptkit.ArtifactRef,
) (*promptkit.Artifact, error) {
r.mu.Lock()
r.calls++
r.mu.Unlock()
r.entered <- struct{}{}
select {
case <-r.release:
return &promptkit.Artifact{Body: []byte("input")}, nil
case <-ctx.Done():
return nil, ctx.Err()
}
}
func (r *capacityArtifactReader) callCount() int {
r.mu.Lock()
defer r.mu.Unlock()
return r.calls
}
func awaitCapacityRequest(
t *testing.T,
requests <-chan promptkit.GenerateRequest,
) promptkit.GenerateRequest {
t.Helper()
select {
case request := <-requests:
return request
case <-time.After(5 * time.Second):
t.Fatal("timed out waiting for client invocation")
return promptkit.GenerateRequest{}
}
}
func awaitCapacityRun(t *testing.T, results <-chan capacityRunResult) capacityRunResult {
t.Helper()
select {
case result := <-results:
return result
case <-time.After(5 * time.Second):
t.Fatal("timed out waiting for Run")
return capacityRunResult{}
}
}
func awaitCapacitySignal(t *testing.T, signal <-chan struct{}, name string) {
t.Helper()
select {
case <-signal:
case <-time.After(5 * time.Second):
t.Fatalf("timed out waiting for %s", name)
}
}
func capacityInt(value int) *int {
return &value
}
func closedCapacityChannel() <-chan struct{} {
channel := make(chan struct{})
close(channel)
return channel
}

40
capacity_error.go Normal file
View File

@@ -0,0 +1,40 @@
package promptkit
import (
"fmt"
"strings"
)
// CapacityError reports bounded admission rejected for a selected backend.
//
// Engine-produced values identify only rejection at Promptkit's bounded
// [Engine.Run] or [Engine.RunPrepared] admission boundary. BackendID is the
// normalized registered backend ID used for routing and capacity; endpoint
// overrides do not change it. Every engine-produced value is nonnil and has a
// nonblank BackendID. Provider errors, active-generation waiting, and caller
// cancellation are not represented by this type.
//
// Callers own returned values and may mutate BackendID without affecting engine
// state or another error. CapacityError and its default Go encoding have no
// stable JSON contract. Consumer-constructed values do not establish that an
// engine rejected work.
type CapacityError struct {
// BackendID is the normalized registered backend ID whose admission was
// rejected.
BackendID string
}
// Error returns diagnostic wording that is not a parsing contract. It is safe
// to call on a nil receiver or a value with a blank BackendID.
func (e *CapacityError) Error() string {
if e == nil || strings.TrimSpace(e.BackendID) == "" {
return ErrCapacityExceeded.Error()
}
return fmt.Sprintf("backend %q admission: %v", e.BackendID, ErrCapacityExceeded)
}
// Unwrap returns ErrCapacityExceeded so errors.Is and errors.As can be used
// together. It is safe to call on a nil receiver or a zero value.
func (e *CapacityError) Unwrap() error {
return ErrCapacityExceeded
}

View File

@@ -4,6 +4,7 @@ import (
"reflect"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
"gitea.maximumdirect.net/eric/promptkit/internal/jsonvalue"
)
func toDomainRunRequest(req RunRequest) (domain.RunRequest, error) {
@@ -15,12 +16,12 @@ func toDomainRunRequest(req RunRequest) (domain.RunRequest, error) {
PromptID: req.PromptID,
PromptVersion: req.PromptVersion,
ProfileID: req.ProfileID,
SessionID: req.SessionID,
APIKey: req.APIKey,
Inputs: toDomainArtifactRefMap(req.Inputs),
Vars: copyStringMap(req.Vars),
Execution: execution,
Validation: toDomainOutputContractPtr(req.Validation),
Metadata: copyStringMap(req.Metadata),
}, nil
}
@@ -33,6 +34,7 @@ func fromDomainPreparedRun(prepared *domain.PreparedRun) *PreparedRun {
PromptVersion: prepared.PromptVersion,
PromptHash: prepared.PromptHash,
SelectedProfileID: prepared.SelectedProfileID,
SelectedBackendID: prepared.SelectedBackendID,
EffectiveModelParams: fromDomainExecutionTarget(prepared.EffectiveModelParams),
OutputContract: fromDomainOutputContract(prepared.OutputContract),
StructuredOutput: fromDomainStructuredOutputSpec(prepared.StructuredOutput),
@@ -58,8 +60,10 @@ func fromDomainRunResult(result *domain.RunResult) *RunResult {
PromptID: result.PromptID,
PromptVersion: result.PromptVersion,
PromptHash: result.PromptHash,
SessionID: result.SessionID,
RenderedPromptHash: result.RenderedPromptHash,
SelectedProfileID: result.SelectedProfileID,
SelectedBackendID: result.SelectedBackendID,
ModelName: result.ModelName,
Endpoint: result.Endpoint,
EffectiveModelParams: fromDomainExecutionTarget(result.EffectiveModelParams),
@@ -132,7 +136,7 @@ func toDomainExecutionTargetOverride(override *ExecutionTargetOverride) (*domain
if override == nil {
return nil, nil
}
extraParams, err := copyPublicJSONMap(override.ExtraParams)
extraParams, err := jsonvalue.CopyMap(override.ExtraParams)
if err != nil {
return nil, err
}
@@ -144,7 +148,7 @@ func toDomainExecutionTargetOverride(override *ExecutionTargetOverride) (*domain
TopP: copyFloat64Ptr(override.TopP),
TimeoutSeconds: copyIntPtr(override.TimeoutSeconds),
ServiceTier: override.ServiceTier,
ReasoningEffort: override.ReasoningEffort,
ReasoningEffort: copyStringPtr(override.ReasoningEffort),
APIKeyEnv: override.APIKeyEnv,
ExtraParams: extraParams,
}, nil
@@ -152,6 +156,7 @@ func toDomainExecutionTargetOverride(override *ExecutionTargetOverride) (*domain
func fromDomainExecutionTarget(target domain.ExecutionTarget) ExecutionTarget {
return ExecutionTarget{
BackendID: target.BackendID,
Endpoint: target.Endpoint,
Model: target.Model,
Temperature: target.Temperature,
@@ -165,6 +170,40 @@ func fromDomainExecutionTarget(target domain.ExecutionTarget) ExecutionTarget {
}
}
func fromDomainProfileInspection(inspection *domain.ProfileInspection) *ProfileInspection {
if inspection == nil {
return nil
}
return &ProfileInspection{
ProfileID: inspection.ProfileID,
EffectiveModelParams: fromDomainExecutionTarget(inspection.EffectiveModelParams),
APIKeyRequired: inspection.APIKeyRequired,
}
}
func fromDomainPromptInspection(inspection *domain.PromptInspection) *PromptInspection {
if inspection == nil {
return nil
}
inputs := make([]PromptInputDefinition, len(inspection.Inputs))
for i, input := range inspection.Inputs {
inputs[i] = PromptInputDefinition{
Name: input.Name,
Required: input.Required,
ContentType: input.ContentType,
Description: input.Description,
}
}
return &PromptInspection{
PromptID: inspection.PromptID,
PromptVersion: inspection.PromptVersion,
PromptHash: inspection.PromptHash,
DefaultProfileID: inspection.DefaultProfileID,
Inputs: inputs,
OutputContract: fromDomainOutputContract(inspection.OutputContract),
}
}
func fromDomainExecutionTargetPresence(presence domain.ExecutionTargetPresence) ExecutionTargetPresence {
return ExecutionTargetPresence{
Temperature: presence.Temperature,
@@ -397,6 +436,14 @@ func copyFloat64Ptr(src *float64) *float64 {
return &v
}
func copyStringPtr(src *string) *string {
if src == nil {
return nil
}
v := *src
return &v
}
func copyIntPtr(src *int) *int {
if src == nil {
return nil

64
doc.go
View File

@@ -1,8 +1,64 @@
// Package promptkit provides an embeddable engine for preparing and executing
// prompt-defined LLM workflows.
//
// Applications construct an Engine with NewEngine, select filesystem or
// in-memory definition sources with options, and use Prepare or Run to execute
// requests. Concrete repositories, validators, and outbound clients remain
// internal implementation details.
// Applications construct an [Engine] with [NewEngine], select filesystem or
// in-memory sources and optional engine-scoped [Backend] registrations, and
// call [Engine.InspectPrompt], [Engine.InspectProfile], [Engine.Prepare],
// [Engine.PrepareExecution], [Engine.Run], or [Engine.RunPrepared]. Concrete
// registries, repositories, validators, and the built-in OpenAI-compatible
// client remain internal implementation details.
//
// # Concurrency and ownership
//
// An Engine supports concurrent InspectPrompt, InspectProfile, Prepare,
// PrepareExecution, Run, and RunPrepared calls. Engine-local backend policies
// bound admitted Run and RunPrepared calls and model generations where
// configured, while different backend pools and unlimited backends continue
// independently. An injected [LLMClient] or [ArtifactReader] can therefore
// still receive concurrent calls and must be safe for that use.
//
// NewEngine copies in-memory profiles and backend definitions. Prepare,
// PrepareExecution, and Run copy request maps, slices, pointer values, and
// JSON-compatible extra parameters before using them. InspectPrompt and
// InspectProfile return copied inspection values. Returned values and values
// passed to extension interfaces are likewise isolated from engine state.
// Callers own those copies and may mutate them after the call that supplied or
// returned them. Returned structured errors are likewise caller-owned and may
// be mutated without affecting engine state or another error.
//
// # Security and sensitive data
//
// The default artifact reader treats [File] paths as caller-selected operating
// system paths. It does not restrict them to an application root or impose an
// inbound request-size policy. Promptkit is not an inbound request or
// untrusted-input security boundary. Applications must validate and restrict
// untrusted input before constructing a request, or install an [ArtifactReader]
// that enforces their filesystem, authorization, and size policies.
//
// Rendered messages, input and output [Artifact] bodies, [RunResult.RawOutput],
// and [ValidationResult.Errors] may contain sensitive data. Credential
// exclusion and redaction do not sanitize those values. Applications and
// injected collaborators are responsible for access control, retention,
// logging, and secret handling appropriate to their data.
//
// # JSON
//
// Stable JSON representations are provided for [PreparedRun], [RunResult],
// [Artifact], [ExecutionTarget], [OutputContract], [ValidationResult],
// [TokenUsage], [RenderedPrompt], [RenderedMessage], [CacheControl],
// [StructuredOutputSpec], [StructuredOutputJSONSpec], [GenerateRequest],
// [GenerateResponse], [ExecutionTargetPresence], and the string value types
// used by those values.
//
// Construction, inspection, handle, and error values, including [Config],
// [Backend], [RunRequest], [ArtifactRef], [ExecutionTargetOverride], [Profile],
// [OpenAICompatibleProfileConfig], [ProfileInspection],
// [PromptInputDefinition], [PromptInspection], [PreparedExecution], and
// [CapacityError], do not have stable JSON representations. Direct API keys
// are nevertheless excluded from JSON for every public value.
//
// JSON timestamps use time.Time's RFC 3339 encoding and are omitted when zero.
// PreparedRun and RunResult durations are encoded as integer milliseconds in
// duration_ms and omitted when zero. Run IDs and all exposed hashes are opaque:
// their spelling, length, character set, and algorithm are not API contracts.
package promptkit

View File

@@ -1,151 +1,431 @@
# Package `promptkit`
Import path:
## Purpose
This guide helps Go consumers assemble Promptkit and choose the main
preparation or execution workflow. The declarations and GoDoc in the
[root package](../../doc.go) own exact field, option, serialization,
concurrency, ownership, failure, and cancellation semantics. The
[framework format reference](../formats.md) owns prompt, profile, and schema
file contracts.
Import the package as:
```go
import "gitea.maximumdirect.net/eric/promptkit"
```
Package `promptkit` is the supported Go contract for in-process prompt
preparation and execution. The declarations and their GoDoc in the
[root package](../../doc.go) own the exact API; this guide explains how the
pieces are used together. The [framework format reference](../formats.md) owns
prompt, profile, and schema file contracts.
The following Go fragments are illustrative and omit surrounding package,
import, and error-handling code. Use the maintained examples for complete
programs.
## Engine Construction And Sources
## Construct An Engine
Construct an engine with [`NewEngine`, `Config`, and
`Option`](../../engine.go). `PromptDir` is required unless a prompt source
option is supplied. `ProfileDir` optionally overlays built-in profiles, and an
empty `SchemaDir` uses the current directory. `Timeout` is the transport-wide
safety cap for the built-in OpenAI-compatible client. An optional `HTTPClient`
is cloned; its positive timeout takes precedence.
Create an engine with
[`NewEngine`](../../engine.go). A directory-backed setup supplies a prompt
directory and may supply profile and schema directories:
Nil options are ignored. Invalid construction, including a nil injected client
or artifact reader, returns an error matching `ErrInvalidConfig`.
```go
engine, err := promptkit.NewEngine(promptkit.Config{
PromptDir: "prompts",
ProfileDir: "profiles",
SchemaDir: "schemas",
})
```
The [source options](../../engine.go) replace their matching directory source:
- `WithPromptFS` and `WithPromptFile` select prompt definitions;
- `WithProfileFS` and `WithProfileFile` overlay built-in profiles;
- `WithProfiles` adds in-memory profiles ahead of file and built-in profiles;
- `WithSchemaFS` and `WithSchemaFile` select JSON Schema documents;
- `WithLLMClient` replaces the built-in model client; and
- `WithArtifactReader` replaces the default reader for every input.
Source selection, path resolution, strict decoding, profile overlays, and
file-to-request precedence are defined in the
Options support single-file or `fs.FS` sources, in-memory profiles,
engine-scoped backends, and injected artifact or model clients. Consult the
[constructor and option GoDoc](../../engine.go) for composition, precedence,
validation, and default transport behavior. Source discovery, format
validation, and profile precedence are defined by the
[framework format reference](../formats.md).
Per-generation timeout values from profiles or requests are independent of
the transport cap and caller context. An explicit request value of zero
disables only the per-generation deadline. The
[outbound integration contract](../integrations/openai-compatible-chat.md#timeout-and-cancellation)
defines the complete timeout layering.
## Supply Embedded Application Defaults
## Preparation And Execution
Use `WithFallbackProfileFS` when an application packages profile definitions
that should apply unless an operator provides an ordinary configured profile
with the same ID. For example, an application can embed its defaults while
continuing to use `ProfileDir` for operator overrides:
[`Engine.Prepare` and `Engine.Run`](../../engine.go) accept the public
[`RunRequest`](../../types.go). `Prepare` resolves the prompt, profile, input
artifacts, validation contract, and rendered messages without calling an LLM.
`Run` performs the same preparation, calls the configured client, and validates
the generated content. The maintained
```go
//go:embed profiles/*.yaml
var applicationProfiles embed.FS
engine, err := promptkit.NewEngine(promptkit.Config{
PromptDir: "prompts",
ProfileDir: operatorProfileDir,
},
promptkit.WithFallbackProfileFS(applicationProfiles, "profiles"),
)
```
Keep application-owned profile IDs and definitions in the embedded source.
Use the ordinary configured profile source for operator overrides. Leave
`operatorProfileDir` empty when the operator did not configure an override
directory; a non-empty path names an authoritative higher-precedence source,
so an unavailable or unreadable directory is an error rather than a reason to
fall back. The
[framework format reference](../formats.md#source-and-profile-precedence)
owns the exact profile format and lookup order; the
[`WithFallbackProfileFS` GoDoc](../../engine.go) owns its option contract and
validation rules.
## Inspect A Prompt Before Preparation
Use [`Engine.InspectPrompt`](../../engine.go) to check one configured prompt's
declared inputs and output workflow without creating placeholder inputs or
resolving a profile:
```go
inspection, err := engine.InspectPrompt(ctx, "meeting.summary", "")
if err != nil {
return err
}
for _, input := range inspection.Inputs {
// Compare the declared input with application configuration.
}
```
Use this configuration-time boundary when the application needs only the
declared prompt interface. Use `InspectProfile` separately when it must also
check a configured profile. Use `Prepare` when it needs inputs, schemas, or
rendered messages, and use prepared execution when that work must remain tied
to later execution. The method's [GoDoc](../../engine.go) owns exact fields,
hash, ownership, and error semantics.
## Prepare Without Model Execution
[`Engine.Prepare`](../../engine.go) resolves the selected prompt and profile,
loads inputs and any structured-output schema, and renders messages without
calling a model client. Choose it when the prepared value is the final
inspection or persistence result and no later execution must be tied to that
exact snapshot:
```go
prepared, err := engine.Prepare(ctx, promptkit.RunRequest{
PromptID: "meeting.summary",
Inputs: map[string]promptkit.ArtifactRef{
"note": promptkit.Inline("Synthetic meeting notes"),
},
})
```
The maintained
[offline preparation example](../../examples/go-library/prepare/main.go)
provides a complete runnable workflow using a prompt file, in-memory profile,
and inline input.
shows a complete runnable setup with a prompt file, in-memory profile, and
inline input. Exact request requirements and prepared-result fields belong to
the [`RunRequest` and `PreparedRun` GoDoc](../../types.go).
[`PreparedRun` and `RunResult`](../../types.go) expose copied public values.
Preparation returns effective settings, hashes, rendered messages, selected
profile, structured-output information, and timing without resolved secrets or
model output. Execution adds the generated artifact and raw output, validation
state, model metadata, usage, run ID, and duration.
## Prepare Now And Execute The Same Snapshot Later
A generated-content validation failure returns a result with
`Validation.Status == ValidationFailed`. An inability to perform validation
returns an error matching `ErrValidation`.
Use [`Engine.PrepareExecution`](../../engine.go) when an application must
inspect or persist preflight details before deciding whether to start model
work, while ensuring that later execution uses those exact rendered messages,
inputs, target settings, and validation resources:
## Requests, Inputs, And Overrides
```go
preparedExecution, err := engine.PrepareExecution(ctx, promptkit.RunRequest{
PromptID: "meeting.summary",
Inputs: map[string]promptkit.ArtifactRef{
"note": promptkit.Inline("Synthetic meeting notes"),
},
})
if err != nil {
return err
}
defer preparedExecution.Discard()
The [request and value declarations](../../types.go) own the available fields,
serialized constants, and result shapes. Use `File`, `Inline`, or
`InlineWithURI` to construct artifact references. The
[framework format reference](../formats.md) defines declared inputs, template
references, output contracts, and the relationship between file values and
request overrides.
details := preparedExecution.Details()
// Inspect or persist an application-selected safe subset of details.
`ExecutionTargetOverride` uses pointers for numeric settings so an explicit
zero remains distinct from no override. `ExtraParams` accepts JSON-compatible
strings, booleans, finite numbers, string-keyed objects, arrays or slices, and
nil. Unsupported values, non-string map keys, non-finite numbers, and cycles
match `ErrInvalidConfig` in profiles or `ErrInvalidRequest` in request
overrides.
result, err := engine.RunPrepared(ctx, preparedExecution)
```
Returned requests, profiles, prepared values, results, artifacts, maps, and
slices are isolated from internal engine state. Consumers and injected
extensions should not retain or mutate values owned by another caller.
Preparation does not call the model or reserve backend capacity.
`RunPrepared` executes from the retained snapshot rather than reloading
consumer sources. The handle is opaque in-process state, while `Details`
contains rendered content and remains subject to the application's data
handling policy. The
[`PreparedExecution` and method GoDoc](../../prepared_execution.go) and
[engine operation GoDoc](../../engine.go) own exact lifecycle, engine-binding,
credential, cancellation, timing, and error semantics.
## Profiles And Credentials
## Execute And Validate
[`OpenAICompatibleProfile`](../../profiles.go) constructs an ordinary
in-memory profile for an OpenAI-compatible chat-completions endpoint.
`WithProfiles` rejects duplicate IDs in one call and gives in-memory profiles
precedence over explicit file sources and built-ins.
[`Engine.Run`](../../engine.go) performs the same preparation, invokes the
configured model client, classifies the generated artifact, and validates the
content in one call. Choose it when the application does not need a preflight
boundary tied to the eventual execution. A completed content check may return
`ValidationFailed` in the result; an operational inability to validate returns
an error.
Raw API keys do not belong in profiles. File-backed profiles may name an
environment variable, while an in-memory profile can require a request key.
A direct `RunRequest.APIKey` is request-scoped and takes precedence over an
environment lookup for the built-in client. Profile fields, ranges, built-ins,
precedence, and credential rules are owned by the
[framework format reference](../formats.md).
The maintained
[offline execution example](../../examples/go-library/run/main.go) injects a
deterministic model client and exercises `Run` without credentials, network
access, or paid calls. It is intentionally separate from the preparation
example so each workflow and its small prompt fixture can be copied and run on
its own.
API keys are excluded from JSON, prepared values, and results. The public
`String` and `GoString` methods report only whether a direct key is present.
Avoid reflection-based dumps of request structs, which can bypass that
redaction.
Use the [`RunResult` and `ValidationResult` GoDoc](../../types.go) for the
returned data and the `Engine.Run` GoDoc for failure and cancellation
semantics. The
[OpenAI-compatible integration contract](../integrations/openai-compatible-chat.md)
owns the built-in client's outbound HTTP behavior.
## Inputs, Profiles, And Overrides
Use `File`, `Inline`, or `InlineWithURI` to construct request inputs. A request
can select a profile explicitly or use the prompt's default profile, and can
replace execution settings or the complete output contract.
The [public value GoDoc](../../types.go) defines nil, empty, zero, replacement,
copy, and credential behavior. The
[framework format reference](../formats.md) defines how those request values
interact with prompt definitions, file-backed and application fallback
profiles, built-ins, schemas, and framework defaults.
For programmatic profiles,
[`OpenAICompatibleProfile`](../../profiles.go) converts ordinary
OpenAI-compatible settings into a value accepted by `WithProfiles`.
### Inspect A Profile Before Prompt Work
Use [`Engine.InspectProfile`](../../engine.go) to validate one configured
profile without constructing a synthetic prompt or placeholder inputs. It
resolves the profile's effective target but does not prepare or execute a
prompt:
```go
inspection, err := engine.InspectProfile(ctx, profileID)
if err != nil {
return err
}
target := inspection.EffectiveModelParams
if target.APIKeyEnv != "" {
// Apply application policy for the named environment variable.
} else if inspection.APIKeyRequired {
// Arrange a direct credential before later execution.
}
```
Use this configuration-time boundary when only the profile and its target need
checking. Use `Prepare` when the application also needs prompt, input, schema,
or rendering work; use prepared execution when that work must remain tied to a
later execution. Inspection reports credential requirements but leaves the
timing of credential enforcement to the application. The method's
[GoDoc](../../engine.go) owns its exact result and error contract.
### Set A Per-Run Session And Reasoning
Supply a direct session ID when one prompt should be correlated with a
consumer-managed conversation or workflow without changing prompt variables:
```go
reasoning := "high"
result, err := engine.Run(ctx, promptkit.RunRequest{
PromptID: "meeting.summary",
SessionID: "conversation-42",
Inputs: map[string]promptkit.ArtifactRef{
"note": promptkit.Inline("Synthetic meeting notes"),
},
Execution: &promptkit.ExecutionTargetOverride{
ReasoningEffort: &reasoning,
},
})
```
A nil reasoning pointer inherits the selected profile, a pointer to a
nonblank string replaces it, and a pointer to a blank string disables
reasoning for that run. Session IDs are correlation metadata, not credentials;
use stable, non-secret values that are safe to expose to collaborators and
providers. The
[`RunRequest` and `ExecutionTargetOverride` GoDoc](../../types.go) owns the
exact normalization, precedence, error, copying, and exposure contract.
### Configure A Local OpenAI-Compatible Endpoint
Choose the smallest configuration that fits how the endpoint will be reused.
#### Use An Endpoint-Only Profile
Put the endpoint directly on an in-memory profile when only that profile needs
it and shared backend identity or capacity policy is unnecessary:
```go
engine, err := promptkit.NewEngine(promptkit.Config{
PromptDir: "prompts",
},
promptkit.WithProfiles(promptkit.Profile{
ID: "local-summary",
Endpoint: "http://localhost:8000/v1",
Model: "example-model",
}),
)
```
Endpoint-only profiles have an empty backend ID and remain unrestricted by
backend capacity policy.
#### Use The Conventional Local Backend
Use `LocalBackend` when profiles should share the conventional `local`
identity, endpoint, and concurrency limit:
```go
engine, err := promptkit.NewEngine(promptkit.Config{
PromptDir: "prompts",
},
promptkit.WithBackend(
promptkit.LocalBackend("http://localhost:8000/v1", 2),
),
promptkit.WithProfiles(promptkit.Profile{
ID: "local-summary",
BackendID: promptkit.BackendLocal,
Model: "example-model",
}),
)
```
The helper is explicit: it does not pre-register a backend or read environment
variables. Supplying a positive limit leaves queue capacity omitted, so normal
backend registration selects the existing default waiting capacity of 1024.
The returned value still enters the engine through `WithBackend`.
#### Configure A Complete Backend
Use a keyed `Backend` value for authentication, extra request parameters, an
explicit queue capacity, a custom ID, or multiple local endpoints:
```go
noWaiting := 0
engine, err := promptkit.NewEngine(promptkit.Config{
PromptDir: "prompts",
},
promptkit.WithBackend(promptkit.Backend{
ID: "local-gpu",
Endpoint: "http://gpu-host:8000/v1",
APIKeyEnv: "LOCAL_GPU_API_KEY",
ExtraParams: map[string]any{"provider_option": "enabled"},
ConcurrencyLimit: 2,
QueueCapacity: &noWaiting,
}),
promptkit.WithProfiles(promptkit.Profile{
ID: "gpu-summary",
BackendID: "local-gpu",
Model: "example-model",
}),
)
```
Use distinct custom IDs when registering multiple local endpoints.
Registrations belong to one engine and custom IDs cannot replace built-ins.
The [`Backend`, `LocalBackend`, and `WithBackend` GoDoc](../../backends.go)
defines exact construction, validation, copying, uniqueness, concurrency, and
request-default behavior.
Both file-backed and in-memory profiles select a registration through
`backend` or `Profile.BackendID`. Profile and request endpoint overrides retain
that routing and capacity identity. `PreparedRun.SelectedBackendID`,
`RunResult.SelectedBackendID`, and the effective `ExecutionTarget.BackendID`
expose it to consumers and injected model clients. Endpoint-only profiles
remain supported and expose an empty backend ID.
### Limit Backend Concurrency
Set `Backend.ConcurrencyLimit` when a backend needs protection from too many
simultaneous model calls. Leaving `QueueCapacity` nil, as in the local-backend
example above, selects the default waiting capacity of 1024.
To accept no waiting backlog beyond the active calls, provide an explicit
zero:
```go
noWaiting := 0
backend := promptkit.Backend{
ID: "local-gpu",
Endpoint: "http://gpu-host:8000/v1",
ConcurrencyLimit: 2,
QueueCapacity: &noWaiting,
}
```
The pointer distinguishes an explicit zero from omission. Keep using keyed
`Backend` literals so additive configuration fields remain source-compatible.
Capacity belongs to one engine and the selected backend ID; endpoint-only
profiles and custom backends without a configured limit remain unrestricted.
Exact validation, defaulting, ownership, and concurrency semantics belong to
the [`Backend` GoDoc](../../backends.go).
## Credentials
File-backed profiles name an environment variable; in-memory profiles can
require a direct request key. Direct keys are request-scoped and are excluded
from supported JSON values and the package's `String` and `GoString`
summaries. The exact precedence and redaction guarantees belong to
[`RunRequest`, `GenerateRequest`, and the profile GoDoc](../../types.go).
## Protect Files And Generated Data
The default artifact reader opens a `File` reference as a caller-selected
operating-system path. It does not constrain paths to an application root,
impose an inbound request-size policy, or establish an untrusted-input security
boundary. Applications must validate and restrict untrusted paths and payloads
before constructing a request, or inject an artifact reader that enforces
their filesystem, authorization, and size policies.
Rendered messages, input and output artifact bodies, raw model output, and
validation diagnostics can contain sensitive data. API-key redaction does not
sanitize those values. Treat prepared values, results, collaborator requests,
errors, and logs according to the application's data-access, retention, and
secret-handling policies.
## Extension Interfaces
The [`LLMClient`, `GenerateRequest`, and
`GenerateResponse`](../../types.go) boundary lets a consumer replace model
generation. Injected clients receive copied rendered messages, effective
settings, explicit numeric-setting presence, structured-output constraints,
and the request-scoped key. They return generated content and token usage.
Inject an [`LLMClient` or `ArtifactReader`](../../types.go) when the built-in
behavior does not fit the application. Their GoDoc defines concurrent use,
context handling, ownership of copied values, nil responses, and preservation
of collaborator errors. Implementations must honor cancellation, safely manage
copies they retain, avoid unsafe logging of content or credentials, and enforce
the application policy that motivated the injection.
The [`ArtifactReader`](../../types.go) boundary replaces the default inline and
file reader for every input. Readers provide artifact content and metadata; the
engine fills an empty artifact name from the input-map key. A reader error
matches `ErrArtifactLoad` while preserving the original identity for
`errors.Is`. A nil artifact with a nil error is also an artifact-load failure.
## Handle Errors
Extensions should honor context cancellation and avoid logging raw prompts,
artifacts, or credentials.
Use `errors.Is` with the
[public error sentinels and operation GoDoc](../../engine.go). The declarations
distinguish invalid construction, invalid requests, absent sources,
source-loading failures, collaborator failures, and operational validation
failures. Specific request conditions may also match the broader
`ErrInvalidRequest`, and injected collaborator identities are preserved where
documented. Invalid or duplicate backend registrations match
`ErrInvalidConfig`; selecting an unknown backend matches `ErrProfileLoad`.
## Errors
When a limited backend has admitted all active and waiting calls, handle
`ErrCapacityExceeded` separately from request errors and provider failures:
The [public error declarations](../../engine.go) and
[mapping](../../errors.go) preserve these sentinel checks through `errors.Is`:
```go
result, err := engine.Run(ctx, request)
if errors.Is(err, promptkit.ErrCapacityExceeded) {
var capacityErr *promptkit.CapacityError
if errors.As(err, &capacityErr) {
// Record capacityErr.BackendID using application-owned diagnostics.
}
- `ErrInvalidConfig`
- `ErrInvalidRequest`
- `ErrPromptNotFound`
- `ErrProfileNotFound`
- `ErrProfileRequired`
- `ErrPromptLoad`
- `ErrProfileLoad`
- `ErrAPIKeyEnvMissing`
- `ErrArtifactLoad`
- `ErrPromptRender`
- `ErrLLMGenerate`
- `ErrValidation`
// Apply application policy: shed work, report overload, or retry later.
}
```
`ErrProfileRequired` and `ErrAPIKeyEnvMissing` also match
`ErrInvalidRequest`, allowing either broad request handling or a specific
condition. Wrapped collaborator errors retain their identity where the public
contract promises it.
A rejected call returns no partial result and does not invoke the model
client. Promptkit does not prescribe retries or map this error to an HTTP
status; those choices remain with the consuming application. The
[`CapacityError` GoDoc](../../capacity_error.go) owns the exact typed-error
contract, while the [`Engine.Run` and error GoDoc](../../engine.go) owns broad
error and cancellation identities.
## Consumer Boundary
## Application Boundary
Promptkit is an importable library. It does not own a command, inbound HTTP
API, process configuration, or deployment policy. Scriptorium is one
downstream application that maps this root package contract into those
application concerns.
API, process configuration, or deployment policy. Applications map the root
package's results and errors into those concerns, including inbound size and
trust policy.

View File

@@ -44,98 +44,3 @@ Start with:
For cross-cutting changes, follow every applicable row. Do not create
placeholder documents for packages, APIs, or integrations that do not yet
exist.
## Maintainer-Run Validation
Promptkit does not currently use hosted CI. Maintainers are responsible for
running the documented checks before accepting changes. Run the default Go
validation from the Promptkit repository root:
```sh
go test ./...
go test -race ./...
go vet ./...
go build ./...
go run ./examples/go-library/prepare
```
Check formatting across every tracked Go file:
```sh
gofmt -l $(git ls-files '*.go')
```
The formatting command must produce no paths. Follow every added or changed
Markdown link and confirm its target exists. Finally, check whitespace:
```sh
git diff --check
```
Documentation-only work does not require unrelated new tests, but it still
requires link validation and `git diff --check`. Run the Go validation whenever
documentation changes commands, examples, generated output, or another
behavior checked by the module.
## Focused Validation
Use focused checks while iterating, then run the complete validation sequence
before accepting the change. The root package supports:
```sh
go test .
go vet .
go build .
```
Filter tests by name without assuming a fixed internal package layout:
```sh
go test ./... -run 'TestName'
```
Replace `TestName` with a useful regular expression. Target only paths that
exist, and consult the internal component overview for their owning
documentation. A filtered or package-specific run does not replace the
complete repository validation.
## Coordinated Work With Scriptorium
Promptkit and Scriptorium must remain independently valid. For temporary local
integration, use either a Go workspace outside both repositories or an
uncommitted replacement in the consuming module.
If the repositories are sibling directories, run the workspace commands from
their parent directory:
```sh
go work init ./promptkit ./scriptorium
go work sync
```
Use the workspace only for coordinated local checks. From the same parent
directory, remove it when finished:
```sh
rm -f go.work go.work.sum
```
Alternatively, from the Scriptorium repository root, temporarily point its
Promptkit dependency at the sibling checkout:
```sh
go mod edit -replace gitea.maximumdirect.net/eric/promptkit=../promptkit
```
After coordinated checks, remove the replacement and reconcile module
metadata:
```sh
go mod edit -dropreplace gitea.maximumdirect.net/eric/promptkit
go mod tidy
```
Never commit `go.work`, `go.work.sum`, or a local filesystem `replace`
directive. Before committing in either repository, inspect its module files and
working tree independently. Published consumer versions must depend on a tagged
Promptkit version, not a workspace, local replacement, or unpublished commit.

View File

@@ -58,6 +58,11 @@ When a request omits a version, the selected prompt ID must identify exactly
one definition. When it supplies a version, the ID and version pair must be
unique.
Exact prompt inspection uses this same configured source, strict decoding,
referenced content-file resolution, and ID/version selection. It reports the
selected definition's declared metadata without changing the prompt format or
executing the definition.
### Inputs
Each `inputs` item has these fields:
@@ -91,7 +96,8 @@ of a named input. Missing variables and input references are errors.
The optional `session_id` uses the same template data and input helper. Its
rendered value is trimmed, omitted when empty, and limited to 256 Unicode code
points.
points. A nonblank direct request session ID bypasses this template completely;
a blank direct value leaves the template behavior unchanged.
### Cache Control
@@ -136,7 +142,7 @@ A profile supplies model execution settings:
```yaml
id: local-summary
endpoint: http://localhost:8000/v1
backend: openrouter
model: example-model
temperature: 0.2
max_tokens: 500
@@ -144,7 +150,6 @@ top_p: 0.95
timeout_seconds: 90
service_tier: flex
reasoning_effort: medium
api_key_env: EXAMPLE_API_KEY
extra_params:
provider_option: enabled
```
@@ -152,7 +157,8 @@ extra_params:
| Field | Required | Meaning |
| --- | --- | --- |
| `id` | yes | Non-empty profile identifier. IDs must be unique within one source. |
| `endpoint` | yes | Non-empty OpenAI-compatible base URL, including an API version path when required. |
| `backend` | unless `endpoint` is present | Backend registry ID. It is trimmed and registry membership is checked when the profile is prepared or inspected. |
| `endpoint` | unless `backend` is present | Non-empty OpenAI-compatible base URL, including an API version path when required. When both connection fields are present, this overrides the backend endpoint without changing backend identity. |
| `model` | yes | Non-empty provider model name. |
| `temperature` | no | Number from 0 through 2. |
| `max_tokens` | no | Integer zero or greater. |
@@ -166,6 +172,13 @@ extra_params:
Raw `api_key` is prohibited in profile YAML. Store only an environment
variable name in `api_key_env`.
Promptkit does not infer a backend from a model or endpoint. Endpoint-only
profiles remain supported and have no effective backend ID.
The engine always provides the built-in `openrouter` ID. Consumers can add
engine-scoped IDs with
[`WithBackend`](../backends.go); exact registration validation belongs to its
GoDoc.
`extra_params` accepts null, booleans, finite numbers, strings, arrays, and
objects with string keys. Keys must be non-empty. With the built-in client,
they also cannot collide with the standard fields listed in the
@@ -175,51 +188,72 @@ they also cannot collide with the standard fields listed in the
Execution settings resolve in this order:
1. framework defaults;
2. the selected profile; and
3. request `ExecutionTargetOverride` values.
1. the framework timeout baseline;
2. the selected backend, when the profile names one;
3. the selected profile; and
4. request `ExecutionTargetOverride` values.
The framework defaults are:
The framework baseline is:
| Setting | Default |
| --- | --- |
| `temperature` | `0` |
| `max_tokens` | `0` |
| `top_p` | `1` |
| `temperature` | Unspecified and omitted from compatible provider requests unless a profile or runtime override selects it. |
| `max_tokens` | Unspecified and omitted from compatible provider requests unless a profile or runtime override selects it. |
| `top_p` | Unspecified and omitted from compatible provider requests unless a profile or runtime override selects it. |
| `timeout_seconds` | `600` |
Numeric zero in a file or in-memory profile means that the profile does not
replace the framework default. Numeric request overrides use pointers, so an
explicit zero is preserved. In particular, an explicit request
`timeout_seconds` of zero disables the per-generation deadline while leaving
the caller context and transport timeout intact.
Numeric zero in a file or in-memory profile does not select a numeric value.
For `temperature`, `max_tokens`, and `top_p`, it leaves the provider control
unspecified. For `timeout_seconds`, it retains the framework deadline. Numeric
request overrides use pointers, so an explicit zero is retained and sent to
compatible providers. In particular, an explicit request `timeout_seconds` of
zero disables the per-generation deadline while leaving the caller context and
transport timeout intact.
Non-empty request strings replace profile strings. A non-empty request
`ExtraParams` map replaces the profile map rather than merging keys.
Non-empty profile strings replace backend defaults, and non-empty request
strings replace both. Request reasoning is the exception: a nil
`ReasoningEffort` pointer inherits the profile, a pointer to a nonblank string
trims and replaces it, and a pointer to a blank string clears it. Backend
identity is retained when either layer overrides the endpoint, so the override
also retains any engine-local capacity policy configured for that backend.
Capacity configuration belongs to the Go
[`Backend` API](../backends.go), not prompt or profile YAML. A non-empty
`extra_params` map at each layer replaces the entire lower-precedence map
rather than merging keys.
The [outbound integration contract](integrations/openai-compatible-chat.md)
defines how the effective settings are serialized.
### Source And Profile Precedence
An explicit request profile ID takes precedence over the prompt's
`default_profile`. If neither is present, preparation fails.
`default_profile`. If neither is present, preparation fails. Exact profile
inspection instead takes one explicit profile ID and does not use a prompt
default.
Profile sources resolve matching IDs in this order:
1. in-memory profiles supplied with `WithProfiles`;
2. a profile file, `fs.FS`, or configured profile directory; and
3. embedded built-in profiles.
2. the ordinary configured source selected by a profile file, `fs.FS`, or
configured profile directory;
3. application fallback profiles supplied with `WithFallbackProfileFS`; and
4. embedded built-in profiles.
A higher-precedence source falls back only when the profile is absent. An
invalid matching profile is an error and does not fall back. In-memory
`Profile` values follow the same ranges as YAML profiles. They use
`APIKeyRequired` for request-scoped credentials instead of `api_key_env`.
A profile source supplies a complete definition; definitions and their fields
are not merged across sources. A higher-precedence source falls back only when
the requested profile ID is absent. An invalid matching profile is an error and
does not fall back. In-memory `Profile` values follow the same ranges as YAML
profiles. They use `APIKeyRequired` for request-scoped credentials instead of
`api_key_env`. Preparation and exact profile inspection use this same source
precedence.
## Built-In Profile Catalog
Built-ins use the OpenRouter-compatible endpoint and
`OPENROUTER_API_KEY`. A custom or in-memory profile with the same ID takes
precedence.
Every built-in selects the `openrouter` backend. The engine's built-in backend
registry supplies `https://openrouter.ai/api/v1` and the environment-variable
name `OPENROUTER_API_KEY`, so individual profiles contain only model and
generation settings. Built-in profile files do not repeat those connection
values. A configured, application fallback, or in-memory profile with the same
profile ID takes precedence.
| Provider | ID | Model |
| --- | --- | --- |
@@ -270,6 +304,10 @@ prompt, profile, schema, or example files:
- a request can provide a direct `APIKey` or override `APIKeyEnv`; and
- a direct request key takes precedence over environment lookup.
After a direct request key, the credential-source precedence is request
`APIKeyEnv`, profile `api_key_env`, then the backend default. An in-memory
profile with `APIKeyRequired` clears an inherited backend environment name and
requires a direct key unless the request explicitly supplies `APIKeyEnv`.
Promptkit validates required credential availability during preparation.
Direct keys are excluded from JSON results and redacted by public string
formatters. Environment-variable names may appear in prepared metadata, but

View File

@@ -13,10 +13,15 @@ that produce these outbound settings.
## Endpoint And Method
Generation sends an HTTP `POST` with `Content-Type: application/json`.
A non-empty endpoint from the execution target overrides the client's
configured base URL. After trailing slashes are removed,
`/chat/completions` is appended. Generation fails before sending when neither
source supplies an endpoint.
Before the client is called, the engine resolves framework, backend, profile,
and request values into one execution target. A non-empty endpoint from that
target overrides the client's configured base URL. After trailing slashes are
removed, `/chat/completions` is appended. Generation fails before sending when
neither source supplies an endpoint.
The target's backend ID is routing metadata for prepared values, results, and
injected clients. The built-in client does not derive the URL from that ID and
does not serialize it in the provider request.
## Authentication
@@ -26,6 +31,12 @@ the client reads that variable and requires a non-empty value. The selected
key is sent as `Authorization: Bearer <key>`. No authorization header is sent
when neither mechanism is configured.
The target contains the already resolved environment-variable name: an
explicit request override takes precedence over profile metadata, which takes
precedence over the backend default. Only the name reaches prepared metadata;
the environment value is read just before the provider call and is never added
to the JSON body.
## Request Body
The request body always contains `model` and `messages`. The execution
@@ -36,20 +47,25 @@ Each ordinary message contains its `role` and string `content`. A
cache-controlled message instead uses a text content block containing `type`,
`text`, and `cache_control`; an empty cache-control TTL is omitted.
A non-empty session ID is trimmed, checked against the internal domain limit,
and sent as top-level `session_id`. It is not sent as a session header.
The effective direct or prompt-rendered session ID is trimmed, limited to 256
Unicode code points, and sent when nonempty as top-level `session_id`. It is
never also sent as a session header.
The client conditionally includes:
- `temperature`, `max_tokens`, and `top_p` when non-zero or explicitly
present;
- non-empty `service_tier` and `reasoning_effort`; and
- `temperature`, `max_tokens`, and `top_p` only when selected by a profile or
runtime override, including an explicit runtime zero; they are absent when
unspecified;
- non-empty `service_tier` and effective `reasoning_effort`; an explicitly
disabled reasoning setting is empty and therefore omitted; and
- `response_format` for JSON Schema structured output, including its name,
strict flag, and schema document.
Extra parameters are merged directly into the top-level body after JSON
serialization is verified. Empty keys and collisions with these reserved
fields are rejected before any provider call:
The engine resolves backend, profile, and request extra-parameter maps by
whole-map replacement rather than key merging. The resulting effective map is
then merged directly into the top-level body after JSON serialization is
verified. Empty keys and collisions with these reserved fields are rejected
before any provider call:
- `model`
- `session_id`
@@ -61,6 +77,9 @@ fields are rejected before any provider call:
- `reasoning_effort`
- `response_format`
`backend_id`, `api_key_env`, and resolved credential values are not provider
request fields.
## Response Handling
Any 2xx response is decoded as an OpenAI-compatible chat response. The client

113
docs/internal/capacity.md Normal file
View File

@@ -0,0 +1,113 @@
# Internal Capacity Management
## Purpose
This document describes the implemented engine-local capacity coordination in
`internal/capacity`. The [architecture policy](../policy/architecture.md) owns
component boundaries, the [backend GoDoc](../../backends.go) owns exact public
configuration semantics, and the
[internal runner document](runner.md) owns orchestration around admission.
Capacity scheduling is outside the provider wire contract. It does not add
fields to execution targets, generated requests, prompt or profile YAML, or
stable JSON values.
## Construction And Pool Lifecycle
Each root `NewEngine` call obtains a normalized capacity-policy snapshot from
its immutable backend registry and constructs a new `Manager`. The manager
creates one pool for each limited backend ID. It has no package-global mutable
state, background workers, shutdown protocol, or persistence, so engines with
the same registrations still have independent capacity.
Unlimited registered backends and endpoint-only profiles have no pool. Their
admission and generation calls take the unrestricted fast path. An endpoint
override does not change the selected backend ID and therefore does not change
the pool.
One pool owns immutable active and total limits plus mutex-protected admission
count, active count, and ordered waiter list. Pool state exists only for the
lifetime of its engine.
## Bounded Execution Admission
For ordinary `Run`, the runner asks the manager to admit after resolving the
prompt, profile, selected backend, effective execution target, credentials, and
output contract, but before schema loading, artifact loading, or rendering.
`PrepareExecution` performs no admission. `RunPrepared` claims its handle,
rechecks credential availability, and then asks the manager to admit the
frozen backend before generation.
Admission is immediate: a limited pool either reserves a slot or returns only
the internal `ErrCapacityExceeded` identity. The runner attaches the selected
backend identity at its use-case boundary, and the root facade translates that
typed value without treating it as an invalid request or generation failure.
The total admitted bound is the active-generation limit plus its configured
waiting capacity. The returned release function is idempotent. The runner
defers it as soon as admission succeeds. An ordinary run holds the lease across
remaining preparation, initial generation, validation, every repair attempt,
and all failure or cancellation exits. Prepared execution holds the normal
lease across generation, validation, every internal repair attempt, and all
execution exits. A repair is part of its original admission and does not
reserve another bounded slot.
## FIFO Generation Permits
`NewClient` wraps the engine's selected internal model client after public
client adaptation or built-in client construction. Initial generation and the
default repairer receive the same wrapper.
For each `Generate` call, the wrapper selects a pool from the request's
effective backend ID. An unlimited call passes directly to the next client. A
limited call acquires an active permit, invokes the next client, and defers
permit release so ordinary returns and panic unwinding both restore capacity.
Preparation and validation never hold an active permit.
When all active permits are occupied, calls join a mutex-protected FIFO waiter
list. Releasing a permit transfers it directly to the oldest remaining waiter
before making it generally available. Pools do not order work relative to
other backend IDs.
The wrapper passes generation requests, responses, and collaborator errors
through unchanged. It owns scheduling only; the concrete model client remains
responsible for provider transport behavior.
## Cancellation And Release
Admission checks the caller context before reserving a slot. A call canceled
while waiting for an active permit removes its waiter under the same pool lock
used to grant permits. If cancellation removes the waiter first, the wrapped
client is not invoked. If a concurrent grant wins first, the call owns the
permit and invokes the client with the original context, allowing the client
to observe cancellation normally.
This grant-or-cancel decision prevents lost and double-released permits.
Admission leases and active permits are released after success, collaborator
errors, validation failures, cancellation, and panic unwinding. Canceled
waiters are unlinked so their contexts and requests are not retained by the
pool.
## Test Ownership
The [manager tests](../../internal/capacity/manager_test.go) own policy
validation, bounded admission, idempotent release, context handling, and
unlimited admission. The
[client tests](../../internal/capacity/client_test.go) own peak enforcement,
FIFO transfer, canceled-waiter removal, grant/cancel races, independent pools,
unlimited calls, passthrough behavior, and panic release.
The [runner tests](../../internal/usecase/runner_test.go) own ordinary early
admission, lease lifetime, failure release, and shared initial/repair
scheduling. The
[prepared-execution use-case tests](../../internal/usecase/prepared_execution_test.go)
own deferred admission, credential ordering, and prepared-execution lease
release. The
[external package capacity tests](../../capacity_contract_test.go) own the
assembled public-engine behavior for configured limits, capacity errors,
endpoint identity, engine independence, and injected clients. The
[prepared-execution contract tests](../../prepared_execution_contract_test.go)
own the public prepared-capacity boundary. The
[root error-boundary tests](../../errors_internal_test.go) own preservation of
the public generation category and context identity when generation is
canceled.

View File

@@ -20,6 +20,11 @@ orchestration. `OpenAICompatibleClient` is the built-in implementation. It
uses internal domain values for rendered prompts, execution targets,
structured output, responses, and token usage.
The runner supplies a fully resolved target after applying backend, profile,
and request precedence. The client uses its endpoint, credential metadata,
generation fields, and extra parameters. `BackendID` remains routing metadata
for the generation boundary and is not mapped into the provider payload.
Construction validates the configured base URL and clones any supplied
`http.Client` so Promptkit can apply its timeout default without mutating the
caller's client. Generation then:
@@ -31,9 +36,28 @@ caller's client. Generation then:
5. performs the outbound request under the applicable deadlines; and
6. decodes the first response choice and token usage.
`internal/llm` owns the set of reserved OpenAI-compatible request fields used
when validating extra parameters. Backend registration consumes the same rule
without making the model client depend on registry configuration.
The implementation has no retry loop, tool-call support, provider catalog,
inbound HTTP behavior, or durable session store.
## Prepared Generation
For [`RunPrepared`](../../engine.go), the runner supplies the model client with
the target, rendered messages, and structured-output constraint retained by
executable preparation. Execution does not reopen or rerender consumer
sources.
Before backend admission, the runner rechecks that the frozen credential
environment-variable name is available. The handle does not retain the
environment value; the model client resolves the value visible when generation
begins. A direct request key remains in private execution state only until the
claimed execution finishes or an unclaimed handle is discarded. Exact public
ownership and redaction semantics belong to the
[`PreparedExecution` GoDoc](../../prepared_execution.go).
## Failure Categories
The package preserves distinct error identities for invalid client
@@ -51,5 +75,7 @@ The
[OpenAI-compatible client tests](../../internal/llm/openai_compatible_client_test.go)
own configuration, client cloning, deterministic deadline precedence,
authentication, request and response mapping, malformed data, error identity,
cancellation, and response-body suppression. They use local test servers and
test transports; the default suite makes no live or paid provider requests.
cancellation, and response-body suppression. The root transport contract test
also verifies that resolved backend settings reach this client without
serializing backend identity. All use local test servers or test transports;
the default suite makes no live or paid provider requests.

View File

@@ -11,19 +11,23 @@ contributor workflow and validation.
| Component | Implemented responsibility | References |
| --- | --- | --- |
| Root `promptkit` package | Provides the supported engine facade, source and injection options, public request and result values, built-in profile construction, extension interfaces, value conversion, redacted formatting, and public error mapping. | [Package GoDoc](../../doc.go), [engine assembly](../../engine.go) |
| Root `promptkit` package | Provides the supported engine facade, source, backend-registration, and injection options, public request, result, prompt-inspection, and profile-inspection values, opaque prepared-execution handles, profile construction, extension interfaces, value conversion, redacted formatting, typed capacity errors, public error mapping, and engine-local profile-source assembly including application fallbacks. | [Package GoDoc](../../doc.go), [prepared execution](../../prepared_execution.go), [backend API](../../backends.go), [engine assembly](../../engine.go) |
| `examples/go-library/prepare` | Demonstrates an offline downstream consumer using a prompt file, in-memory profile, inline input, and `Prepare`. It is not a public library package. | [Example program](../../examples/go-library/prepare/main.go) |
| `examples/go-library/run` | Demonstrates an offline downstream consumer using a prompt file, in-memory profile, inline input, an injected deterministic model client, and `Run`. It is not a public library package. | [Example program](../../examples/go-library/run/main.go) |
| `internal/backend` | Constructs each engine's immutable registry from the built-in OpenRouter definition and consumer additions, validates and defensively copies definitions through the shared JSON-value package, and consumes the LLM-owned OpenAI-compatible reserved request-field rule. | [Backend registry](../../internal/backend/registry.go) |
| `internal/capacity` | Owns engine-local bounded execution admission and FIFO model-generation permits for limited backend IDs, including cancellation-safe waiter removal and client wrapping. | [Internal capacity management](capacity.md) |
| `internal/domain` | Defines internal framework values for requests, artifacts, prompt definitions, profiles, execution targets, rendering, generation, and validation. | [Domain declarations](../../internal/domain/domain.go) |
| `internal/defaults` | Defines application-neutral framework constants and constructs the default execution target. It contains no CLI, server, or inbound HTTP limits. | [Framework defaults](../../internal/defaults/defaults.go) |
| `internal/filecatalog` | Provides deterministic YAML discovery and path helpers for operating-system filesystems and `fs.FS` sources. | [File catalog](../../internal/filecatalog/catalog.go) |
| `internal/jsonvalue` | Validates and deeply copies JSON-compatible extra-parameter and prepared-schema trees while preserving supported concrete value types. | [JSON values](../../internal/jsonvalue/jsonvalue.go) |
| `internal/promptdef` | Loads strictly decoded, validated prompt definitions from filesystem and `fs.FS` sources, including version selection and contained file-backed message content. | [Framework formats](../formats.md), [prompt-definition repository](../../internal/promptdef/filesystem_repository.go) |
| `internal/profile` | Loads strictly decoded, validated execution profiles from filesystem and `fs.FS` sources and composes repositories with error-preserving fallback. | [Framework formats](../formats.md), [profile repositories](../../internal/profile/filesystem_repository.go) |
| `internal/profile/builtin` | Embeds the built-in execution profile catalog and combines it with an optional primary repository. | [Built-in catalog](../formats.md#built-in-profile-catalog), [repository](../../internal/profile/builtin/repository.go) |
| `internal/profile` | Loads strictly decoded, validated execution profiles, including backend selection, from filesystem and `fs.FS` sources and composes repositories with error-preserving fallback. | [Framework formats](../formats.md), [profile repositories](../../internal/profile/filesystem_repository.go) |
| `internal/profile/builtin` | Embeds the built-in profile catalog, whose entries select OpenRouter. | [Built-in catalog](../formats.md#built-in-profile-catalog), [repository](../../internal/profile/builtin/repository.go) |
| `internal/prompt` | Renders prompt messages from Go templates with artifact, variable, session, and cache-control data. | [Go-template renderer](../../internal/prompt/go_renderer.go) |
| `internal/artifact` | Resolves ordinary inline and unrestricted caller-selected file references into copied artifacts with metadata and hashes. | [Internal sources and validation](sources.md) |
| `internal/validate` | Validates basic, JSON, and JSON Schema output using operating-system filesystem or `fs.FS` schema sources. | [Framework formats](../formats.md#schemas), [internal sources and validation](sources.md) |
| `internal/llm` | Defines the internal generation boundary and implements outbound OpenAI-compatible chat requests, response decoding, authentication, and deadline handling. | [Internal model client](llm.md) |
| `internal/usecase` | Coordinates preparation and execution across internal sources, rendering, artifact loading, generation, validation, and optional repair. | [Internal runner](runner.md) |
| `internal/validate` | Validates basic, JSON, and JSON Schema output using operating-system filesystem or `fs.FS` schema sources and creates frozen validation plans for prepared execution. | [Framework formats](../formats.md#schemas), [internal sources and validation](sources.md) |
| `internal/llm` | Defines the internal generation boundary and implements outbound OpenAI-compatible chat requests from resolved execution targets, including response decoding, authentication, deadline handling, and ownership of the OpenAI-compatible reserved request-field policy. | [Internal model client](llm.md) |
| `internal/usecase` | Resolves prompt definitions and hashes, profiles, backends, and targets for exact inspection and request settings for preparation, and coordinates ordinary execution and one-attempt prepared execution across internal sources, rendering, artifact loading, generation, validation, capacity, and optional repair. | [Internal runner](runner.md), [prepared-execution implementation](../../internal/usecase/prepared_execution.go) |
The root package assembles these internal components without exposing their
representations. Consumers depend only on the root facade.

View File

@@ -17,7 +17,10 @@ and override semantics consumed by the runner.
## Collaborators
`Runner` coordinates narrow internal interfaces for prompt definitions,
profiles, artifacts, rendering, model generation, and validation. Schema
profiles, backend resolution, artifacts, rendering, model generation, and
validation. The root engine supplies one immutable registry containing the
built-in backend and validated consumer additions, one engine-local run
admitter, and a model client wrapped by the same capacity manager. Schema
documents are loaded through the validator's optional schema-loader interface.
An output repairer can be injected internally, but the ordinary runner
constructor does not enable one.
@@ -25,43 +28,121 @@ constructor does not enable one.
Each invocation carries its state in request, prepared-run, and result values.
The runner has no durable run or session store.
## Preparation Flow
## Shared Prompt Selection
`Prepare` performs the reusable pre-generation workflow:
The runner uses one prompt-selection and hashing boundary for ordinary
preparation and exact prompt inspection. Preparation retains its early
request-ID check before direct-session normalization; both operations then use
the configured prompt repository to select one definition, load referenced
message content, and calculate the same prompt hash.
1. validate the prompt selection and load the prompt definition;
2. hash the loaded definition;
Inspection stops after that structural lookup. It does not parse templates or
touch profile, artifact, schema, renderer, validator, admission, or model
collaborators. The root [`Engine.InspectPrompt`](../../engine.go) GoDoc owns
the public operation's exact contract.
## Shared Profile Selection
The runner uses one profile-selection and target-resolution boundary for
ordinary preparation and exact profile inspection. Preparation first selects a
request profile or a prompt default; inspection begins with its required
explicit profile ID. Both then apply the ordinary source precedence, resolve a
named backend, and construct the effective target from framework, backend, and
profile values.
Inspection stops after the resulting endpoint and model are structurally
validated. It does not check credential availability or perform prompt,
artifact, schema, rendering, admission, or model-client work. The root
[`Engine.InspectProfile`](../../engine.go) GoDoc owns the public operation's
exact contract.
## Shared Preparation Pipeline
`Prepare` and `Run` share one private preparation pipeline split at the point
where a run can be assigned to its selected backend pool. The resolution phase
performs only the work needed to validate routing and admission:
1. validate the required prompt selection and normalize any direct session ID;
2. load the prompt definition and hash the original definition;
3. select the request profile or the prompt's default profile;
4. resolve application-neutral defaults, profile values, and explicit request
overrides in that order;
5. validate endpoint, model, numeric overrides, and credential requirements;
6. resolve the output contract and load a structured-output schema when
required;
7. load and hash input artifacts;
8. render and hash the prompt; and
9. return the effective settings, source identities, messages, hashes, and
preparation timing.
4. resolve the profile's backend ID, when present;
5. resolve application-neutral defaults, backend defaults, profile values,
and explicit request overrides in that order;
6. validate endpoint, model, numeric overrides, and credential requirements;
7. resolve the effective output contract without loading its schema; and
8. retain the definition, source identities, effective settings, output
contract, and preparation start time in invocation-local state.
The completion phase consumes that state without reloading the prompt,
profile, or backend:
1. load structured-output schema metadata when required;
2. load and hash input artifacts;
3. render messages and the prompt-defined session;
4. apply any direct session ID;
5. hash the effective rendered prompt; and
6. construct the prepared value and preparation timing.
`Prepare` runs both phases consecutively and never performs capacity admission.
`Run` performs backend admission between the phases. This structure preserves
one execution-precedence and error-ordering implementation while allowing a
full backend pool to reject work before expensive schema, artifact, and
rendering operations.
Pointer-based numeric overrides preserve an explicit zero. Invalid negative or
out-of-range values fail as invalid requests. A direct API key takes
precedence over environment lookup for execution; secret values remain
excluded from serialized metadata.
out-of-range values fail as invalid requests. Endpoint overrides do not change
the selected backend identity. Non-empty extra-parameter maps replace whole
lower-precedence maps. A direct API key takes precedence over environment
lookup; otherwise request, profile, and backend environment-variable names
apply in that order. A profile requiring a direct key clears an inherited
backend environment name unless the request supplies its own name. Secret
values remain excluded from serialized metadata.
Reasoning overrides are tri-state: nil inherits the profile, a pointer to a
nonblank string trims and replaces it, and a pointer to a blank string clears
it. A nonblank direct session is normalized before source loading, bypasses
the prompt session template, and is applied after ordinary message rendering.
A blank direct value retains prompt-template behavior. The runner clears the
template only on a value copy of the definition, so the definition hash always
describes the original source while the rendered-prompt hash includes the
effective direct or rendered session.
The registry is read-only after engine construction. Concurrent `Prepare` and
`Run` calls resolve independent defensive backend values and keep all
invocation state local.
## Run Flow
`Run` calls `Prepare` rather than maintaining a second preparation path. It
performs one initial generation call, builds the named output artifact, and
validates that artifact. Invalid generated content remains a validation result;
an inability to perform validation is an operational error.
`Run` records its start time, performs the shared resolution phase, and asks
its `RunAdmitter` to reserve capacity for the effective backend ID. A nil
admitter is an internal unlimited fallback. After successful admission, `Run`
immediately defers the returned release function, performs the completion
phase, makes one initial generation call, builds the named output artifact,
and validates that artifact. Invalid generated content remains a validation
result; an inability to perform validation is an operational error.
The admission lease covers completion-phase preparation, initial generation,
validation, every repair, and every exit. It bounds accepted work without
serializing preparation or validation behind the active-generation limit.
The wrapped model client separately acquires a FIFO active permit only around
each actual generation call.
When an internal repairer is present, a JSON or JSON Schema content failure can
trigger bounded repair attempts. Repair receives the effective execution
target, validation errors, prior output, and structured-output specification.
This capability remains internal and is not a public option.
target and session ID, validation errors, prior output, and structured-output
specification. The default repairer uses the same wrapped client as initial
generation, so each repair reacquires the selected backend's active permit
while remaining inside its original admission lease. Repair never performs a
second bounded admission. This capability remains internal and is not a public
option.
A successful result includes the output artifact and raw output, validation
state, prompt and rendered-prompt hashes, selected profile, effective settings,
input hashes, token usage, a generated run identifier, and UTC timing.
state, effective session ID, prompt and rendered-prompt hashes, selected
profile and backend, effective settings, input hashes, token usage, a generated
run identifier, and UTC timing. The same effective session reaches initial
generation and any repair attempt through the rendered prompt. The same
effective target, including backend identity, reaches generation and any
repair attempt.
## Failure Categories
@@ -69,17 +150,40 @@ Package errors distinguish invalid requests, required profile selection,
credential failures, and prompt, profile, artifact, rendering, generation, and
validation failures. Wrapping preserves the package identities mapped by the
public facade and retains collaborator identities where they are part of the
internal contract. Context cancellation propagates through the invoked
collaborator and is classified by the owning operation.
internal contract.
Admission capacity exhaustion retains the internal capacity identity. At the
use-case boundary, the runner attaches the selected backend ID in an internal
typed error, and the root facade copies that value into the public
[`CapacityError`](../../capacity_error.go) without parsing diagnostic text. It
is not recategorized as an invalid request or generation failure, and no
partial result is returned. A context already done at admission retains its
context identity directly. Cancellation while waiting for an active generation
permit prevents client invocation when it wins the grant race; the model-client
boundary then preserves the context error through the generation-failure
category. Deferred release restores the admission lease on preparation,
generation, validation, repair, and cancellation failures.
Other context cancellation propagates through the invoked collaborator and is
classified by the owning operation.
An overlong direct session is an invalid request before source loading, while
an invalid or overlong prompt session template remains a prompt-render failure.
An unknown selected backend, or a selected backend with no configured resolver,
is classified as a profile-load failure.
## Test Ownership And Changes
The [runner tests](../../internal/usecase/runner_test.go) own preparation order,
selection and override precedence, schema-before-generation behavior, hashing,
generation and validation outcomes, bounded repair, credentials and redaction,
error categories, artifact metadata, usage, and timing.
selection and override precedence, the two-phase boundary, early admission,
lease lifetime and release, direct-session resolution, schema-before-generation
behavior, hashing, generation and validation outcomes, backend propagation,
bounded repair, shared initial/repair capacity, credentials and redaction,
error categories, artifact metadata, usage, and timing. The
[capacity subsystem document](capacity.md) identifies the focused pool,
waiter, and wrapped-client tests.
Changes to orchestration should continue to use the existing package
interfaces, keep request state local to an invocation, and preserve `Run`'s use
of `Prepare`. Source, renderer, validator, or model-client contract changes
belong first in their owning package and document.
interfaces, keep request state local to an invocation, and preserve the shared
resolution and completion pipeline. Source, renderer, validator, or
model-client contract changes belong first in their owning package and
document.

View File

@@ -16,6 +16,11 @@ validation modes, built-in catalog, and source precedence.
definitions, selects an ID and optional version, and resolves file-backed
message content within the selected operating-system or `fs.FS` source.
Exact prompt inspection performs one point-in-time lookup through that same
repository and validates referenced message content before returning declared
metadata. It does not parse templates or read profile, input, or schema
sources, and it does not retain the definition for a later execution.
Its package tests own prompt selection, strict decoding, definition validation,
duplicate detection, and source containment:
[prompt-definition repository tests](../../internal/promptdef/repository_test.go).
@@ -23,14 +28,30 @@ duplicate detection, and source containment:
## Profiles And Built-Ins
`internal/profile` loads and validates execution profiles from an
operating-system filesystem or an `fs.FS`. It supports a primary repository
with fallback only when the primary reports that a profile is absent.
operating-system filesystem or an `fs.FS`. Its overlay repository consults the
next repository only when the higher-precedence repository reports that a
profile is absent. Strict YAML decoding recognizes the optional `backend`
field, trims its value, and requires a model plus at least one non-blank
backend or endpoint. Loading does not check registry membership because the
available registry belongs to the assembled engine; the runner checks
membership during preparation and exact profile inspection.
`internal/profile/builtin` embeds the maintained built-in profile catalog and
can place a caller-selected repository ahead of that catalog. Profile behavior
is owned by the
[profile repository tests](../../internal/profile/repository_test.go), while
catalog completeness, duplicate IDs, and overlay behavior are owned by the
The root engine assembles profile repositories in precedence order: in-memory
profiles, one ordinary configured source, an application fallback source, then
the embedded built-in catalog. An explicit file or `fs.FS` profile source
replaces `Config.ProfileDir` within the ordinary configured-source category.
Exact profile inspection performs one point-in-time lookup through those
profile sources and checks the resolved target without reading prompt, input,
or schema sources. It does not retain that lookup for a later execution.
`internal/profile/builtin` embeds the maintained built-in profile catalog.
Every embedded profile selects `openrouter` and inherits its endpoint and
credential environment-variable name from the built-in backend registry rather
than repeating those values. Profile loading and overlay behavior are owned by
the [profile repository tests](../../internal/profile/repository_test.go),
while catalog completeness, the backend-selection invariant, and duplicate IDs
are owned by the
[built-in repository tests](../../internal/profile/builtin/repository_test.go).
## Ordinary Artifacts
@@ -61,6 +82,22 @@ filesystem or an `fs.FS`. Invalid generated content is returned as a validation
result; inability to load, register, or compile a schema is an operational
error.
For executable preparation, the built-in validators create a frozen validation
plan. None, basic, and JSON modes retain the effective output contract without
source access. JSON Schema mode loads the root document, resolves and compiles
every transitive reference during preparation, and retains the compiled
validator. The provider-facing structured-output metadata uses that same
captured root document.
`PrepareExecution` also completes prompt and profile selection, artifact
loading and hashing, session and message rendering, and target resolution.
`RunPrepared` uses the retained source-derived state and validation plan; it
does not reopen prompt, profile, input, or schema sources and does not rerender
the request. By contrast, ordinary `Prepare` produces a preparation value only:
a later `Run` performs its own source resolution and preparation.
The [validator tests](../../internal/validate/standard_validator_test.go) own
basic, JSON, JSON Schema, source resolution, schema loading, compilation, and
content-failure behavior.
basic, JSON, JSON Schema, source resolution, schema loading, compilation,
frozen-reference behavior, and content-failure behavior. Prepared execution
orchestration is owned by the
[use-case tests](../../internal/usecase/prepared_execution_test.go).

View File

@@ -21,10 +21,16 @@ The implemented internal components consist of:
- `internal/domain`, which owns framework data values shared by later internal
components;
- `internal/backend`, which owns validated immutable OpenAI-compatible backend
definitions and the built-in OpenRouter definition;
- `internal/capacity`, which owns engine-local bounded run admission and
model-generation scheduling for limited backends;
- `internal/defaults`, which owns application-neutral framework defaults and
constructs the default execution target;
- `internal/filecatalog`, which discovers YAML files and provides source-path
helpers for filesystem and `fs.FS` consumers;
- `internal/jsonvalue`, which validates and defensively copies JSON-compatible
extra-parameter trees;
- `internal/promptdef`, which loads and validates prompt definitions from
filesystem and `fs.FS` sources;
- `internal/profile`, which loads, validates, and overlays execution profiles
@@ -41,21 +47,25 @@ The implemented internal components consist of:
- `internal/usecase`, which coordinates preparation and execution across the
internal framework components.
The `examples/go-library/prepare` package is a maintained downstream consumer
of the root facade. It does not expose a library package or participate in
internal assembly.
The `examples/go-library/prepare` and `examples/go-library/run` packages are
maintained downstream consumers of the root facade. They do not expose library
packages or participate in internal assembly.
The root facade assembles the internal repositories, renderer, validator,
outbound client, and use-case runner while translating public values and
errors at the library boundary. The defaults and renderer depend on the domain
model. Prompt-definition and profile repositories use the domain model, file
catalog, and YAML decoder. The built-in profile repository supplies an
embedded `fs.FS` to the profile package. Artifact reading uses the domain model
and application-neutral defaults. Validation uses the domain model, file
The root facade assembles one immutable backend registry, one capacity manager,
the internal repositories, renderer, validator, outbound client, and use-case
runner while translating public values and errors at the library boundary. The
registry contains built-ins plus validated engine-scoped consumer additions.
The facade constructs the capacity manager from the registry's immutable
policy snapshot, wraps the selected built-in or injected model client, and
supplies bounded admission to the runner. The defaults and renderer depend on
the domain model. Prompt-definition and profile repositories use the domain
model, file catalog, and YAML decoder. The built-in profile repository supplies
an embedded `fs.FS` to the profile package. Artifact reading uses the domain
model and application-neutral defaults. Validation uses the domain model, file
catalog, and JSON Schema implementation. The model client uses the domain
model, application-neutral defaults, and an injected or standard-library HTTP
client. The use-case runner depends on the narrow interfaces owned by each
internal component.
internal component, including backend lookup and run admission.
The current implementation follows this dependency direction:
@@ -72,9 +82,15 @@ downstream consumers, including Scriptorium
narrow injected abstractions
```
The facade coordinates internal components and adapts the supported public
extension interfaces to narrow internal abstractions. Internal components must
not depend on consumers or on Scriptorium.
The backend registry depends on the domain model and shared JSON-value
validation, has no mutation API after construction, and consumes the
OpenAI-compatible reserved request-field rule owned by the model client. The
capacity component depends on the domain model and the narrow internal
model-client boundary, not on provider transport implementation. The model
client does not depend on registry or capacity configuration. The facade
coordinates internal components and adapts the supported public extension
interfaces to narrow internal abstractions. Internal components must not depend
on consumers or on Scriptorium.
## Repository And Consumer Boundary

View File

@@ -78,6 +78,7 @@ mechanisms, not secret values.
| Framework file formats | `docs/formats.md` | Prompt-definition and profile YAML fields, schema references, defaults, validation modes, built-in profiles, credentials, and file-to-request precedence. | Exported Go declarations, outbound wire behavior, internal parsing mechanics, and application configuration. |
| Consumer guidance | `docs/consumers/`, when consumer workflows require dedicated guidance | Task-oriented use of implemented public APIs, minimal examples, and consumer responsibilities. | Exact exported declarations and internal mechanics. |
| Durable integration contracts | `docs/integrations/`, when integrations exist | External formats and protocols, compatibility behavior, and upstream or downstream responsibilities. | Internal transformations and public Go declarations. |
| Supplemental release guidance | None. `docs/releases/` may be used when a release benefits from a changelog or migration guide. | No canonical content. These files may briefly summarize release-specific changes, compatibility, and consumer migration paths, and may be corrected, consolidated, archived, or removed when no longer useful. | Public API and behavior contracts, formats, integrations, architecture, release procedure, and the authoritative annotated-tag release record. |
| Implemented component inventory | `docs/internal/overview.md` | Current packages and components, their implemented responsibilities, and links to focused internal documents. | Normative architecture, contributor workflow, external contracts, and proposed components. |
| Internal subsystem behavior | Other files under `docs/internal/`, when a subsystem needs durable detail | Implementation flow, internal collaborators and state transitions, package-local guarantees and failures, and relevant tests. | Global architecture invariants, public API definitions, and future package plans. |
| Architectural decision history | `docs/adr/`, when repository-local decisions require records | Significant decisions, context, alternatives, rationale, consequences, and supersession history. | Current behavior reference, implementation status, and task sequencing. |
@@ -85,9 +86,9 @@ mechanisms, not secret values.
| Complete copyable artifacts | `examples/` | Valid inputs, Go programs, and other files intended to be copied or run. | Field-by-field reference, exact API declarations, and prose explanation. |
Conditional owners do not require placeholder files or directories. Create a
consumer, integration, subsystem, ADR, roadmap, or example document only when
the corresponding implemented interface, decision, planned effort, or
maintained artifact exists.
consumer, integration, release, subsystem, ADR, roadmap, or example document
only when the corresponding implemented interface, release, decision, planned
effort, or maintained artifact exists.
## Boundary Rules
@@ -109,6 +110,23 @@ but must link to its canonical definition rather than restate it.
The [framework format reference](../formats.md) owns exact prompt, profile, and
schema-file contracts. Integration documents own external wire formats.
### Supplemental Release Guidance
Files under `docs/releases/` may provide changelog-style summaries and
migration guidance for a particular release. They are navigation and
orientation aids, not canonical owners of public APIs, behavior, formats,
integrations, architecture, release procedure, or other durable facts. When a
reader needs detail beyond a short release-specific note, the release document
must link to the applicable canonical documentation rather than reproduce its
contract.
The annotated tag message required by the
[release procedure](../release.md#write-the-release-note) remains the
authoritative release record. Supplemental release documents may be corrected,
consolidated, archived, or removed at any time when they are no longer useful,
provided maintained documentation does not depend on them and the annotated
tag record remains intact.
### Security Topics
This policy owns what documentation and examples may contain. Architecture owns
@@ -160,6 +178,10 @@ durable owners, update incoming links, and archive or remove the roadmap
according to repository practice. Do not preserve completed roadmaps as a
second current-state reference.
Supplemental release documents may likewise be removed without preserving a
replacement. Before removal, update maintained incoming links so current
documentation does not depend on an optional historical guide.
Before completing documentation work:
- verify affected behavior and examples;

View File

@@ -7,27 +7,91 @@ tags. It does not publish runnable binaries or binary packages and does not
currently use hosted CI. The release maintainer performs and records the
required validation.
The first planned release is `v0.1.0`. Do not create that tag until the
framework has been extracted and the resulting public library has passed this
procedure. Later tags use the `vMAJOR.MINOR.PATCH` form. While Promptkit remains
pre-`v1`, release notes must identify intentional public API changes and any
consumer migration required by them.
`v0.1.0` is the initial published release. Later releases use semantic
`vMAJOR.MINOR.PATCH` tags. Before `v1`, minor releases may change the public
API and patch releases preserve compatibility within their minor line. Every
pre-`v1` release note must summarize compatibility, identify public API
changes, and state any action required of consumers.
## Prepare The Release
Promptkit releases are source-only. The annotated tag message is the release
note; there is no separate hosted release or binary packaging step.
Work from a clean checkout of the intended release commit, outside any Go
workspace and without a local module replacement. Confirm the source commit is
already published through the normal branch workflow.
## Establish The Candidate
From the Promptkit repository root, verify the checkout:
Choose a version that has not been published and export it as
`RELEASE_VERSION`. Run every command in this procedure from the Promptkit
repository root in the same POSIX shell. Do not reuse `v0.1.0` or another
existing version.
The following guard derives the release commit from `HEAD` and stops on a
missing or malformed version, a checkout other than synchronized `main`,
uncommitted changes, an active Go workspace, a module replacement, a vendor
tree, or an existing local or remote tag:
```sh
gowork=$(go env GOWORK)
test -z "$gowork" || test "$gowork" = off
test -z "$(git status --short)"
git fetch --tags origin
set -eu
: "${RELEASE_VERSION:?export an unpublished vMAJOR.MINOR.PATCH version}"
if ! printf '%s\n' "$RELEASE_VERSION" |
grep -Eq '^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$'
then
printf '%s\n' "invalid release version: $RELEASE_VERSION" >&2
exit 1
fi
RELEASE_COMMIT=$(git rev-parse --verify 'HEAD^{commit}')
export RELEASE_COMMIT
check_release_candidate() {
test "$(git branch --show-current)" = main
test -z "$(git status --porcelain)"
gowork_value=$(go env GOWORK)
case "$gowork_value" in
''|off) ;;
*)
printf '%s\n' "active Go workspace: $gowork_value" >&2
return 1
;;
esac
test -z "$(git ls-files go.work go.work.sum)"
test ! -e vendor
if grep -Eq '^[[:space:]]*replace([[:space:]]|\()' go.mod
then
printf '%s\n' 'go.mod contains a replacement' >&2
return 1
fi
git fetch origin main --tags
test "$RELEASE_COMMIT" = \
"$(git rev-parse --verify 'refs/remotes/origin/main^{commit}')"
if git show-ref --verify --quiet "refs/tags/$RELEASE_VERSION"
then
printf '%s\n' "local tag already exists: $RELEASE_VERSION" >&2
return 1
fi
if test -n "$(
git ls-remote --tags origin \
"refs/tags/$RELEASE_VERSION" \
"refs/tags/$RELEASE_VERSION^{}"
)"
then
printf '%s\n' "remote tag already exists: $RELEASE_VERSION" >&2
return 1
fi
}
check_release_candidate
```
Do not continue unless the guard completes successfully. In particular, push
the intended commit through the normal `main` branch workflow before release;
the tag procedure is not a substitute for publishing the source commit.
## Validate The Candidate
Confirm the module and root package metadata:
```sh
@@ -42,7 +106,7 @@ gitea.maximumdirect.net/eric/promptkit 1.25.5
promptkit gitea.maximumdirect.net/eric/promptkit
```
Run the same default Go validation required by the
Run the complete maintainer validation required by the
[development guide](development.md):
```sh
@@ -53,84 +117,158 @@ go build ./...
go run ./examples/go-library/prepare
```
Check every tracked Go file and repository whitespace:
Check every tracked Go file. This command must produce no output:
```sh
gofmt -l $(git ls-files '*.go')
unformatted=$(
git ls-files '*.go' |
while IFS= read -r go_file
do
gofmt -l "$go_file"
done
)
test -z "$unformatted"
```
Follow every maintained Markdown link and confirm that its local or published
target exists. Review the repository for generated binaries, test or coverage
output, credentials, template residue, downloaded assets, and other files that
do not belong in source control.
Recheck module and repository hygiene, whitespace, and the clean checkout:
```sh
test -z "$(git ls-files go.work go.work.sum)"
test ! -e vendor
if grep -Eq '^[[:space:]]*replace([[:space:]]|\()' go.mod
then
printf '%s\n' 'go.mod contains a replacement' >&2
exit 1
fi
git diff --check
test -z "$(git status --porcelain)"
```
The formatting command must produce no paths. Follow every maintained Markdown
link and confirm its target exists. Review the repository for generated
binaries, test or coverage output, credentials, template residue, and other
files that do not belong in source control.
## Write The Release Note
Confirm that no workspace override is tracked and that `go.mod` contains no
`replace` directive:
Prepare a plain-text annotated-tag message outside the repository and export
its path as `RELEASE_NOTES_FILE`. Use this form, replacing each summary with
release-specific text; write `None.` when there are no public API changes or
consumer actions:
```sh
git ls-files go.work go.work.sum
rg -n '^replace\b' go.mod
```text
Promptkit vMAJOR.MINOR.PATCH
Validated commit: full commit ID
Compatibility: compatibility summary
Public API changes: changes or None.
Consumer action: required action or None.
```
Both commands must produce no output. Re-run `git status --short` and require a
clean result after every validation and review check.
## Create And Publish The Tag
Choose the semantic version from the intended compatibility change. Record the
release commit before tagging:
After writing it, require all release-note fields, the selected version, and
the validated commit to be present:
```sh
release_version=v0.1.0
release_commit=$(git rev-parse HEAD)
: "${RELEASE_NOTES_FILE:?export the path to the release-note file}"
test -f "$RELEASE_NOTES_FILE"
test -s "$RELEASE_NOTES_FILE"
grep -F "Promptkit $RELEASE_VERSION" "$RELEASE_NOTES_FILE"
grep -F "Validated commit: $RELEASE_COMMIT" "$RELEASE_NOTES_FILE"
grep -F 'Compatibility:' "$RELEASE_NOTES_FILE"
grep -F 'Public API changes:' "$RELEASE_NOTES_FILE"
grep -F 'Consumer action:' "$RELEASE_NOTES_FILE"
```
Replace the example version for later releases and keep both values in the same
shell for the remaining commands. Confirm the tag does not already exist
locally or remotely:
Inspect the complete message and confirm that it accurately records the
compatibility impact, public API changes, and required consumer action.
## Create And Inspect The Tag
Run the candidate guard again immediately before tag creation. This ensures
that validation or release-note preparation did not change the checkout and
that the commit is still published and untagged:
```sh
test -z "$(git tag --list "$release_version")"
test -z "$(git ls-remote --tags origin "refs/tags/$release_version")"
check_release_candidate
```
Create an annotated tag whose message identifies the release and records that
the documented validation passed for the tagged commit:
Create the annotated tag from the prepared release note and bind it explicitly
to the validated commit:
```sh
git tag --annotate "$release_version" \
--message "Promptkit $release_version; documented validation passed for $release_commit"
git tag --annotate "$RELEASE_VERSION" \
--file "$RELEASE_NOTES_FILE" \
"$RELEASE_COMMIT"
```
Inspect the tag before publication:
Inspect both the tag message and its source commit before publication:
```sh
git show --no-patch --decorate "$release_version"
test "$(git rev-list -n 1 "$release_version")" = "$release_commit"
test "$(git cat-file -t "refs/tags/$RELEASE_VERSION")" = tag
git show --no-patch --decorate "refs/tags/$RELEASE_VERSION"
test "$(
git rev-parse --verify "refs/tags/$RELEASE_VERSION^{commit}"
)" = "$RELEASE_COMMIT"
```
Publish the tag without relying on a hosting-provider-specific release
interface:
If inspection finds an error, delete the unpublished local tag, correct the
release note or candidate, and repeat the guards. Never move or recreate a tag
that has been published.
## Publish The Selected Tag
Push only the selected tag ref. Do not use `git push --tags`:
```sh
git push origin "refs/tags/$release_version"
git push origin \
"refs/tags/$RELEASE_VERSION:refs/tags/$RELEASE_VERSION"
```
## Verify Publication
Confirm that the remote tag object matches the local annotated tag and still
resolves to the intended source commit:
Compare the remote annotated-tag object with the local object, then compare the
remote peeled source commit with the validated commit:
```sh
remote_tag=$(git ls-remote --tags origin "refs/tags/$release_version" | awk '{print $1}')
test "$remote_tag" = "$(git rev-parse "refs/tags/$release_version")"
test "$(git rev-list -n 1 "refs/tags/$release_version")" = "$release_commit"
remote_tag=$(
git ls-remote --tags origin "refs/tags/$RELEASE_VERSION" |
awk 'NR == 1 { print $1 }'
)
remote_commit=$(
git ls-remote --tags origin "refs/tags/$RELEASE_VERSION^{}" |
awk 'NR == 1 { print $1 }'
)
test -n "$remote_tag"
test "$remote_tag" = \
"$(git rev-parse --verify "refs/tags/$RELEASE_VERSION")"
test "$remote_commit" = "$RELEASE_COMMIT"
```
Promptkit must publish the required tag before Scriptorium or another consumer
publishes a release that depends on that version. Released consumer modules
must not use a local replacement or unpublished Promptkit revision.
Finally, resolve the version as an ordinary Go module in a temporary module
outside this repository and without a workspace or replacement:
```sh
resolution_dir=$(mktemp -d)
(
trap 'rm -rf "$resolution_dir"' 0 1 2 15
cd "$resolution_dir"
GOWORK=off go mod init example.com/promptkit-release-check
GOWORK=off go mod download \
"gitea.maximumdirect.net/eric/promptkit@$RELEASE_VERSION"
resolved_version=$(
GOWORK=off go list -m -f '{{.Version}}' \
"gitea.maximumdirect.net/eric/promptkit@$RELEASE_VERSION"
)
test "$resolved_version" = "$RELEASE_VERSION"
)
```
Promptkit must publish and verify the required version before Scriptorium or
another consumer publishes a release that depends on it. This ordering does
not replace the consumer project's own release procedure. Released consumers
must select the published Promptkit tag through ordinary module resolution,
without a workspace, replacement, vendored Promptkit source, or unpublished
revision.
## Policy Changes

243
docs/releases/v0.2.0.md Normal file
View File

@@ -0,0 +1,243 @@
# Promptkit v0.2.0
This supplemental changelog and migration guide summarizes the consumer-facing
changes from `v0.1.0` to `v0.2.0`. The annotated `v0.2.0` tag is the
authoritative release record. Exact current contracts belong to the linked
GoDoc and durable documentation.
## Summary
`v0.2.0` adds three major capabilities:
- an engine-scoped registry for reusable OpenAI-compatible backend
definitions;
- bounded, backend-specific run admission and model-generation concurrency;
and
- direct per-run session IDs and tri-state reasoning-effort overrides.
Existing endpoint-only profiles remain supported. Consumers can adopt backend
registration and runtime overrides incrementally rather than rewriting all
profiles during the upgrade.
## Compatibility At A Glance
Promptkit remains pre-`v1`, and this minor release includes source-level and
behavioral changes that deserve review.
| Area | `v0.1.0` consumer impact |
| --- | --- |
| Endpoint-only profiles | Continue to work without migration. |
| Built-in profiles | Continue to use OpenRouter and `OPENROUTER_API_KEY`; they now select the built-in `openrouter` backend. |
| Custom backends | Registration is optional. Existing profiles may keep their endpoint and credential configuration. |
| Reasoning overrides | String assignments must migrate to the new pointer field. |
| `RunRequest.Metadata` | Removed; delete assignments to this field. |
| OpenRouter concurrency | Now limited to 16 active generations with waiting capacity of 1024 per engine. |
| Public JSON | `v0.2.0` formalizes supported JSON representations; consumers relying on `v0.1.0` encodings should review the notes below. |
| Unkeyed public struct literals | May require updates because fields were added. Keyed literals are recommended. |
## Upgrade
After the `v0.2.0` tag is published, update the module dependency with:
```sh
go get gitea.maximumdirect.net/eric/promptkit@v0.2.0
go mod tidy
```
Run the consuming project's ordinary tests and race-enabled tests after the
upgrade, especially if it calls one engine concurrently or persists Promptkit
JSON values.
## Backend Registry
Consumers may now register reusable OpenAI-compatible backend definitions with
`WithBackend`, then select them by ID from file-backed or in-memory profiles.
A backend can supply its endpoint, API-key environment-variable name,
request-wide extra parameters, and optional capacity policy.
Registrations are immutable and belong to one engine. Consumer registrations
can add new IDs but cannot replace Promptkit's reserved `openrouter` backend.
Profiles that select a backend may still override its endpoint without losing
the backend's routing or capacity identity.
An existing endpoint-only in-memory profile remains valid:
```go
promptkit.Profile{
ID: "local",
Endpoint: "http://localhost:8000/v1",
Model: "example-model",
}
```
Adopting the registry is optional and can be done when several profiles should
share connection or capacity settings:
```go
engine, err := promptkit.NewEngine(
promptkit.Config{PromptDir: "prompts"},
promptkit.WithBackend(promptkit.Backend{
ID: "local",
Endpoint: "http://localhost:8000/v1",
APIKeyEnv: "LOCAL_LLM_API_KEY",
}),
promptkit.WithProfiles(promptkit.Profile{
ID: "local-summary",
BackendID: "local",
Model: "example-model",
}),
)
```
See the
[local-endpoint consumer guide](../consumers/pkg-promptkit.md#configure-a-local-openai-compatible-endpoint)
for task-oriented usage. The
[`Backend` and `WithBackend` GoDoc](../../backends.go) owns exact registration,
validation, copying, defaulting, and uniqueness semantics. The
[framework format reference](../formats.md) owns the profile `backend` field
and execution precedence.
## Backend-Specific Concurrency
Each registered backend may now define:
- an active model-generation limit; and
- a bounded number of additional admitted `Run` calls.
Promptkit owns scheduling for both its built-in model client and an injected
`LLMClient`. `Run` remains synchronous: an admitted caller waits for its
ordinary result, while a call beyond the bounded admission capacity returns
`ErrCapacityExceeded`. Capacity is engine-local and keyed by backend ID.
Endpoint-only profiles and custom backends without a configured limit remain
unlimited.
The built-in OpenRouter backend now permits 16 active generations and 1024
additional admitted calls per engine. Applications that can exceed this bound
should handle capacity exhaustion separately from provider and request
failures:
```go
result, err := engine.Run(ctx, request)
if errors.Is(err, promptkit.ErrCapacityExceeded) {
// Apply application-specific overload or retry policy.
}
```
Promptkit does not prescribe retries or map this error to an HTTP status. See
the
[concurrency consumer guidance](../consumers/pkg-promptkit.md#limit-backend-concurrency)
and the [`Backend` GoDoc](../../backends.go) for the canonical configuration
contract. Runtime behavior and public error identities belong to the
[`Engine.Run` GoDoc](../../engine.go).
## Per-Run Session IDs
`RunRequest.SessionID` can now supply a consumer-managed correlation ID for one
`Prepare` or `Run` invocation. A nonblank direct value overrides the prompt's
session template and is exposed in prepared values, results, injected-client
requests, and provider observability. Session IDs should therefore be stable,
non-secret values.
```go
result, err := engine.Run(ctx, promptkit.RunRequest{
PromptID: "meeting.summary",
SessionID: "conversation-42",
})
```
The built-in OpenAI-compatible client sends a nonempty effective session as the
top-level `session_id` request-body field, not as an `x-session-id` header. See
the
[session and reasoning consumer guide](../consumers/pkg-promptkit.md#set-a-per-run-session-and-reasoning),
the [`RunRequest` GoDoc](../../types.go), and the
[OpenAI-compatible request contract](../integrations/openai-compatible-chat.md#request-body)
for exact normalization, length, exposure, and wire behavior.
## Per-Run Reasoning Effort
`ExecutionTargetOverride.ReasoningEffort` changed from `string` to `*string` so
one request can distinguish inheritance, replacement, and explicit clearing.
Update a `v0.1.0` override like this:
```go
// v0.1.0
Execution: &promptkit.ExecutionTargetOverride{
ReasoningEffort: "high",
}
```
to:
```go
// v0.2.0
reasoning := "high"
Execution: &promptkit.ExecutionTargetOverride{
ReasoningEffort: &reasoning,
}
```
The three states are:
- `nil` inherits the selected profile's value;
- a pointer to a nonblank string replaces it for that invocation; and
- a pointer to an empty or whitespace-only string clears it for that
invocation.
This allows consumers to consolidate profiles that differed only by reasoning
effort. The [`ExecutionTargetOverride` GoDoc](../../types.go) owns the exact
override contract.
## Other Migration Notes
### Remove `RunRequest.Metadata`
`RunRequest.Metadata` is no longer part of the public request. Remove any
assignment to that field. Use application-owned state keyed by `RunResult.RunID`
or a direct `SessionID` when correlation is needed; these identifiers have
different purposes, so choose according to the application's lifecycle.
### Review Persisted JSON
`v0.2.0` defines stable JSON representations for the public result, artifact,
execution, validation, and model-client values listed in the
[package documentation](../../doc.go). Consumers that treated `v0.1.0`
reflection-derived encodings as stable should update fixtures and stored-data
adapters.
In particular:
- `RunResult` encodes elapsed time as integer milliseconds in `duration_ms`
instead of encoding `time.Duration` under `duration`;
- result JSON can include the new `session_id` and `selected_backend_id`
fields;
- execution-target JSON can include `backend_id`; and
- artifact and target-presence fields now use their documented lower-case
names.
The `v0.2.0` `RunResult` decoder reads `duration_ms`; it does not translate a
persisted `v0.1.0` `duration` field. Transform old payloads before decoding
when preserving their elapsed duration matters.
### Prefer Keyed Struct Literals
New fields were added to several public structs. Replace positional composite
literals with keyed literals so future additive fields do not cause another
source migration.
## Migration Checklist
- Update the module dependency and run the consumer's tests.
- Change reasoning overrides from strings to pointers.
- Remove uses of `RunRequest.Metadata`.
- Review unkeyed Promptkit struct literals.
- Decide whether shared endpoints should move into registered backends.
- If using built-in OpenRouter profiles at high concurrency, handle
`ErrCapacityExceeded` and review the new engine-local bound.
- Review stored JSON, fixtures, and downstream decoders.
- Optionally replace profile-specific session or reasoning variants with
per-run overrides.
For complete consumer workflows, use the
[package consumer guide](../consumers/pkg-promptkit.md) and maintained
[offline execution example](../../examples/go-library/run/main.go).

74
docs/releases/v0.3.0.md Normal file
View File

@@ -0,0 +1,74 @@
# Promptkit v0.3.0
This supplemental changelog summarizes the consumer-facing changes from
`v0.2.0` to `v0.3.0`. The annotated `v0.3.0` tag is the authoritative release
record. Exact current contracts belong to the linked GoDoc and durable
documentation.
## Summary
`v0.3.0` adds a concise way to register the common local OpenAI-compatible
backend configuration:
- `BackendLocal` provides the conventional, non-reserved backend ID `"local"`;
and
- `LocalBackend` constructs an ordinary `Backend` from an endpoint and
concurrency limit.
The helper is explicit and additive. It does not pre-register a backend, read
environment variables, select a model, or replace the complete `Backend`
configuration interface.
## Compatibility
Existing `v0.2.0` consumers require no migration. Endpoint-only profiles,
complete custom `Backend` values, the built-in OpenRouter backend, and existing
registrations using the literal ID `"local"` continue to work unchanged.
## Upgrade
Update the module dependency with:
```sh
go get gitea.maximumdirect.net/eric/promptkit@v0.3.0
go mod tidy
```
Run the consuming project's ordinary tests and race-enabled tests after the
upgrade.
## Configure A Local Backend
Register the convenience value through the existing `WithBackend` option and
select it from one or more profiles:
```go
engine, err := promptkit.NewEngine(
promptkit.Config{PromptDir: "prompts"},
promptkit.WithBackend(
promptkit.LocalBackend("http://localhost:8000/v1", 2),
),
promptkit.WithProfiles(promptkit.Profile{
ID: "local-summary",
BackendID: promptkit.BackendLocal,
Model: "example-model",
}),
)
```
Use an endpoint-only profile when shared backend identity and capacity policy
are unnecessary. Continue to use a complete keyed `Backend` value for custom
IDs, authentication, extra request parameters, explicit queue capacity, or
multiple local endpoints.
See the
[local-endpoint consumer guide](../consumers/pkg-promptkit.md#configure-a-local-openai-compatible-endpoint)
for task-oriented configuration choices. The
[`BackendLocal`, `LocalBackend`, and `WithBackend` GoDoc](../../backends.go)
owns their exact construction, registration, validation, and concurrency
semantics.
## Consumer Action
None. Adopt the convenience constructor when it simplifies local endpoint
configuration.

189
docs/releases/v0.4.0.md Normal file
View File

@@ -0,0 +1,189 @@
# Promptkit v0.4.0
This supplemental changelog and adoption guide summarizes the consumer-facing
changes from `v0.3.0` to `v0.4.0`. The annotated `v0.4.0` tag is the
authoritative release record. Exact current contracts belong to the linked
GoDoc and durable documentation.
## Summary
`v0.4.0` adds four complementary capabilities:
- opaque prepared-execution handles for preparing once, inspecting safe
details, and executing the same frozen snapshot;
- exact prompt-definition inspection without profile resolution or execution;
- exact profile inspection without selecting a prompt or checking credential
availability; and
- structured backend identity on engine admission-capacity rejection.
These APIs let consumers perform more precise preflight work and retain useful
operational context without reproducing Promptkit's internal resolution logic.
## Compatibility
The release is additive for `v0.3.0` consumers. Existing uses of `Prepare`,
`Run`, backend registration, endpoint-only profiles, local-backend helpers,
runtime overrides, public JSON values, and error sentinels continue to work
without migration.
Capacity rejection now returns a structured error while continuing to match
`ErrCapacityExceeded` through `errors.Is`. Error-string wording and direct
sentinel equality were not public contracts.
The new inspection values, capacity error, and prepared-execution handle do not
have stable JSON representations. `PreparedExecution.Details` returns the
existing stable `PreparedRun` value.
## Upgrade
Update the module dependency with:
```sh
go get gitea.maximumdirect.net/eric/promptkit@v0.4.0
go mod tidy
```
Run the consuming project's ordinary and race-enabled tests after upgrading.
No source migration is required.
## Prepare Once And Execute The Same Snapshot
Consumers that need to persist preparation details before generation can now
prepare an opaque, engine-bound execution:
```go
prepared, err := engine.PrepareExecution(ctx, request)
if err != nil {
// Handle preparation failure.
}
defer prepared.Discard()
details := prepared.Details()
// Persist a consumer-selected, appropriately protected preparation record.
result, err := engine.RunPrepared(ctx, prepared)
```
Preparation freezes the selected sources, rendered messages, effective
settings, input content, structured-output metadata, and validation resources
needed by execution. `Details` returns a fresh, caller-owned,
credential-redacted `PreparedRun`.
A handle belongs to its creating engine and permits one execution attempt.
`RunPrepared` consumes that attempt on success and on operational failure.
`Discard` is idempotent and releases an unclaimed handle's execution-only
state. Consumers should discard handles they will not execute, particularly
when a direct request API key may be retained privately until claim or
discard.
Prepared execution does not reserve backend admission during preparation.
Credential availability and backend admission are checked when execution
begins. The execution context is independent of the preparation context.
See the
[prepared-execution consumer guide](../consumers/pkg-promptkit.md#prepare-now-and-execute-the-same-snapshot-later),
the [`PreparedExecution` GoDoc](../../prepared_execution.go), and the
[`Engine.PrepareExecution` and `Engine.RunPrepared` GoDoc](../../engine.go)
for the exact lifecycle, ownership, cancellation, capacity, timing, and
failure contracts.
## Inspect A Prompt
`Engine.InspectPrompt` resolves one prompt ID and optional version through the
engine's configured prompt source:
```go
inspection, err := engine.InspectPrompt(ctx, "report.summary", "")
```
The result includes prompt identity, the opaque prompt hash, declared default
profile ID, declared input metadata, and normalized output contract. It
structurally loads the selected definition and referenced message content but
does not resolve a profile, load schemas or artifacts, render templates,
reserve capacity, or contact a model.
Use inspection for exact configuration checks and metadata discovery. Use
`PrepareExecution` rather than relying on a prior inspection when later
execution must freeze one exact source state, because filesystem-backed
inspection is only a point-in-time lookup.
See the
[prompt-inspection consumer guide](../consumers/pkg-promptkit.md#inspect-a-prompt-before-preparation)
and [`Engine.InspectPrompt` GoDoc](../../engine.go) for exact selection,
ownership, and error behavior.
## Inspect A Profile
`Engine.InspectProfile` resolves one explicit profile independently of a
prompt:
```go
inspection, err := engine.InspectProfile(ctx, "report-production")
```
The result includes the resolved effective execution target and whether a
later request must provide a direct credential. Environment-variable names may
be reported, but inspection does not read credential values or require the
named variable to be populated.
Inspection applies the engine's profile source precedence and resolves any
selected backend. It does not load a prompt, render content, reserve capacity,
or contact a model.
See the
[profile-inspection consumer guide](../consumers/pkg-promptkit.md#inspect-a-profile-before-prompt-work)
and [`Engine.InspectProfile` GoDoc](../../engine.go) for the exact resolution,
credential, ownership, and error contracts.
## Identify Capacity-Rejected Backends
Calls rejected at Promptkit's bounded engine admission boundary continue to
match `ErrCapacityExceeded`. Consumers can additionally obtain the selected
registered backend ID without parsing diagnostic text:
```go
result, err := engine.Run(ctx, request)
if errors.Is(err, promptkit.ErrCapacityExceeded) {
var capacityErr *promptkit.CapacityError
if errors.As(err, &capacityErr) {
// Record capacityErr.BackendID using application-owned diagnostics.
}
// Apply application-owned overload or retry policy.
}
```
The structured error applies to `Run` and `RunPrepared` admission rejection.
It does not represent provider throttling, quota exhaustion, cancellation
while waiting for generation capacity, or another model-client failure.
Promptkit does not prescribe retry timing or transport status mapping.
See the
[error-handling consumer guide](../consumers/pkg-promptkit.md#handle-errors),
the [`CapacityError` GoDoc](../../capacity_error.go), and the
[`ErrCapacityExceeded` GoDoc](../../engine.go) for the canonical contracts.
## Public API Additions
The release adds:
- `Engine.PrepareExecution`;
- `Engine.RunPrepared`;
- `PreparedExecution`, including `Details`, `Discard`, `String`, and
`GoString`;
- `Engine.InspectPrompt`;
- `PromptInspection`;
- `PromptInputDefinition`;
- `Engine.InspectProfile`;
- `ProfileInspection`; and
- `CapacityError`.
No public API was removed.
## Consumer Action
None. Existing `v0.3.0` workflows may upgrade without adopting the new APIs.
Consumers that adopt prepared execution should discard unused handles.
Consumers that need backend-specific capacity diagnostics may add an
`errors.As` check while retaining their existing `errors.Is` classification.

125
docs/releases/v0.5.0.md Normal file
View File

@@ -0,0 +1,125 @@
# Promptkit v0.5.0
This supplemental changelog and migration guide summarizes the consumer-facing
changes from `v0.4.0` to `v0.5.0`. The annotated `v0.5.0` tag is the
authoritative release record. Exact current contracts belong to the linked
GoDoc and durable documentation.
## Summary
`v0.5.0` makes provider requests less prescriptive and adds an application
fallback layer for profile definitions:
- unset optional provider controls are omitted from OpenAI-compatible request
bodies instead of being populated with framework values; and
- `WithFallbackProfileFS` lets an application package profile defaults that
operators can override through the existing ordinary profile sources.
These changes let compatible providers apply their own model defaults while
giving applications stable embedded profile IDs without weakening operator
configuration precedence.
## Compatibility
The release adds one public function and removes no public declaration.
Existing source code should continue to compile.
There is one intentional behavior change: when no profile or runtime override
selects `top_p`, Promptkit no longer sends the former framework value of `1`.
It omits `top_p` and lets the provider choose its behavior. Unset
`temperature` and `max_tokens` are likewise omitted. Explicit nonzero profile
values and runtime values—including explicit runtime zero values—retain their
precedence and wire effect.
Consumers that relied on Promptkit always sending `top_p: 1` should add that
value to the relevant profile or runtime override before upgrading. Consumers
that did not rely on the implicit sampling value require no migration.
Application fallback profiles are opt-in. Engines that do not call
`WithFallbackProfileFS` retain the previous profile-source behavior.
## Upgrade
Update the module dependency with:
```sh
go get gitea.maximumdirect.net/eric/promptkit@v0.5.0
go mod tidy
```
Run the consuming project's ordinary and race-enabled tests after upgrading.
If request payloads or model behavior are asserted in fixtures, review them for
the optional-parameter omission described below.
## Omitted Optional Provider Controls
The built-in OpenAI-compatible client now includes `temperature`,
`max_tokens`, and `top_p` only when a profile or runtime override selects the
value. An explicit runtime zero remains present because runtime override
pointers distinguish zero from an unspecified value.
Promptkit's positive generation deadline remains a framework concern and is
not a provider request-body default. Required request fields, session IDs,
structured output, reasoning selection, credentials, and explicit extra
parameters retain their existing behavior.
See the [framework default and precedence reference](../formats.md#defaults-and-overrides),
the [`ExecutionTargetOverride` GoDoc](../../types.go), and the
[OpenAI-compatible request-body contract](../integrations/openai-compatible-chat.md#request-body)
for current details.
## Embedded Application Fallback Profiles
Applications can package ordinary profile YAML in an `fs.FS` and register it
as a fallback source:
```go
//go:embed profiles/*.yaml
var applicationProfiles embed.FS
engine, err := promptkit.NewEngine(promptkit.Config{
PromptDir: "prompts",
ProfileDir: operatorProfileDir,
},
promptkit.WithFallbackProfileFS(applicationProfiles, "profiles"),
)
```
Leave `operatorProfileDir` empty when no operator source is configured. A
configured ordinary source is authoritative: a matching definition overrides
the application fallback, while a read or validation failure remains an error
instead of silently reaching a lower layer.
Profile definitions resolve in this order:
1. in-memory profiles supplied with `WithProfiles`;
2. the ordinary configured source selected by `WithProfileFile`,
`WithProfileFS`, or `Config.ProfileDir`;
3. the application source supplied with `WithFallbackProfileFS`; and
4. Promptkit's embedded built-in profiles.
Only an absent profile ID falls through. Sources provide complete profiles and
do not merge fields. Loading remains lazy, and the new source uses the existing
strict profile YAML and credential rules.
See the
[embedded-default consumer guidance](../consumers/pkg-promptkit.md#supply-embedded-application-defaults),
the [`WithFallbackProfileFS` GoDoc](../../engine.go), and the
[profile source reference](../formats.md#source-and-profile-precedence) for
current details.
## Public API Changes
The release adds:
- `WithFallbackProfileFS`.
No public declaration was removed or changed.
## Consumer Action
- Review any workflow that depended on Promptkit's implicit `top_p: 1` and
configure the value explicitly when required.
- Optionally adopt `WithFallbackProfileFS` when an application should package
overridable profile defaults.
- Run consumer tests after updating the module dependency.

View File

@@ -0,0 +1,635 @@
# Codebase Audit Sequence
## Purpose
This document defines the staged sequence for auditing Promptkit before further
feature development. The audit is intended to identify high-confidence
opportunities to improve correctness, efficiency, duplication, implementation
clarity, and test-suite quality without changing production behavior during the
review itself.
The audit findings belong in `audit.md`. A later, separate planning pass will
translate accepted findings into a staged remediation plan in
`implementation.md`. Neither this sequence nor the findings log owns current
behavior; the canonical sources identified by the
[documentation policy](../policy/documentation.md) remain authoritative.
Each stage below is deliberately scoped for one LLM coding-agent prompt. Run
the stages in order and do not combine them. A stage may discover a concern
outside its scope, but it should record that concern for the owning later stage
rather than expanding its own review.
## Governing Policies And Boundaries
Every stage must follow:
- the [development guide](../development.md), including its task-specific
reading guide;
- the [architecture policy](../policy/architecture.md), especially the public
facade, internal-package, dependency-direction, and consumer boundaries;
- the [testing policy](../policy/testing.md), including its risk-based,
behavior-oriented standard; and
- the [documentation policy](../policy/documentation.md), including canonical
ownership and the temporary nature of roadmap documents.
This is an audit, not an implementation pass:
- Do not change production code, tests, examples, fixtures, public contracts,
or current-state documentation.
- Limit repository edits to the audit artifacts explicitly authorized for the
stage.
- Do not silently repair an issue while investigating it.
- Do not treat coverage, complexity, similarity, lint, or graph output as a
finding without confirming the underlying behavior in source and tests.
- Do not recommend centralization merely because code looks similar. The code
must implement the same semantic rule, and consolidation must improve
ownership or reduce a credible drift risk.
- Do not recommend performance work without identifying a relevant execution
path and establishing a defensible cost model, measurement, or complexity
problem.
- Preserve unrelated working-tree changes. Record the audit baseline rather
than requiring an otherwise unrelated dirty tree to be cleaned.
## Finding Standard
Record each actionable finding in `audit.md` with:
- a stable ID in the form `SNN-FNN`, where the first number is the stage;
- category: correctness, efficiency, duplication, clarity, testing, or
contract-documentation consistency;
- severity: critical, high, medium, or low;
- confidence: confirmed, high, medium, or low;
- affected packages, files, symbols, and tests;
- the contract, invariant, policy, or maintenance concern at issue;
- concrete evidence and a concise explanation of the failure mode or cost;
- the recommended direction, without implementation-level sequencing;
- the verification or regression protection that remediation would require;
and
- status: accepted, deferred, rejected, superseded, or resolved.
Use **confirmed** confidence when the problem is reproduced or follows
unavoidably from a complete trace. Use **high** confidence when direct source
and test evidence establishes the problem but a safe reproduction is not
practical. Medium- and low-confidence concerns belong in a separate
observations section until a later stage confirms or rejects them; they must
not enter the remediation plan as if they were findings.
Severity describes impact, not implementation effort:
- **Critical:** credible data disclosure, data corruption, deadlock, unbounded
resource consumption, or a broadly unusable public contract.
- **High:** violation of an important public contract or invariant, a likely
concurrency or resource-lifecycle defect, or a failure with substantial
downstream impact.
- **Medium:** a real but narrower behavioral defect, meaningful avoidable cost,
duplicated policy with credible drift risk, or a material testing gap.
- **Low:** a bounded clarity, maintainability, or testing-friction problem with
a concrete improvement and little behavioral risk.
When a reviewed area yields no finding, record the important behavior or risk
that was inspected and found adequately implemented or tested. This coverage
ledger prevents later reviewers from mistaking silence for omission.
## Per-Stage Procedure
Unless a stage says otherwise, its single agent prompt should:
1. Read the required policies, focused internal documentation, production
files, and tests for that stage.
2. Use the code knowledge graph for symbol discovery, callers, callees, and
cross-package traces; confirm important conclusions against source.
3. Trace normal, boundary, and failure paths through the narrowest relevant
public or package contract.
4. Review correctness, meaningful runtime cost, semantic duplication,
responsibility clarity, and the value and ownership of tests in scope.
5. Run the narrowest existing tests needed to validate conclusions. Use
race-enabled or repeated focused tests when concurrency or nondeterminism is
in scope. Do not add permanent tests during the audit.
6. Add the stage result to `audit.md`: accepted findings, unresolved
observations, areas verified, commands run, and any handoff to a later
stage.
7. Recheck the working tree and confirm that only the authorized audit artifact
changed.
## Stage 0: Initialize The Audit And Establish The Baseline
Create `audit.md` and establish a reproducible starting point before reviewing
individual components.
Record:
- the audited commit, branch, Go version, module identity, and working-tree
state;
- unrelated pre-existing changes that all later stages must preserve;
- the implemented package and public-facade inventory;
- the baseline validation results; and
- the finding template, status vocabulary, and coverage ledger used by later
stages.
Refresh the code knowledge graph for the recorded commit. Run the repository's
ordinary tests, race tests, vet, build, maintained offline preparation example,
Go formatting check, Markdown link check, and repository-hygiene checks. Run
package coverage once as a diagnostic and record the result without defining a
coverage target or committing generated output. Measure coarse package test
duration only if it can be done without adding tooling or changing tests.
Compare the validation requirements stated by the testing policy, development
guide, and release procedure. Record a finding if their ownership or command
sets are materially inconsistent; do not edit those documents in this stage.
**Exit condition:** `audit.md` contains the baseline, ledger structure, and
validation result, and no component-level audit has begun.
## Stage 1: Public Values, Conversion, Errors, And Formatting
Review the root facade's public request, result, inspection, prepared-run, and
error values together with public-to-internal and internal-to-public
conversion. Scope the review to `doc.go`, `types.go`, `convert.go`, `errors.go`,
`capacity_error.go`, `formatting.go`, and `prepared_execution.go`, plus the
directly relevant portions of root tests.
Focus on:
- zero-value and nil behavior;
- defensive copying, aliasing, and immutable snapshots;
- lossless conversion and field precedence;
- error identity through `errors.Is` and `errors.As`;
- containment of internal representations;
- safe `String`, `GoString`, and diagnostic formatting;
- accidental disclosure of credentials, prompt content, generated content, or
other private state; and
- conversion or copying logic that represents the same rule in multiple
places.
Review only tests that own these value and boundary contracts. Defer engine
assembly, execution coordination, and transport behavior to their later
stages.
**Exit condition:** all root value-conversion and error-formatting paths have a
recorded audit result without evaluating engine orchestration.
## Stage 2: Public Configuration And Extension Adapters
Review the smaller public construction and extension surfaces in
`backends.go`, `profiles.go`, `artifact_reader.go`, `json.go`, and
`llm_adapter.go`, together with their directly relevant root and internal
adapter tests.
Focus on:
- validation performed at the public boundary;
- ownership and copying of caller-supplied maps, slices, filesystems, readers,
and clients;
- adapter error propagation and cancellation;
- consistency between convenience constructors and general configuration;
- whether extension interfaces are as narrow as their consumers require;
- whether public helpers duplicate internal policy or merely translate it;
and
- whether tests protect consumer-visible behavior rather than private adapter
choreography.
Do not review how `NewEngine` combines these values; that belongs to Stage 3.
**Exit condition:** every non-engine public configuration helper and adapter
has a recorded result and any assembly questions are handed to Stage 3.
## Stage 3: Engine Construction, Options, And Source Assembly
Review the construction and configuration portions of `engine.go` and the
corresponding tests in `engine_test.go`. Limit the scope to `NewEngine`, option
application, dependency defaults, backend registration, profile and prompt
source composition, fallback-profile placement, validator and client
selection, capacity-manager construction, and construction-time validation.
Focus on:
- deterministic option precedence;
- required versus optional dependencies;
- isolation between engine instances;
- freezing or copying consumer configuration at the correct boundary;
- correct dependency direction and absence of process-global mutable state;
- failure atomicity and useful public errors;
- consistency between configured backends and capacity policies; and
- assembly logic that is repeated or split across unclear owners.
Do not audit the runtime behavior of `Run`, `Prepare`, or inspection methods;
that belongs to Stage 4 and the internal use-case stages.
**Exit condition:** engine construction and source assembly are fully accounted
for, including tests, without expanding into runtime orchestration.
## Stage 4: Engine Operations And Root Contract Coverage
Review the remaining public methods in `engine.go` and their directly relevant
root tests, including the external-package contracts in
`public_contract_test.go` and `prepared_execution_contract_test.go` only where
they exercise the engine boundary under review.
Focus on:
- request translation and context propagation;
- ordinary run, preparation, inspection, and prepared-execution entry points;
- public error mapping and preservation of injected dependency errors;
- result and prepared-state ownership;
- consistency between method and package-level convenience functions;
- public behavior that is asserted redundantly in root internal tests and
external-package contract tests; and
- important public behavior that is tested only through internal packages.
Treat internal runner, transport, validation, and capacity mechanics as black
boxes in this stage. Hand questions about their implementation to their owning
later stages.
**Exit condition:** the public execution boundary and its contract-test
ownership are recorded without duplicating internal component audits.
## Stage 5: Internal Domain And JSON-Compatible Values
Review `internal/domain` and `internal/jsonvalue`, including all of their tests.
Focus on:
- domain invariants and invalid states;
- session normalization;
- prepared-run and schema immutability;
- deep-copy correctness for every supported JSON-compatible shape;
- numeric-type preservation and rejection policy;
- cycles, excessive nesting, unsupported values, and nil distinctions;
- avoidable repeated copying on execution paths; and
- whether generic value machinery has a single clear owner.
Trace important callers to confirm that these packages enforce the invariants
their consumers assume, but do not audit the callers' broader behavior.
**Exit condition:** shared value semantics and their test ownership are fully
recorded.
## Stage 6: Backend Registry, Defaults, And Built-In Profiles
Review `internal/backend`, `internal/defaults`, and
`internal/profile/builtin`, including their focused tests and the relevant
backend-policy traces into engine assembly and the LLM reserved-field rule.
Focus on:
- immutable registry construction and lookup;
- built-in versus consumer ID collision rules;
- endpoint, credential-environment, header, parameter, and concurrency
validation;
- defensive copies at registry boundaries;
- application-neutral default ownership;
- built-in profile/backend consistency;
- reserved request-field ownership without dependency inversion; and
- duplicated validation or default policy across public and internal layers.
Defer scheduling mechanics to Stage 15 and actual HTTP request construction to
Stage 14.
**Exit condition:** registry and default-policy correctness are recorded, with
transport and scheduling questions handed to their owning stages.
## Stage 7: File Discovery And Prompt Definitions
Review `internal/filecatalog` and `internal/promptdef`, including their tests
and fixtures. Read the framework format reference and internal source document
before evaluating behavior.
Focus on:
- deterministic discovery and duplicate handling;
- filesystem and `fs.FS` parity;
- root and relative-path normalization;
- strict YAML decoding and version selection;
- prompt ID, message, input, cache-control, and validation declarations;
- inline versus file-backed content rules;
- containment of referenced files where promised;
- malformed input and contextual error behavior;
- unnecessary repeated directory scans or file reads; and
- fixture and case duplication that does not protect distinct parser risks.
Do not audit rendering, artifact loading, profile loading, or schema validation
in this stage.
**Exit condition:** discovery and prompt-definition parsing have complete
findings and coverage-ledger entries.
## Stage 8: Profile Sources And Repository Composition
Review `internal/profile` excluding its built-in subpackage, including all
repository tests and profile fixtures. Read the profile format contract first.
Focus on:
- strict decoding and profile validation;
- filesystem and `fs.FS` parity;
- repository overlay and fallback precedence;
- distinction between absence and a malformed authoritative source;
- preservation of useful error identity and context;
- conversion to immutable execution profiles;
- duplicate IDs and deterministic selection;
- repeated parsing, validation, or copying; and
- whether tests at repository, engine, and public-contract layers have clear,
nonduplicative ownership.
Defer resolution of a profile with runtime overrides and backend definitions to
Stage 11.
**Exit condition:** profile-source and repository-composition behavior are
fully recorded.
## Stage 9: Artifact Loading And Prompt Rendering
Review `internal/artifact` and `internal/prompt`, including all focused tests.
Read the internal source document and format reference first.
Focus on:
- inline and file artifact ownership, metadata, hashing, and error behavior;
- copied versus shared byte storage;
- caller-selected path semantics and architecture-policy boundaries;
- template parsing and execution;
- artifact, variable, session, and cache-control rendering;
- missing, extra, nil, and malformed input behavior;
- deterministic output and safe diagnostics;
- unnecessary repeated reads, hashes, parses, or allocations on common paths;
and
- tests coupled to incidental template or struct implementation.
Do not audit the runner's decision about when rendering occurs.
**Exit condition:** input materialization and rendering are accounted for
through their package boundaries.
## Stage 10: Output Validation And Frozen Validation Plans
Review `internal/validate`, including all tests, schema fixtures used by the
root contract suite, and traces from preparation into frozen validation plans.
Read the format and internal source documents first.
Focus on:
- basic, JSON, and JSON Schema mode semantics;
- schema-path resolution and filesystem/`fs.FS` parity;
- schema compilation, transitive references, and source-lifetime independence;
- output normalization and preservation;
- malformed schema and malformed model-output errors;
- thread safety of reusable validators and prepared plans;
- expensive recompilation or copying on repeated execution; and
- whether parser, validator, runner, and public tests each own distinct risks.
Do not audit repair decisions or provider request construction.
**Exit condition:** validation behavior, plan lifetime, and focused test value
are fully recorded.
## Stage 11: Inspection And Execution-Target Resolution
Review `internal/usecase/profile_inspection.go`,
`internal/usecase/prompt_inspection.go`, and the preparation and target-
resolution portions of `internal/usecase/runner.go`, together with their
focused tests. Use graph traces to define the exact helper and call-path scope
before reviewing.
Focus on:
- prompt and profile selection;
- backend lookup and endpoint overrides;
- reasoning, session, and other runtime precedence;
- merge semantics for default, profile, backend, and per-run values;
- inspection fidelity versus actual execution;
- credential-name versus credential-value handling;
- prompt-definition and schema freezing during preparation;
- stable error identity and context; and
- duplicated resolution rules across inspection, preparation, and execution.
Do not review model invocation, repair execution, or prepared-handle lifecycle;
those belong to Stages 12 and 13.
**Exit condition:** all selection, merge, inspection, and preparation rules are
traced and recorded once.
## Stage 12: Ordinary Execution, Validation, And Repair Coordination
Review `internal/usecase/runner.go`, `internal/usecase/repairer.go`, and
`internal/usecase/capacity_error.go` only for the ordinary execution path after
preparation, together with the corresponding sections of `runner_test.go`.
Use the Stage 11 resolution result as an established input rather than
reauditing it.
Focus on:
- rendering, generation, validation, and optional repair transitions;
- context cancellation and dependency-error propagation;
- partial result and usage accounting;
- exact attempt count and repair eligibility;
- avoidance of unintended retries;
- capacity-error translation;
- cleanup and failure behavior on every exit path;
- repeated orchestration or request construction; and
- oversized tests, helpers, or case matrices that obscure distinct behavior.
Treat LLM transport and capacity scheduling as injected package contracts;
their mechanics belong to Stages 14 and 15.
**Exit condition:** the ordinary execution state machine and its test ownership
are fully recorded.
## Stage 13: Prepared Execution Lifecycle
Review `internal/usecase/prepared_execution.go`, its focused tests, and the
prepared-execution portions of the root facade and external contract tests.
Do not repeat the public value review from Stages 1 and 4 or the resolution
review from Stage 11.
Focus on:
- single-attempt or other lifecycle guarantees;
- concurrent use and synchronization;
- discard behavior and resource release;
- frozen source, target, credential, capacity, timing, and schema semantics;
- independence of returned details and results;
- context and error behavior;
- consistency between ordinary and prepared execution where promised;
- private-state containment in formatting; and
- redundant assertions across internal, root, and external-package tests.
Run focused race tests and repeated tests for lifecycle behavior where useful.
**Exit condition:** prepared execution has one complete lifecycle analysis and
a clear map of which test layer owns each guarantee.
## Stage 14: OpenAI-Compatible Transport
Review `internal/llm`, including all transport tests. Read the
OpenAI-compatible integration contract and internal LLM document first.
Focus on:
- request endpoint, headers, authentication, and JSON body construction;
- omission versus explicit zero-value behavior;
- reserved-field enforcement and extra-parameter collision handling;
- session ID and reasoning encoding;
- structured-output and cache-control translation;
- client and per-generation deadlines;
- cancellation, body closure, bounded response reads, and decode failures;
- non-success HTTP response behavior;
- response choices, usage, and malformed-success handling;
- wire-visible compatibility and safe error disclosure;
- unnecessary marshaling, copying, or buffering; and
- whether the large transport test file can be simplified without losing
protocol-risk coverage.
Use `httptest`-based existing tests; do not contact a live provider.
**Exit condition:** every outbound and inbound wire path has a recorded result,
including focused test ownership.
## Stage 15: Capacity, Admission, And Concurrency
Review `internal/capacity`, its tests, `capacity_contract_test.go`, and the
integration points already identified in engine and use-case stages. Read the
internal capacity document first.
Focus on:
- bounded run admission and queue-capacity enforcement;
- per-backend limited and unlimited scheduling;
- FIFO behavior and cancellation-safe waiter removal;
- permit release on success, error, panic-relevant boundaries, and
cancellation;
- goroutine, timer, and waiter lifecycle;
- starvation, deadlock, race, and engine-isolation risks;
- lock scope and meaningful contention or allocation costs;
- preservation of injected-client concurrency where promised;
- relational testing of configured limits rather than duplicated defaults;
and
- duplication between internal concurrency tests and public contract tests.
Run focused ordinary, race-enabled, and repeated tests. Repetition must remain
bounded and diagnostic; a test that passes many times is not proof of
correctness without a source-level synchronization analysis.
**Exit condition:** concurrency invariants have both a source trace and a
test-ownership assessment.
## Stage 16: Repository-Wide Test Strategy And Maintained Examples
Perform a suite-level review after every component has been audited. Review
the testing policy, test inventory, fixtures, external-package root tests,
`architecture_test.go`, and both maintained examples. Use the component-stage
coverage ledger instead of repeating every individual test assertion.
Construct a risk-to-owner matrix for:
- public compatibility and error identity;
- parsing, validation, and serialization;
- immutability and data integrity;
- external wire behavior;
- cancellation, failure propagation, and recovery;
- concurrency and resource lifecycle; and
- representative assembled consumer workflows.
Identify only evidence-backed cases of:
- consequential behavior with no credible test owner;
- the same semantic rule asserted redundantly at several layers;
- tests coupled to private helpers, internal constants, exact noncontractual
wording, or collaborator choreography;
- low-value or obsolete cases whose lifetime cost exceeds their protection;
- missing failure, cancellation, race, or boundary coverage;
- nondeterminism, shared state, environment dependence, fixed ports, or test
ordering assumptions;
- helpers and fixtures whose complexity is not justified; and
- maintained examples that duplicate one another without protecting distinct
workflows.
Use coverage and timing only to direct attention. Do not propose tests solely
to raise percentages or remove tests solely to shorten the suite.
**Exit condition:** every important risk has a named test owner or an accepted
finding, and every proposed test deletion or consolidation states what
protection remains.
## Stage 17: Cross-Cutting Duplication, Efficiency, And Architecture Review
Review the codebase as a whole using the completed component findings, graph
traces, complexity signals, similarity signals, and package dependency map.
Do not reopen settled package behavior without new cross-cutting evidence.
Focus on:
- one semantic policy implemented by multiple packages;
- repeated public/internal transformations with credible drift risk;
- interfaces broader than their actual consumers;
- responsibilities split across packages or concentrated in the facade
contrary to the architecture policy;
- repeated parsing, copying, schema compilation, request construction, or
source traversal on important paths;
- avoidable lock contention or serial work supported by the concurrency audit;
- abstractions that add indirection without enforcing a boundary; and
- discrepancies between implemented package responsibilities and their
canonical architecture or internal documentation.
For each possible consolidation, state why the code represents one rule, which
package should own it, and why the resulting dependency direction remains
valid. For each efficiency finding, state the path frequency, input scale,
complexity or measurement evidence, and the benchmark or invariant needed to
verify a remediation.
**Exit condition:** all cross-cutting opportunities are either accepted with
high confidence, retained as explicitly lower-confidence observations, or
rejected with a short rationale.
## Stage 18: Consolidate And Close The Audit
Perform a findings-only synthesis. Do not change code and do not write the
remediation plan yet.
- Recheck every accepted finding against the final audited tree.
- Merge duplicates and mark superseded IDs without erasing their history.
- Separate shared root causes from downstream symptoms.
- Confirm that every accepted item is confirmed or high confidence.
- Confirm that severity describes impact rather than effort.
- Reject speculative cleanup, coverage-driven test work, and centralization
without a clear owner or drift risk.
- Record dependencies and a recommended remediation order.
- Distinguish behavioral fixes, safe refactors, performance work, test gaps,
test consolidation, and documentation synchronization.
- Add an audit summary stating what was reviewed, what validation ran, the
accepted finding counts by category and severity, and any residual
uncertainty.
- Re-run baseline validation if audit-only investigation could have affected
repository state, and confirm that only authorized roadmap files differ from
the recorded baseline.
The recommended ordering should place correctness, data-integrity,
resource-lifecycle, and concurrency defects first; policy duplication and
missing protection for consequential behavior next; then clarity, test
consolidation, and demonstrated efficiency improvements. Actual implementation
stages must be decided in the later `implementation.md` planning pass, where
files, dependencies, acceptance criteria, and validation can be made
decision-complete.
**Exit condition:** `audit.md` is a complete, internally consistent input to a
separate remediation-planning prompt, with no code or test changes mixed into
the audit.
## Completion Criteria
The audit is complete only when:
- every production component and public boundary appears in the coverage
ledger;
- every test file and maintained example has been reviewed at its owning stage
or in the suite-wide stage;
- important cross-package paths have been traced end to end;
- concurrency-sensitive behavior has received source and race-test review;
- every accepted finding meets the evidence and confidence standard;
- lower-confidence observations are visibly separated from remediation
candidates;
- proposed test additions, deletions, and consolidations are justified against
the testing policy;
- proposed simplifications identify a durable responsibility owner;
- proposed efficiency work has a relevant cost model or measurement plan; and
- the repository remains unchanged except for the authorized audit roadmap
artifacts.

4606
docs/roadmap/audit.md Normal file

File diff suppressed because it is too large Load Diff

61
docs/roadmap/deferred.md Normal file
View File

@@ -0,0 +1,61 @@
# Deferred Feature Ideas
## Purpose
This document catalogs feature ideas that remain potentially useful but have
been deliberately postponed. These ideas are not awaiting ordinary selection
from the [future feature catalog](future.md); each has a stated reason to wait
and should be reconsidered only when its trigger becomes relevant.
Deferred entries are not commitments, schedules, active implementation plans,
or descriptions of current behavior. When an entry is reactivated, move it to
`future.md` for evaluation or directly into a focused roadmap after its open
design dependencies have been resolved.
## Deferred Ideas
### Semantic Execution-Target Fingerprints
**Reason for deferral:** A stable digest requires a deliberate semantic-
equality and versioning design. Notarius can safely use conservative source
hashes and a Promptkit release marker today, while Weatherreporter does not
currently reuse LLM-dependent checkpoints.
Promptkit could expose an opaque equality value for a resolved profile and its
effective generation target. This would let checkpointing consumers detect
generation-affecting configuration changes without hashing YAML presentation
or depending on Promptkit's built-in catalog layout.
The digest should change with semantically relevant state such as the resolved
model, endpoint, backend routing identity, request defaults, extra parameters,
profile generation settings, and selected built-in profile semantics. It
should exclude credential values, concurrency and queue policy, source paths,
comments, formatting, and other representation-only changes. Whether a
credential environment-variable name affects equality must be decided
explicitly. The encoding should remain opaque and internally versioned so
Promptkit can deliberately invalidate earlier digests when its resolution
semantics change.
Reconsider this idea when a downstream consumer needs Promptkit-owned
checkpoint equality or when a broader semantic identity design is selected.
### Eager Source Validation
**Reason for deferral:** Exact prompt and profile inspection may already
provide a sufficiently small validation surface. Experience from downstream
adoption should establish whether an engine-wide operation would add enough
value to justify its broader contract.
Promptkit could provide an explicit offline operation that discovers and
structurally validates configured prompt, profile, and schema sources without
model generation. The normal `NewEngine` path would remain lazy.
An eager operation would need coherent handling for duplicate prompt IDs and
versions, strict YAML decoding, referenced content files, profile/backend
membership, schema syntax and transitive references, context cancellation,
and source-specific public errors. Credential declarations must remain
separate from credential values; checking current environment availability,
if supported at all, should be an explicit option and must not expose secrets.
Reconsider this idea after downstream use of `InspectPrompt`,
`InspectProfile`, and fixture-based preparation demonstrates a concrete gap.

82
docs/roadmap/future.md Normal file
View File

@@ -0,0 +1,82 @@
# Future Feature Ideas
## Purpose
This document catalogs reasonably specific ideas that may be useful in future
Promptkit development. It is an idea pool, not a commitment, schedule, or
description of current behavior.
Ideas belong here while they are worth retaining but have not been selected
for active development. Keep each entry at the level of intended capability,
consumer value, and important scope boundaries. Defer API design,
implementation details, sequencing, and acceptance criteria until an idea is
selected.
Ideas that have been deliberately postponed rather than left available for
ordinary selection belong in the [deferred catalog](deferred.md).
## Using This Catalog
- Add an idea when its purpose and likely value can be stated clearly.
- Keep entries independent enough that maintainers can evaluate and select
them individually.
- Note significant dependencies or boundary concerns, but do not turn entries
into implementation plans.
- Treat inclusion as an invitation to evaluate, not as approval or priority.
- When an idea is selected, move its active planning to a focused roadmap or,
when it requires a durable architectural decision, an ADR. Update
current-state documentation only when implementation lands.
- Move an idea to `deferred.md` when maintainers decide to retain it but wait
for a stated design dependency, demand signal, or reconsideration trigger.
- Remove ideas that are no longer relevant. Retain a rejected idea only when
its rationale is likely to prevent repeated reconsideration.
Future capabilities must continue to respect the
[architecture policy](../policy/architecture.md), particularly Promptkit's
role as an application-neutral library and its boundary with downstream
consumers.
## Ideas
### Public bounded output repair
After the codebase-audit remediations are complete, Promptkit should make its
bounded output-repair capability available through the public engine. A
consumer should be able to request a limited number of corrective generation
attempts when JSON or JSON Schema output fails content validation, without
having to reproduce Promptkit's generation, validation, capacity, and result-
accounting orchestration.
- Repair is validation recovery, not a general provider retry, failover, or
backoff policy. Transport failures, cancellation, and operational schema or
validation errors must retain their ordinary error behavior.
- Repair must stop after the first valid result or the configured attempt
bound. Exhausting the bound should preserve the final invalid result and its
validation diagnostics rather than inventing success.
- Initial generation and every repair attempt must use the same resolved
backend, effective execution settings and presence semantics, session,
credential boundary, structured-output contract, and backend-capacity
policy.
- Results should report the number of repair attempts and cumulative usage for
every model call made by the run.
- Ordinary and prepared execution should expose coherent behavior, including
cancellation, frozen prepared state, error identity, and capacity lifetime.
Select this work only after the accepted audit findings affecting shared
execution invariants, validation, orchestration, transport, and repair
internals have been remediated.
## Entry Format
Use a short heading followed by a concise summary. Add focused bullets when
they help preserve important scope boundaries without becoming an
implementation plan:
```markdown
### Idea name
Describe the intended capability, who benefits, and the most important scope
boundary or dependency.
- Optionally record an important behavior or boundary.
```

View File

@@ -0,0 +1,840 @@
# Audit Remediation Implementation Plan
## Purpose
This document is the decision-complete implementation plan for the accepted
findings in the [codebase audit](audit.md). It is written for a
`gpt-5.6-terra` coding agent that will implement one numbered stage per prompt,
in order.
The audit remains the evidence and rationale for each finding. This plan owns
implementation order, selected policy decisions, required code and test work,
and stage gates. It does not activate the future public output-repair feature
described in the [future feature catalog](future.md); it only corrects and
protects the retained internal repair machinery on which that later feature
may build.
## Implementation Policies
Every stage must follow the [development guide](../development.md),
[architecture policy](../policy/architecture.md),
[testing policy](../policy/testing.md), and
[documentation policy](../policy/documentation.md). Before changing a
subsystem, read the focused current-state documents identified by the
development guide and inspect the exact implementation and tests named by the
stage.
Apply these rules throughout:
- Implement exactly one stage per agent prompt. Do not combine stages or begin
a later stage early.
- Inspect the working tree before editing and preserve unrelated changes.
- Use the code knowledge graph to locate symbols, callers, and dependency
paths; confirm important conclusions against source.
- Keep the root package as the public facade and implementation under
`internal/`. Do not expose internal representations or add a public package.
- Put source-neutral invariants in their assigned internal owner while
preserving source-specific normalization, error classification, and public
translation at existing boundaries.
- Add regression protection at the narrowest stable owner in the same stage
as a behavioral fix. Retain only representative integration coverage at
higher layers.
- Do not add tests to raise coverage percentages. Do not preserve tests that
assert an incidental algorithm, private constant, dormant serialization
shape, or duplicated lower-layer truth table.
- Keep all tests deterministic, offline, race-safe, credential-free, and free
of fixed-port or mutable-service assumptions.
- Do not add engine-wide caches, generic facade abstractions, scheduler
changes, provider retry policy, or new consumer configuration unless a stage
explicitly requires it.
- Update canonical GoDoc and current-state documents in the same stage as the
behavior they describe. Do not describe a later stage as already
implemented.
- Format changed Go files. Run the stage's focused commands, then at least
`go test ./...` and `go vet ./...`. Run focused race tests wherever the
stage changes ownership, cancellation, shared state, or lifecycle behavior.
- Do not commit, push, tag, or publish unless separately instructed.
## Decisions Fixed By This Plan
The implementing agent must not reopen these choices:
1. **JSON-compatible numbers:** accept every value Go can faithfully encode as
a JSON number: every signed and unsigned integer width, finite `float32` and
`float64` values, and a `json.Number` whose text is valid JSON-number syntax.
Do not impose the current IEEE-754 safe-integer restriction. Reject NaN,
infinities, and malformed `json.Number` text. Preserve supported concrete
numeric types when copying.
2. **JSON-shaped traversal bounds:** allow at most 100 JSON container levels
and 100,000 produced JSON value nodes per `Copy` or `CopyMap` operation.
Count the root, each map/slice/array container, and every produced child
value; map keys are not separate nodes. Pointer and interface indirection
do not add JSON depth or an extra node. Repeated appearances of an acyclic
shared value count each produced occurrence. Continue rejecting active-path
cycles and return deterministic, path-aware validation errors on either
bound.
3. **Execution timeout bound:** a positive `TimeoutSeconds` must fit in
`time.Duration` after multiplication by `time.Second`. Derive the maximum
from `math.MaxInt64` and `time.Second`; do not duplicate its numeric literal
in tests or documentation.
4. **Output contracts:** the only valid formats are `text`, `markdown`, and
`json`; the only valid validation modes are `none`, `basic`, `json`, and
`json_schema`; repair attempts are non-negative; and `json_schema` requires
a nonblank schema path. A non-nil request replacement defaults an empty
format to `text` before shared validation. It does not default an empty
validation mode.
5. **Prompt content paths:** every `content_file` is an exact, relative path
resolved from its prompt file and contained by the configured prompt source
root. Directory, `fs.FS`, and single-file sources all reject absolute and
escaping paths. A single-file source's root is the containing directory of
that selected prompt file. Trimming determines only whether a value is
blank; it must not change the path opened. OS containment must account for
symlinks; containment inside an injected `fs.FS` remains expressed in that
filesystem's namespace.
6. **Profile IDs:** normalize file-backed IDs with `strings.TrimSpace` once,
just as in-memory IDs are normalized. Use the normalized value for
selection, duplicate detection, results, and diagnostics. A whitespace-only
ID is invalid, and IDs that become equal after normalization are
duplicates.
7. **Ordinary artifact files:** the built-in `File` reader supports regular
files, including symlinks whose targets are regular files. It rejects
directories, FIFOs, devices, sockets, and other non-regular targets before
consuming them. It remains unrestricted by an application root and does
not introduce an application-specific byte limit.
8. **Validation cancellation:** do not return early by abandoning goroutines
around `fs.FS` or the JSON Schema dependency. Promptkit must check
cancellation before, between, and after work it controls; read opened files
in context-checked chunks; and let a canceled context win before publishing
a result after synchronous decode, compile, or validation calls. Go's
`fs.FS` and the current JSON Schema library expose no general mechanism to
preempt a blocked `Open`, `Read`, compile, or validation method, so canonical
documentation must describe this synchronous limitation rather than claim
impossible asynchronous interruption.
9. **Successful provider-response limit:** the built-in OpenAI-compatible
client accepts at most 16 MiB (`16 << 20` bytes) for the complete successful
HTTP response body, including surrounding whitespace. The limit is fixed,
internal, and application-neutral. Exactly the limit is allowed; the first
byte beyond it fails as `internal/llm.ErrMalformedResponse`. Do not add a
public setting. Non-success response parsing remains outside this audit
remediation and belongs to the separate structured-generation-error
roadmap.
10. **Repair machinery:** retain and fix the internal repairer, cumulative
usage, and bounded repair state machine. The public engine must continue to
install no repairer and remain single-pass. Do not activate public repair
in this plan.
## Stage 1: Centralize Execution-Setting And Session Invariants
**Findings:** S05-F01, S17-F01, S14-F02. This stage also resolves the
source-specific evidence in S02-F02, S08-F01, and S11-F01.
Add a source-neutral execution-setting validator to `internal/domain`. It must
validate temperature, maximum tokens, top-p, and timeout on a domain execution
target: temperature and top-p must be finite and within their closed ranges,
maximum tokens must be non-negative, and timeout must be non-negative and no
greater than the derived duration-safe maximum. Keep optional-pointer presence,
profile required fields, normalization, and error wrapping outside this
validator.
Use that owner from:
- in-memory profile validation in the root package;
- OS and `fs.FS` profile validation;
- resolved request/target validation in `internal/usecase`; and
- the built-in model client as a defensive final boundary.
Remove the duplicated scalar comparisons from those callers. Preserve
`ErrInvalidConfig` for in-memory construction, profile-load identities for file
profiles, `ErrInvalidRequest` for runtime overrides, and the LLM package's
defensive invalid-request identity. Explicit numeric zero must retain its
presence semantics.
Update `internal/domain.NormalizeSessionID` to reject invalid UTF-8 before
trimming or rune counting. Preserve the existing blank and 256-code-point
rules. Direct requests must still map failures to `ErrInvalidRequest`, while
session-template failures remain renderer failures.
Add one domain-owned table for every exact setting boundary, finite neighbors,
NaN, both infinities, negative values, and the timeout representability edge.
Retain small boundary-integration cases for in-memory profiles, both file
source forms, request overrides through `Prepare` and `PrepareExecution`, and
the model-client defense. Add malformed UTF-8 session cases before, within,
and after otherwise valid content.
Update the architecture policy and internal component overview so
`internal/domain` explicitly owns source-neutral invariants for its shared
execution values, without claiming ownership of source-specific policy.
Run focused domain, profile, use-case, root, and LLM tests, including the
affected race-enabled request and profile cases, followed by the repository
test and vet gates.
## Stage 2: Centralize Output-Contract Legality
**Finding:** S17-F02, including the request-boundary symptom S11-F02.
Add one pure `internal/domain` validator for `OutputContract`. It must enforce
the format, validation-mode, non-negative repair-attempt, and JSON-Schema path
rules fixed above. It must not load schemas or apply source/request defaults.
Make prompt-definition normalization call the shared validator after its file-
specific normalization. Keep prompt-required fields and contextual
`ErrInvalidPromptDefinition` ownership in `internal/promptdef`. Make request
resolution default an empty replacement format to `text`, then call the same
validator and translate failure to `ErrInvalidRequest` before artifact,
rendering, validation, admission, or generation work. Keep schema loading and
compilation in `internal/validate`.
Add a domain table covering every supported and unsupported enum, empty values,
negative and non-negative repair counts, and schema-path relationships. Retain
small prompt-source and use-case integration tables that prove correct error
categories and parity between `Prepare` and `PrepareExecution`; do not repeat
the entire domain table at those layers.
Update the architecture and internal overview language added in Stage 1 to
include source-neutral output-contract invariants. Run focused domain,
prompt-definition, use-case, and root tests, then repository test and vet
gates.
## Stage 3: Make JSON-Compatible Value Handling Coherent And Bounded
**Findings:** S05-F02, S05-F03, S05-F04.
Refactor `internal/jsonvalue` around the numeric and traversal decisions fixed
by this plan. Remove the safe-integer restriction and apply one numeric rule to
all supported representations. Preserve concrete named and unnamed scalar,
map, slice, and array types where the existing contract promises preservation;
keep nil versus empty container distinctions and `Copy` versus `CopyMap` empty-
key behavior.
Extend the traversal state to track JSON container depth and produced-node
work. Enforce the 100-level and 100,000-node limits before allocation or
descent would cross them. Continue using active-path identity for cycle
detection; do not use alias memoization that would make distinct JSON paths
share mutable output. Errors must identify the structural path and whether the
depth or work budget was exceeded.
Expand the focused package tables by behavior branch: signed and unsigned
integer widths, ordinary and named finite floats, `json.Number`, pointers and
interfaces, named maps/slices/arrays, nil and empty values, mixed nested trees,
arrays, mutation isolation, active cycles, alternating just-below/at/over
depth, and shared acyclic subgraphs just below and over the work budget. Tests
must derive their edges from package constants or relationships instead of
copying unexplained literals.
Retain only representative public/backend/profile/prepared integration cases
that prove error translation and ownership. Update public GoDoc only if it
currently states the narrower safe-integer behavior; otherwise the existing
finite JSON-compatible-number contract remains canonical. Update the relevant
public value GoDoc and format/internal documentation to state that excessively
deep or large JSON-shaped values are rejected for safety; keep the exact
numeric limits owned by the internal constants rather than duplicating them
throughout consumer documentation. Run focused package and caller tests,
focused race tests, repository tests, and vet.
## Stage 4: Consolidate Stable Public JSON And Remove Dormant Internal JSON
**Findings:** S02-F01, S02-F05, S05-F05.
Refactor `json.go` so each public value has one ordinary field mapping. Use
private aliases or embedded wire representations for ordinary fields and keep
only timestamp, millisecond-duration, and intentional omission exceptions
explicit. Preserve every existing JSON name and omission rule.
Before converting `duration_ms`, reject values outside the millisecond range
that can be multiplied by `time.Millisecond` without overflow. Derive both
edges from `time.Duration` bounds. Return a contextual decode error and do not
partially update the receiver on failure.
Add fully populated `PreparedRun` and `RunResult` contract cases. Verify all
ordinary fields, intentional omissions, zero and nonzero timing, complete
round trips, the largest safe positive and negative millisecond values, and
their first unsafe neighbors.
Remove unused JSON tags and serialization tests from
`internal/domain.PreparedRun` after confirming production never marshals that
type. Keep credential absence protected at preparation/clone producers and
move any useful cache-control JSON assertion to the public `PreparedRun`
contract. Do not retain a parallel internal wire format.
Run focused domain and root JSON tests, repository tests, and vet.
## Stage 5: Harden Public Ownership And Diagnostic Contracts
**Findings:** S01-F01, S02-F03, S02-F04, S13-F01.
Extend the existing run-request formatting test with distinct input URI,
input-body, variable, and API-key sentinels. Require their absence from
`String`, `GoString`, `%v`, `%+v`, and `%#v` while retaining positive structural
summary assertions.
Add one focused public-LLM-adapter ownership test. Have the injected client
mutate and retain prompt messages, cache-control pointers, nested target extra
parameters, and structured-output schema values; prove the domain/prepared
source remains unchanged and later details or execution cannot race with those
mutations.
Add one direct all-field mapping test for `OpenAICompatibleProfile`. Populate
every field distinctly and compare the complete returned `Profile`. Keep only
the existing higher-level cases that prove normal validation and nested-value
ownership.
Make copied `PreparedExecution` values format opaquely by using value-receiver
formatting behavior shared by non-nil pointers and values. A nil pointer may
use Go's normal `<nil>` formatting, but formatting must never panic or expose
internal types, field names, addresses, credentials, or content. Cover original
pointers, copied values, zero values, and nil pointers under string, Go-string,
and ordinary fmt verbs, and prove formatting does not claim or discard a
handle.
Run focused root tests and the affected prepared/adapter race tests, followed
by repository tests and vet.
## Stage 6: Correct Engine Construction Edges And Immutable Defaults
**Findings:** S03-F01, S03-F02, S06-F01.
Change the shared single-file option helper so trimming is used only for the
blank-input check. Perform `Stat`, path decomposition, storage, diagnostics,
and later access with the exact caller path for prompt, profile, and schema
files. Add one compact table covering existing leading- and trailing-whitespace
names through all three options.
Strengthen engine construction tests with three discriminating cases:
- reverse the argument order of in-memory, ordinary, fallback, and built-in
profile categories while retaining fixed category precedence;
- collide `Config.ProfileDir` with an ordinary profile option and prove the
option replaces the configuration source; and
- place a valid same-category replacement after an invalid option and prove
construction still fails at the earlier invalid option.
Convert `internal/defaults.LLMRequestTimeoutDefault` from a variable to a
constant without changing its value or adding a setter. Do not add a test that
mutates or pins a noncontractual default; existing client deadline behavior is
the verification owner.
Run focused engine construction, default-client construction, and race tests,
then repository tests and vet.
## Stage 7: Contain And Preserve Prompt Content Paths
**Finding:** S07-F01.
Refactor prompt content resolution so both repository forms receive an
explicit source-root abstraction. Enforce the path decision fixed by this plan
before any content read. Use exact parsed path text after a separate blank
check. For OS sources, canonicalize the root and resolved target sufficiently
to reject symlink escape; for injected `fs.FS`, use its clean relative path
namespace. A parent component that remains inside the root is valid. Absolute,
escaping, and symlink-escaping targets are invalid.
Apply the same behavioral table to an OS directory, `WithPromptFS`, and a
single-file source: ordinary sibling, nested parent still within root, parent
escape, absolute path, symlink escape where supported, and existing names with
leading or trailing whitespace. Prove rejected targets cause no outside read
and public operations preserve `ErrPromptLoad`.
Update the framework format reference and internal source document to make the
single-file root and absolute-path rule explicit. Run focused prompt-definition
and public source tests, including race tests, then repository tests and vet.
## Stage 8: Correct Prompt Selection, Strictness, Coverage, And Lookup Cost
**Findings:** S07-F02, S07-F03, S07-F04, S07-F06.
Correct both existing prompt repository paths before consolidating them in the
next stage:
- Recover selector metadata from YAML `id` and `version`; never use a filename
stem as an identity.
- Apply normalized ID and requested-version selection before semantic
normalization or `content_file` reads.
- Associate strict YAML, semantic, and content errors only with a reliably
matching selected definition. An unidentifiable malformed file is unrelated
to point lookup; a reliably selected malformed file remains authoritative.
- Require exactly one YAML document. Comments and trailing whitespace are
allowed; a second empty or populated document and malformed trailing YAML
are `ErrInvalidYAML`.
- Continue scanning the YAML metadata required for duplicate detection, but
open content only for selected candidates. A selected content file is opened
once; unrelated and different-version bodies are never opened.
Add paired OS and `fs.FS` regressions for same-stem/different-ID malformed
files, same-ID/different-version invalid files, selected malformed definitions,
additional YAML documents, duplicates, and counting filesystem behavior.
Add a compact normalization table for the previously uncovered missing
version, blank input name, blank message role, invalid output format, negative
repair attempts, and explicit blank default profile. Output-contract rows
should exercise the shared Stage 2 owner rather than recreate its full table.
Run focused prompt-definition, use-case inspection, and root source tests,
focused race tests, repository tests, and vet.
## Stage 9: Unify Prompt Repository Semantics
**Finding:** S07-F05.
After Stage 8 establishes correct behavior in both paths, replace their
duplicated discovery-to-selection algorithms with one source-neutral prompt
selection and normalization flow. Introduce only the small internal source
adapter needed for YAML discovery, bytes, exact content opening, display paths,
and root containment. Keep genuine OS and `fs.FS` mechanics at the adapter
edge.
Move exact selection, version filtering, strict one-document decoding,
selected-error classification, normalization, duplicate handling, and
not-found behavior into the shared flow. Preserve point-in-time source access;
do not cache catalogs or definitions across operations.
Turn the Stage 8 behavior matrix into a shared suite over both adapters and
retain source-specific tests only for distinct path and I/O failures. Delete
superseded duplicate helpers and tests only after the shared suite protects
their meaningful behavior. Use a counting filesystem and a before/after
benchmark over small and large prompt catalogs to confirm unrelated content is
not read and the refactor adds no second scan; do not enforce wall-clock
thresholds.
Update the internal source document to describe the unified semantic owner.
Run focused package, integration, race, repository test, and vet gates.
## Stage 10: Correct Profile Source Validation And Identity
**Findings:** S08-F02, S08-F03, S08-F04, S08-F05. The non-finite scalar
symptom S08-F01 is already resolved by Stage 1.
Make file profile normalization pass `extra_params` through
`internal/jsonvalue.CopyMap` before publishing a domain profile. Preserve
`ErrInvalidProfile` and source path context for empty keys, non-finite values,
nested invalid data, or traversal-budget failures. Do not move reserved
OpenAI-compatible field policy into the profile package.
Use YAML metadata ID as the only selector; never infer authority from a
filename. Normalize the decoded ID once according to this plan. Require exactly
one YAML document in both metadata and strict selected decoding, so trailing
raw credentials, unknown fields, empty documents, and malformed YAML cannot be
ignored. Reliably selected malformed definitions must stop overlay fallback;
unrelated malformed files must not.
Add shared OS and `fs.FS` tables for invalid/valid extra parameters,
same-stem/different-ID malformed files beside a valid profile, fallback
behavior, additional documents, leading/trailing/blank IDs, normalized
duplicates, and exact inspection/preparation of the normalized ID. Retain only
representative public error-translation cases.
Update the framework format and internal source documents if needed to state
ID normalization and one-document behavior. Run focused profile, use-case,
root, and race tests, followed by repository tests and vet.
## Stage 11: Eliminate Duplicate Profile Decoding
**Finding:** S08-F06.
Refactor point lookup so each file receives one metadata pass and only
canonical ID matches receive strict full decoding and normalization. Reuse
bytes already read for metadata; do not decode every unrelated full profile or
turn the repository into a cache. Preserve deterministic duplicate detection,
strict selected errors, overlay fallthrough only on not-found, and fresh
point-in-time reads on every operation.
Add counting/parser-observation tests where stable behavior can be observed,
plus benchmarks for small and large catalogs reporting time and allocations.
Exercise valid selection, unrelated malformed files, selected malformed files,
duplicates, overlay fallthrough, and repeated lookup. Do not add brittle exact
allocation thresholds to ordinary tests.
Run focused profile and root integration tests, benchmarks for diagnostic
comparison, race tests, repository tests, and vet.
## Stage 12: Correct Artifact Semantics, Cancellation, And Hash Tests
**Findings:** S09-F01, S09-F02, S09-F05.
Treat an explicitly typed empty inline reference as a valid zero-byte artifact,
including `InlineWithURI`. Keep absence at the input map/reference boundary and
compute the same metadata and opaque equality value used for other bodies.
For ordinary file references, inspect the target before opening and again
after opening; reject anything that is not a regular file under the decision
above. Replace unbounded `io.ReadAll` with a normal synchronous chunked read
that checks `ctx.Err()` before open, before and after each read, and before
publishing the artifact. Do not return partial artifacts, add a hidden size
limit, or launch an abandoned reader goroutine.
Add source-parity cases for empty and nonempty inline, inline-with-URI, and file
content. Add a platform-appropriate FIFO regression proving the known FIFO is
rejected without requiring an external writer, and cancellation cases for a
pre-canceled file and a progressing regular-file read. Run them repeatedly and
under the race detector.
Replace exact SHA-256 literals with relational assertions: nonempty and stable
for repeat reads, equal for equal inline/file bodies, unequal for changed
bodies, and propagated opaquely through preparation. Do not document or test a
specific algorithm.
Update public GoDoc and the internal source document to describe regular-file
support and cancellation checkpoints. Run focused package, use-case, root,
race, repository test, and vet gates.
## Stage 13: Make Rendering Cancellation-Aware And Reuse Artifact Text
**Findings:** S09-F03, S09-F04.
Check context before session work, before and after each template parse and
execution, before and after every message, and before returning the completed
prompt. Make the `input` helper return an error when cancellation is observed.
Do not run template execution in a detached goroutine.
Within one `Render` call, lazily convert each named artifact body to text once
and memoize that string for the session and all messages. Build the cached
string in 64 KiB chunks with one pre-grown `strings.Builder`, checking the
context between chunks. Preserve bytes exactly, including invalid UTF-8; do not
cache across render calls or mutate artifacts. Unknown and nil inputs retain
their current errors, and a canceled conversion must not publish or cache a
partial string.
Add deterministic tests for pre-cancellation, cancellation during the chunked
input conversion, and cancellation observed after final-message execution.
Require the context identity and no partial prompt while preserving active-
context template errors. Add benchmarks for one and repeated references across
session and messages; report allocations without hard-coded timing limits.
Run focused renderer/use-case/root tests, benchmarks, repeated race tests,
repository tests, and vet.
## Stage 14: Preserve Exact JSON Validation Semantics
**Findings:** S10-F01, S10-F05.
Create one helper for decoding exactly one JSON value with
`json.Decoder.UseNumber` and required EOF after trailing whitespace. Use it for
schema documents and JSON Schema instance values so large integers, precise
decimals, and exponents retain exact `json.Number` semantics through
compilation, prepared metadata, copying, and validation.
For plain `ValidationJSON`, use a non-materializing complete-document syntax
check such as `json.Valid`; do not build a generic tree. Preserve the current
result distinction: malformed generated JSON is a completed failed validation,
not an operational error, and original output bytes remain unchanged.
Add focused OS and `fs.FS` cases around `2^53`, `1e400`, precise decimals,
ordinary numbers, malformed syntax, and trailing values. Exercise schema
`const`, minimum/maximum, and `multipleOf`, and verify the public structured
schema retains exact numeric values. Add benchmarks for scalar, object, and
large-array JSON validation with allocation reporting but no wall-clock
contract.
Update format/internal validation documentation only where it currently
implies float64-limited semantics. Run focused validator/use-case/root tests,
benchmarks, race tests, repository tests, and vet.
## Stage 15: Escape Schema Resources And Compile Once Per Operation
**Findings:** S10-F02, S10-F03.
Represent schema compiler resources with `url.URL` rather than string
concatenation. Use canonical escaped file URLs for OS paths and a private
scheme URL whose path segments are escaped for `fs.FS`. Preserve separators,
decode resource paths exactly once at the loader boundary, and continue
rejecting remote and escaping references. Legal filenames containing percent,
space, `#`, `?`, or Unicode must compile, including contained relative
references.
Unify JSON Schema preparation around `validate.PreparedValidation`:
- the shared preparation pipeline must create one operation-local compiled
plan and derive provider-facing root schema metadata from that plan;
- `Prepare` may discard the plan after returning metadata;
- `Run` must retain and use the plan for its one operation so the schema graph
is not loaded or compiled again during validation; and
- `PrepareExecution` must retain the same plan in its frozen payload.
Remove the document-only `SchemaDocumentLoader` capability if it has no
remaining production caller. Do not add an engine-wide or cross-operation
schema cache. Keep a clear private preparation carrier in `internal/usecase`
if needed so public `domain.PreparedRun` remains free of validator interfaces.
Replace the existing legal-filename expected failure with valid behavior and
retain a genuine compiler-registration failure only if reachable through a
valid source. Add public parity tests for invalid keywords, malformed and
missing direct/second-level references, unsupported dialects, escapes, remote
references, and valid multi-document graphs. A counting source must show each
document read once per operation and fresh reads across separate operations.
Update internal source, validator, and runner documentation for the unified
plan lifetime. Run focused validator/use-case/root tests, race tests,
repository tests, and vet.
## Stage 16: Make Validation Cancellation Authoritative
**Finding:** S10-F04.
Apply the cancellation decision fixed above. Thread context through schema
resource loaders and all Promptkit-controlled read/decode helpers. Read opened
schema files in context-checked chunks. Check the context immediately before
and after JSON decoding, schema compilation, and schema execution; if
cancellation occurred during a synchronous dependency call, return the context
error instead of a schema or successful validation result. Do not publish a
partial plan or validation result.
Do not place arbitrary `fs.FS` calls or JSON Schema work in goroutines merely
to race them against `ctx.Done()`. Tests must therefore distinguish:
- prompt cancellation before work;
- cancellation between controlled read chunks;
- cancellation that becomes authoritative immediately after a synchronous
compile or validation call returns; and
- the documented limitation that Promptkit cannot preempt a dependency method
that never returns.
Use deterministic controlled readers/contexts rather than sleeps. Assert no
goroutine growth or leaked work and preserve operational validation and public
context identities. Update validator GoDoc and internal source/runner documents
to state the synchronous cancellation boundary accurately.
Run focused cancellation tests normally, repeatedly, and under the race
detector, followed by repository tests and vet.
## Stage 17: Repair And Protect The Retained Internal Repair Path
**Findings:** S12-F01, S12-F02, S12-F03.
Retain the internal repair architecture. Extend `RepairRequest` with
`ExecutionTargetPresence` and carry the resolved presence bits unchanged into
the default repairer's `GenerateRequest`. Factor one use-case-local constructor
for common initial/repair generation fields—effective target, presence,
credential, backend identity, session, and structured output—while keeping the
initial and repair prompts intentionally separate.
Accumulate every completed generation response's five token-usage fields into
run-level usage. The final content/raw output/artifact continues to come from
the last candidate, while usage includes initial generation and every completed
repair exactly once. A repair call that returns an error still returns no
partial public result under current error semantics.
Replace the one-attempt-only repair coverage with a compact state-machine
table for:
- initial success with no repair;
- ineligible basic validation despite a positive budget;
- explicit zero and inherited-zero presence across initial and repair calls;
- success before a larger budget is exhausted;
- exact exhaustion of a larger budget; and
- advancement of attempt number, maximum, prior output, diagnostics, final
status, cumulative usage, and collaborator call count.
Retain the distinct capacity integration proving initial and repair generation
use the same backend pool and one whole-run admission lease, but simplify it if
the new focused table makes repair-state assertions redundant.
Update the internal runner and capacity documents for presence fidelity and
cumulative usage. Do not alter `NewRunner` to install a repairer, public GoDoc
that says the engine is single-pass, or the future public repair roadmap.
Run focused use-case, prepared, capacity, and race tests, followed by repository
tests and vet.
## Stage 18: Preserve Transport Error Identities And Test Deterministically
**Findings:** S14-F01, S04-F01, S16-F02. Timeout overflow S14-F02 is already
resolved through Stage 1's shared bound.
Preserve the underlying `http.Client.Do` error in the chain while retaining
`internal/llm.ErrRequestFailed` and the public generation category. Do not add
headers, request content, endpoints, or provider bodies to error text.
Cancellation and deadline identities must survive caller cancellation, caller
deadline, generation deadline, and whole-request client timeout.
Replace the port-9999 test with a controlled round tripper or `httptest`
endpoint that records the selected URL and returns a deliberate result. It must
make no host-dependent connection and must separately prove empty configured
base acceptance and request-endpoint precedence.
Extend the existing external ordinary-run cancellation test to require both
`ErrLLMGenerate` and `context.Canceled`; retain lower-layer tests only for their
distinct error owners.
Update the OpenAI-compatible integration and internal LLM documents for error
identity behavior. Run focused LLM and root tests with repetition and the race
detector, followed by repository tests and vet.
## Stage 19: Validate And Compose Effective Provider Endpoints
**Finding:** S14-F03.
Add one source-neutral OpenAI-compatible base-endpoint validator in
`internal/domain`, alongside the effective execution target invariant. It must
trim surrounding configuration whitespace, require absolute HTTP or HTTPS with
a host, and reject user information, query, and fragment. Use it from backend
registration, in-memory and file profiles, resolved request overrides, and the
built-in client defense while preserving each boundary's existing config,
profile-load, invalid-request, or LLM error category.
An empty configured base URL remains valid for a built-in client because a
resolved request endpoint may supply it later. Validate only a nonempty
configured base at construction, and always validate the final selected
endpoint before transport. Backends and endpoint-only profiles retain their
existing nonempty endpoint requirements.
Compose the completion URL through parsed URL operations (prefer
`url.JoinPath`) so nested paths and trailing slashes reach exactly one
`/chat/completions` suffix. Never append to a raw string.
Add endpoint tables for HTTP and HTTPS, hosts, nested paths, repeated trailing
slashes, queries, fragments, user information, relative paths, missing hosts,
unsupported schemes, and request/profile/config error mapping. Require every
invalid selected endpoint to fail before transport.
Update the architecture/internal overview for domain endpoint invariants and
the OpenAI-compatible integration and internal LLM documents for URL behavior.
Run focused domain/backend/profile/use-case/LLM/root tests, repetition and race
tests where ownership crosses packages, followed by repository tests and vet.
## Stage 20: Bound And Strictly Frame Successful Provider Responses
**Findings:** S14-F04, S14-F05.
Enforce the 16 MiB successful-response decision without first copying the
entire body. Reject an over-limit `Content-Length` immediately, but also wrap
the body in a counting/limited reader that reads at most one byte beyond the
limit so chunked or dishonest responses cannot bypass it. Exactly-limit bodies
remain valid. Always close the body; do not drain an unbounded oversized
stream.
Decode exactly one response object. After the first decode, require only
trailing JSON whitespace and EOF. A second value, non-whitespace suffix,
truncated body, malformed JSON, or size overflow returns
`ErrMalformedResponse` with no partial response and no provider content in the
error.
Add streaming tests just below, at, and one byte over the limit with and
without `Content-Length`, plus a continuing oversized stream. Assert bounded
bytes read, timely return, no partial result, and closure. Add trailing
whitespace success and trailing garbage/second-value failures. Retain ordinary
response mapping and redaction cases.
Document the fixed successful-response boundary and strict one-document rule
in the integration and internal LLM documents. Explicitly leave bounded
non-success error-envelope parsing to the structured-generation-error roadmap.
Run focused transport tests normally, repeatedly, and under race, followed by
repository tests and vet.
## Stage 21: Consolidate Transport Test Scaffolding
**Finding:** S14-F06.
After transport behavior is stable, introduce one small recording-provider
fixture for common request capture and successful/error response setup.
Organize focused tables around request mapping, authentication, endpoint
composition, timeout/error identity, and response framing. Keep specialized
round trippers/readers for cancellation, deadlines, byte counts, continuing
streams, and body closure.
Retain every existing durable assertion for method, path, headers,
authentication, omission and explicit presence, reserved fields, cache
control, structured output, response mapping, usage, error redaction, and
timeout precedence. Retain all Stage 18 through 20 regressions. Delete repeated
servers, generic-map decoding, and response literals only where the fixture
makes the owning behavior clearer; do not replace wire assertions with a broad
snapshot.
Run the LLM suite normally, with shuffle/repetition, and under the race
detector. Deliberately inspect the resulting test inventory against the audit's
transport matrix before running repository tests and vet.
## Stage 22: Restore One Canonical Maintainer Validation Workflow
**Finding:** S16-F01.
Make `docs/development.md` the canonical owner of the complete local maintainer
workflow, as assigned by the documentation policy. Its validation section must
include, from the repository root:
```sh
go test ./...
go test -race ./...
go vet ./...
go build ./...
go run ./examples/go-library/prepare
go run ./examples/go-library/run
```
It must also own the Go formatting, local Markdown link, `git diff --check`,
workspace/vendor/replacement, generated-output, credential, and working-tree
hygiene checks used before accepting changes.
Change the testing policy to state the semantic requirements and link to that
canonical workflow instead of maintaining a partial competing command list.
Change the release procedure to invoke the development-guide validation as a
release prerequisite rather than presenting a separately maintained copy;
retain release-specific metadata, candidate, tag, and publication commands in
the release document.
Run both examples offline and confirm that a missing or invalid Run example
fixture makes its command fail. Validate all changed Markdown links and ensure
current-state documentation describes only the implemented workflow.
## Stage 23: Complete Traceability And Final Validation
This final stage introduces no new behavior. Review the final tree against the
finding-to-stage table below and the evidence in `audit.md`. Confirm every
canonical group is implemented and every source-specific symptom retains its
required regression and error boundary. Do not mark a finding resolved merely
because a nearby refactor landed.
Run the complete development-guide workflow, including both examples, all
formatting and link checks, and repository hygiene. Also run shuffled ordinary
tests and repeated race-enabled tests for the changed concurrency,
cancellation, prepared, validation, repair, and transport packages. Run the
accepted performance benchmarks for prompt lookup, profile lookup, rendering,
and JSON validation and record only qualitative before/after conclusions; do
not establish release timing promises.
Inspect canonical GoDoc, formats, integration, architecture, and internal
documents against the final implementation. Confirm the public engine still
performs no output repair and the future repair entry remains future work.
Confirm the structured-generation-error feature was not implemented as part of
transport remediation.
Leave `audit-sequence.md`, `audit.md`, and this plan in place for maintainer
review. Retire them only in a separately authorized roadmap-cleanup pass after
the remediation has been reviewed and accepted.
## Finding-To-Stage Traceability
| Stage | Canonical findings | Historical or source-specific records handled with the canonical owner |
| ---: | --- | --- |
| 1 | S05-F01, S17-F01, S14-F02 | S02-F02, S08-F01, S11-F01 |
| 2 | S17-F02 | S11-F02 |
| 3 | S05-F02, S05-F03, S05-F04 | None |
| 4 | S02-F01, S02-F05, S05-F05 | None |
| 5 | S01-F01, S02-F03, S02-F04, S13-F01 | None |
| 6 | S03-F01, S03-F02, S06-F01 | None |
| 7 | S07-F01 | None |
| 8 | S07-F02, S07-F03, S07-F04, S07-F06 | None |
| 9 | S07-F05 | None |
| 10 | S08-F02, S08-F03, S08-F04, S08-F05 | S08-F01 was handled in Stage 1 |
| 11 | S08-F06 | None |
| 12 | S09-F01, S09-F02, S09-F05 | None |
| 13 | S09-F03, S09-F04 | None |
| 14 | S10-F01, S10-F05 | None |
| 15 | S10-F02, S10-F03 | None |
| 16 | S10-F04 | None |
| 17 | S12-F01, S12-F02, S12-F03 | None |
| 18 | S14-F01, S04-F01, S16-F02 | S14-F02 was handled in Stage 1 |
| 19 | S14-F03 | None |
| 20 | S14-F04, S14-F05 | None |
| 21 | S14-F06 | None |
| 22 | S16-F01 | None |
The table maps all 49 canonical remediation groups exactly once. S02-F02 is
the one superseded historical finding retained as evidence under S17-F01;
S08-F01, S11-F01, and S11-F02 retain their source-specific regression
responsibilities without being double-counted as canonical groups.
## Open Questions
None. The numeric contract, resource bounds, path and identity rules,
validation-cancellation limitation, repair retention, transport response
limit, and documentation ownership required to implement these stages are
fixed above.

View File

@@ -0,0 +1,71 @@
# Structured Generation Errors
## Purpose
Promptkit should give downstream applications actionable, machine-readable
details when the built-in OpenAI-compatible client receives a non-success HTTP
response. Today the client reports only the status code and discards the
provider response body. This makes ordinary configuration failures—such as an
unsupported strict JSON Schema keyword—unnecessarily difficult to diagnose.
## Target End State
Failures from the built-in transport are available through a public typed error
that works with `errors.As` while continuing to match `ErrLLMGenerate` through
`errors.Is`. The error should expose:
- the HTTP status code;
- a normalized provider error code or type when supplied; and
- a bounded provider message extracted from a recognized OpenAI-compatible
JSON error envelope.
The ordinary `Error()` string should remain safe and concise: it should include
the status and provider code or type, but not automatically include the
provider message. Consumers that deliberately want the provider's diagnostic
text can retrieve it from the typed error and apply their own disclosure and
logging policy.
This contract should be available for both ordinary and prepared execution.
Errors returned by injected model clients must continue to preserve their own
identity and should not be converted into fabricated HTTP details.
## Safety And Compatibility Boundaries
- Never expose the raw response body, response headers, endpoint, credentials,
request messages, schema document, or generated content through this API.
- Read only a small fixed maximum response body, reject malformed or
unrecognized envelopes, normalize invalid UTF-8 and control characters, and
cap every retained diagnostic field independently.
- Treat the extracted provider message as untrusted and potentially sensitive:
its GoDoc must tell consumers not to log or display it without applying their
own policy.
- Preserve the existing generic behavior when a response is empty, non-JSON,
oversized, or does not match a recognized error envelope.
- Do not assign retryability from an HTTP status. Promptkit supplies facts;
downstream applications retain retry and presentation policy.
## Recommended API Direction
Prefer one immutable public `GenerationError` value, constructed internally and
carrying accessors for HTTP status, provider code or type, and provider message.
This keeps the exact representation evolvable while giving consumers an
idiomatic `errors.As` contract. Public Go declarations and GoDoc should own the
final exact names and semantics.
The internal OpenAI-compatible client should parse only the conventional
top-level `error` envelope and pass normalized details through the use-case and
public error-mapping layers. The integration documentation should continue to
own wire behavior; the public declarations should own the consumer contract.
## Acceptance Criteria
- A downstream consumer can distinguish a provider HTTP 400 from other
generation failures and obtain a bounded provider explanation when present.
- The typed error still satisfies `errors.Is(err, ErrLLMGenerate)`.
- Existing cancellation, capacity, validation, and injected-client error
identities remain unchanged.
- Tests cover recognized string and numeric provider codes, absent and malformed
envelopes, oversized bodies and fields, control characters, and error-chain
behavior without making live provider requests.
- Current-state GoDoc and the OpenAI-compatible integration and internal-client
documents are updated only when the implementation lands.

457
engine.go
View File

@@ -12,7 +12,10 @@ import (
"time"
artifactadapter "gitea.maximumdirect.net/eric/promptkit/internal/artifact"
"gitea.maximumdirect.net/eric/promptkit/internal/backend"
"gitea.maximumdirect.net/eric/promptkit/internal/capacity"
"gitea.maximumdirect.net/eric/promptkit/internal/defaults"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
"gitea.maximumdirect.net/eric/promptkit/internal/llm"
"gitea.maximumdirect.net/eric/promptkit/internal/profile"
"gitea.maximumdirect.net/eric/promptkit/internal/profile/builtin"
@@ -22,42 +25,107 @@ import (
"gitea.maximumdirect.net/eric/promptkit/internal/validate"
)
// ErrInvalidConfig indicates invalid public engine configuration.
// ErrInvalidConfig identifies invalid engine construction, including missing
// required configuration, invalid options or backend registrations, and a nil
// Engine receiver.
var ErrInvalidConfig = errors.New("invalid engine configuration")
var (
ErrInvalidRequest = errors.New("invalid run request")
ErrPromptNotFound = errors.New("prompt not found")
ErrProfileNotFound = errors.New("profile not found")
ErrProfileRequired = errors.New("profile selection is required")
ErrPromptLoad = errors.New("failed to load prompt definition")
ErrProfileLoad = errors.New("failed to load execution profile")
// ErrInvalidRequest identifies a request whose required values, overrides,
// credentials, or effective settings are invalid.
ErrInvalidRequest = errors.New("invalid run request")
// ErrPromptNotFound identifies a requested prompt ID or version that is not
// present in the selected prompt source. It does not also match
// ErrPromptLoad.
ErrPromptNotFound = errors.New("prompt not found")
// ErrProfileNotFound identifies a selected profile ID that is absent from
// every configured profile source. It does not also match ErrProfileLoad.
ErrProfileNotFound = errors.New("profile not found")
// ErrProfileRequired identifies a request for which neither RunRequest.ProfileID
// nor the selected prompt's default profile is present. Such an error also
// matches ErrInvalidRequest.
ErrProfileRequired = errors.New("profile selection is required")
// ErrPromptLoad identifies a failure to read, decode, validate, select, or
// hash a prompt definition, except for the not-found case represented by
// ErrPromptNotFound.
ErrPromptLoad = errors.New("failed to load prompt definition")
// ErrProfileLoad identifies a failure to read, decode, validate, or select
// an execution profile or resolve its backend, except for the profile
// not-found case represented by ErrProfileNotFound.
ErrProfileLoad = errors.New("failed to load execution profile")
// ErrAPIKeyEnvMissing identifies an APIKeyEnv whose environment variable is
// unset or empty when no direct RunRequest.APIKey takes precedence. Such an
// error also matches ErrInvalidRequest.
ErrAPIKeyEnvMissing = errors.New("api_key_env points to an unset environment variable")
ErrArtifactLoad = errors.New("failed to load artifact")
ErrPromptRender = errors.New("failed to render prompt")
ErrLLMGenerate = errors.New("failed to generate output")
ErrValidation = errors.New("failed to validate output")
// ErrArtifactLoad identifies a failure to resolve an input artifact. Errors
// returned by an injected ArtifactReader remain available through errors.Is.
ErrArtifactLoad = errors.New("failed to load artifact")
// ErrPromptRender identifies a failure to render prompt messages or the
// session ID from the resolved inputs and variables.
ErrPromptRender = errors.New("failed to render prompt")
// ErrCapacityExceeded identifies a Run or RunPrepared rejected because the
// selected backend already admitted ConcurrencyLimit + QueueCapacity calls.
// A [CapacityError] reports the selected backend ID. It is not an invalid
// request, an LLM or provider rate-limit response, or ErrLLMGenerate.
ErrCapacityExceeded = errors.New("backend capacity exceeded")
// ErrLLMGenerate identifies a model-client failure or a nil successful
// response. Errors returned by an injected LLMClient remain available
// through errors.Is.
ErrLLMGenerate = errors.New("failed to generate output")
// ErrValidation identifies an operational failure to load or compile a
// schema or validate output. A completed validation whose Status is
// ValidationFailed is returned in RunResult without this error.
ErrValidation = errors.New("failed to validate output")
)
// Engine prepares and runs Promptkit prompt requests.
// Engine inspects prompts and profiles and prepares and runs Promptkit prompt
// requests.
//
// An Engine is safe for concurrent calls to [Engine.InspectPrompt],
// [Engine.InspectProfile], [Engine.Prepare], [Engine.PrepareExecution],
// [Engine.Run], and [Engine.RunPrepared]. Each Engine owns independent
// backend-capacity pools that coordinate Run and RunPrepared admission and
// model generation. Injected collaborators may still be invoked concurrently
// across different backend pools or for unlimited backends.
type Engine struct {
runner *usecase.Runner
}
// Config configures a public Promptkit engine.
// Config selects the directory-backed sources and built-in model-client
// transport used by [NewEngine]. Config has no stable JSON representation.
type Config struct {
PromptDir string
// PromptDir is the directory searched recursively for prompt definitions.
// It is required unless a WithPromptFS or WithPromptFile option supplies the
// prompt source.
PromptDir string
// ProfileDir is an optional ordinary configured source whose profiles take
// precedence over application fallback and embedded built-in profiles. An
// empty value selects the lower-precedence sources unless a profile-source
// option supplies the ordinary source.
ProfileDir string
SchemaDir string
// SchemaDir is the root for JSON Schema files. An empty value uses the
// current directory. WithSchemaFS or WithSchemaFile replaces this source.
SchemaDir string
// Timeout is the transport-wide safety cap for the built-in LLM client
// when HTTPClient is absent or has a non-positive timeout.
// when HTTPClient is absent or has a non-positive timeout. A zero or negative
// value selects the 10-minute default.
Timeout time.Duration
// HTTPClient is cloned for the built-in LLM client. Its positive Timeout
// takes precedence over Config.Timeout as the transport-wide safety cap.
// takes precedence over Timeout. A zero or negative client Timeout inherits
// Timeout or the 10-minute default. The supplied client is not mutated. This
// field is ignored when WithLLMClient is used.
HTTPClient *http.Client
}
// Option customizes engine construction.
//
// NewEngine applies options in argument order and ignores nil options. Within
// each prompt-source, ordinary-profile-source, fallback-profile-source,
// in-memory-profile, schema-source, model-client, and artifact-reader
// category, the last non-nil valid option replaces earlier options in that
// category. WithBackend is the additive exception: unique registrations
// accumulate, and a repeated backend ID is an error rather than a replacement.
// An invalid option fails construction even if a later option would replace it.
type Option interface {
apply(*engineOptions) error
}
@@ -69,20 +137,30 @@ func (f optionFunc) apply(options *engineOptions) error {
}
type engineOptions struct {
llmClient llm.Client
artifactReader artifactadapter.Reader
promptDefs promptdef.Repository
profiles profile.Repository
memoryProfiles profile.Repository
validator validate.Validator
promptSource bool
profileSource bool
memorySource bool
validatorSource bool
artifactSource bool
llmClient llm.Client
artifactReader artifactadapter.Reader
promptDefs promptdef.Repository
profiles profile.Repository
fallbackProfiles profile.Repository
memoryProfiles profile.Repository
backends []domain.Backend
validator validate.Validator
promptSource bool
profileSource bool
fallbackProfileSource bool
memorySource bool
validatorSource bool
artifactSource bool
}
// WithLLMClient injects a custom LLM client for execution.
// WithLLMClient replaces the built-in model client used by [Engine.Run] and
// [Engine.RunPrepared].
//
// A nil client makes NewEngine fail with ErrInvalidConfig. The Engine schedules
// Generate calls according to the selected backend's capacity policy, but the
// client may still be called concurrently across different backend pools or for
// unlimited backends. The client is not used by [Engine.Prepare] or
// [Engine.PrepareExecution].
func WithLLMClient(client LLMClient) Option {
return optionFunc(func(options *engineOptions) error {
if client == nil {
@@ -93,7 +171,11 @@ func WithLLMClient(client LLMClient) Option {
})
}
// WithArtifactReader injects a reader for every input artifact reference.
// WithArtifactReader replaces the default reader for every input artifact
// reference, regardless of its ArtifactRef.Type.
//
// A nil reader makes NewEngine fail with ErrInvalidConfig. The reader may be
// called concurrently.
func WithArtifactReader(reader ArtifactReader) Option {
return optionFunc(func(options *engineOptions) error {
if reader == nil {
@@ -109,6 +191,9 @@ func WithArtifactReader(reader ArtifactReader) Option {
//
// The source uses the same strict prompt YAML rules as configured prompt
// directories, and prompt content_file paths resolve within this source.
// fsys must be non-nil and root must be non-empty; otherwise NewEngine fails
// with ErrInvalidConfig. This option replaces Config.PromptDir and earlier
// prompt-source options.
func WithPromptFS(fsys fs.FS, root string) Option {
return optionFunc(func(options *engineOptions) error {
if fsys == nil {
@@ -125,7 +210,9 @@ func WithPromptFS(fsys fs.FS, root string) Option {
// WithPromptFile loads prompt definitions from the single prompt file at path.
//
// Relative prompt content_file paths resolve from the file's directory.
// Relative prompt content_file paths resolve from the file's directory. path
// must name an existing non-directory file when NewEngine applies the option.
// This option replaces Config.PromptDir and earlier prompt-source options.
func WithPromptFile(path string) Option {
return optionFunc(func(options *engineOptions) error {
fsys, root, err := fileSource(path)
@@ -140,8 +227,12 @@ func WithPromptFile(path string) Option {
// WithProfileFS loads execution profiles from fsys under root.
//
// Profiles from this source overlay built-in profiles. Profile YAML must use
// api_key_env for environment-based credentials; raw API keys are rejected.
// Profiles from this ordinary configured source take precedence over
// application fallback and built-in profiles. Profile YAML must use api_key_env
// for environment-based credentials; raw API keys are rejected. fsys must be
// non-nil and root must be non-empty; otherwise NewEngine fails with
// ErrInvalidConfig. This option replaces Config.ProfileDir and earlier file or
// FS profile-source options, but remains below WithProfiles in precedence.
func WithProfileFS(fsys fs.FS, root string) Option {
return optionFunc(func(options *engineOptions) error {
if fsys == nil {
@@ -158,8 +249,12 @@ func WithProfileFS(fsys fs.FS, root string) Option {
// WithProfileFile loads execution profiles from the single profile file at path.
//
// The profile overlays built-in profiles. Profile YAML must use api_key_env for
// environment-based credentials; raw API keys are rejected.
// The profile takes precedence over application fallback and built-in profiles.
// Profile YAML must use api_key_env for environment-based credentials; raw API
// keys are rejected. path must name an existing non-directory file when
// NewEngine applies the option. This option replaces Config.ProfileDir and
// earlier file or FS profile-source options, but remains below WithProfiles in
// precedence.
func WithProfileFile(path string) Option {
return optionFunc(func(options *engineOptions) error {
fsys, root, err := fileSource(path)
@@ -172,8 +267,46 @@ func WithProfileFile(path string) Option {
})
}
// WithFallbackProfileFS supplies application-owned fallback profile
// definitions from fsys under root.
//
// Profile lookup checks, in order, profiles supplied by WithProfiles; the
// ordinary configured source selected by WithProfileFile, WithProfileFS, or
// Config.ProfileDir; this fallback source; and Promptkit's embedded built-in
// profiles. Each source supplies a complete profile definition; profile fields
// are not merged between sources. Only an absent profile ID proceeds to the
// next source. A matching read, parse, duplicate, validation, or credential
// format failure stops resolution.
//
// Files use the ordinary strict profile YAML and api_key_env credential rules.
// Loading and validation are lazy: NewEngine validates this option's arguments
// but does not read profile files. fsys must be non-nil and root must be
// nonblank; otherwise NewEngine returns an error matching ErrInvalidConfig.
// Repeating this option replaces the earlier valid fallback source.
//
// This option controls profile-definition lookup, not provider or generation
// failover.
func WithFallbackProfileFS(fsys fs.FS, root string) Option {
return optionFunc(func(options *engineOptions) error {
if fsys == nil {
return ErrInvalidConfig
}
if strings.TrimSpace(root) == "" {
return ErrInvalidConfig
}
options.fallbackProfiles = profile.NewFSRepository(fsys, root)
options.fallbackProfileSource = true
return nil
})
}
// WithProfiles configures in-memory profiles that take precedence over
// configured profile files and built-in profiles.
// ordinary configured, application fallback, and built-in profiles.
//
// NewEngine validates and copies every profile. IDs must be unique within one
// call. An invalid profile, duplicate ID, or unsupported ExtraParams value
// makes construction fail with ErrInvalidConfig. Repeating WithProfiles
// replaces the complete earlier in-memory set rather than merging it.
func WithProfiles(profiles ...Profile) Option {
return optionFunc(func(options *engineOptions) error {
repo, err := newMemoryProfileRepository(profiles)
@@ -189,7 +322,9 @@ func WithProfiles(profiles ...Profile) Option {
// WithSchemaFS loads JSON Schema documents from fsys under root.
//
// Prompt schema_path values resolve within this source when schema validation
// or structured output is requested.
// or structured output is requested. fsys must be non-nil and root must be
// non-empty; otherwise NewEngine fails with ErrInvalidConfig. This option
// replaces Config.SchemaDir and earlier schema-source options.
func WithSchemaFS(fsys fs.FS, root string) Option {
return optionFunc(func(options *engineOptions) error {
if fsys == nil {
@@ -206,7 +341,9 @@ func WithSchemaFS(fsys fs.FS, root string) Option {
// WithSchemaFile loads JSON Schema documents from the single schema file at path.
//
// Prompt schema_path values refer to the file's base name.
// Prompt schema_path values refer to the file's base name. path must name an
// existing non-directory file when NewEngine applies the option. This option
// replaces Config.SchemaDir and earlier schema-source options.
func WithSchemaFile(path string) Option {
return optionFunc(func(options *engineOptions) error {
fsys, root, err := fileSource(path)
@@ -220,6 +357,17 @@ func WithSchemaFile(path string) Option {
}
// NewEngine constructs an Engine from configuration and options.
//
// Options are applied in order according to [Option]. PromptDir is required
// unless a prompt-source option is present. Construction validates option
// arguments, in-memory profiles, and backend registrations but defers reading
// and validating prompt, file-backed profile, and schema contents until Prepare
// or Run needs them.
//
// NewEngine returns an error matching ErrInvalidConfig for invalid
// configuration, options, or backend-capacity policies. Each constructed
// Engine has independent backend-capacity pools. Construction does not perform
// model requests or require credentials.
func NewEngine(cfg Config, opts ...Option) (*Engine, error) {
var options engineOptions
for _, opt := range opts {
@@ -239,12 +387,16 @@ func NewEngine(cfg Config, opts ...Option) (*Engine, error) {
promptDefs = promptdef.NewFilesystemRepository(cfg.PromptDir)
}
profiles := builtin.NewRepositoryWithDirectory(cfg.ProfileDir)
if options.profileSource {
profiles = builtin.NewRepositoryWithPrimary(options.profiles)
profiles := newProfileRepository(cfg.ProfileDir, options)
backendRegistry, err := backend.NewRegistry(options.backends)
if err != nil {
return nil, fmt.Errorf("%w: failed to construct backend registry: %v", ErrInvalidConfig, err)
}
if options.memorySource {
profiles = profile.NewOverlayRepository(options.memoryProfiles, profiles)
capacityManager, err := capacity.NewManager(backendRegistry.CapacityPolicies())
if err != nil {
return nil, fmt.Errorf("%w: failed to construct backend capacity manager: %v", ErrInvalidConfig, err)
}
validator := options.validator
@@ -267,6 +419,7 @@ func NewEngine(cfg Config, opts ...Option) (*Engine, error) {
return nil, fmt.Errorf("%w: %v", ErrInvalidConfig, err)
}
}
llmClient = capacity.NewClient(capacityManager, llmClient)
artifacts := options.artifactReader
if !options.artifactSource {
@@ -277,14 +430,36 @@ func NewEngine(cfg Config, opts ...Option) (*Engine, error) {
runner: usecase.NewRunner(
promptDefs,
profiles,
backendRegistry,
artifacts,
prompt.NewGoRenderer(),
llmClient,
validator,
capacityManager,
),
}, nil
}
func newProfileRepository(profileDir string, options engineOptions) profile.Repository {
repository := builtin.NewRepository()
if options.fallbackProfileSource {
repository = profile.NewOverlayRepository(options.fallbackProfiles, repository)
}
if options.profileSource {
repository = profile.NewOverlayRepository(options.profiles, repository)
} else if strings.TrimSpace(profileDir) != "" {
repository = profile.NewOverlayRepository(profile.NewFilesystemRepository(profileDir), repository)
}
if options.memorySource {
repository = profile.NewOverlayRepository(options.memoryProfiles, repository)
}
return repository
}
func fileSource(name string) (fs.FS, string, error) {
cleanName := strings.TrimSpace(name)
if cleanName == "" {
@@ -305,7 +480,110 @@ func fileSource(name string) (fs.FS, string, error) {
return os.DirFS(dir), filepath.ToSlash(base), nil
}
// Prepare resolves a prompt request without calling an LLM.
// InspectPrompt resolves one explicit prompt definition without selecting a
// profile or starting execution work.
//
// InspectPrompt requires a nonblank promptID. It passes nonblank promptID and
// promptVersion values unchanged to the engine's ordinary, case-sensitive
// prompt selection. An empty version succeeds only when that source has one
// selected ID; a nonempty version selects one exact ID/version pair. The
// configured prompt source is used without merging, fallback, or enumeration.
//
// A successful result proves that the selected definition and any referenced
// message content files were structurally loaded. Inputs are returned in
// definition order. DefaultProfileID is declared metadata only and is not
// resolved. OutputContract is the normalized declared contract, with a JSON
// Schema path when declared but without loading or compiling that schema.
// PromptHash is the same opaque equality value as PreparedRun.PromptHash for
// the selected definition and observed source state; its spelling, length,
// encoding, algorithm, and security properties are not contracts.
//
// This method does not return prompt bodies, templates, source paths, schemas,
// rendered messages, or execution settings. It does not resolve a profile or
// credential, read artifacts or schemas, render, validate, admit capacity,
// contact a provider, or generate model output. The returned PromptInspection
// and its input slice are caller-owned. Filesystem-backed inspection is a
// point-in-time lookup and does not freeze a definition for later execution.
//
// A nil Engine returns an error matching ErrInvalidConfig. A blank prompt ID
// matches ErrInvalidRequest. An absent exact ID or version matches
// ErrPromptNotFound and not ErrPromptLoad. Malformed, unreadable, duplicate,
// ambiguous, referenced-content, or hashing failures match ErrPromptLoad.
// Cancellation during lookup matches ErrPromptLoad while preserving the
// context error. InspectPrompt returns no partial result on error.
func (e *Engine) InspectPrompt(
ctx context.Context,
promptID string,
promptVersion string,
) (*PromptInspection, error) {
if e == nil || e.runner == nil {
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
}
inspection, err := e.runner.InspectPrompt(ctx, promptID, promptVersion)
if err != nil {
return nil, mapPublicError(err)
}
return fromDomainPromptInspection(inspection), nil
}
// InspectProfile resolves one explicit profile without selecting a prompt or
// starting execution work.
//
// InspectProfile trims surrounding whitespace from profileID and looks up the
// resulting nonblank ID exactly and case-sensitively through the engine's
// in-memory, ordinary configured-source, application fallback, and built-in
// profile precedence. It applies the framework timeout baseline, selected
// backend, and then selected profile to EffectiveModelParams without a request
// override. BackendID is empty for an endpoint-only profile.
//
// APIKeyEnv in the returned target is an environment-variable name, never its
// value. APIKeyRequired instead reports a direct credential requirement and is
// mutually exclusive with a nonblank APIKeyEnv. InspectProfile neither derives
// an ID from a prompt default_profile nor checks credential availability, so an
// absent or blank named environment variable is not an error.
//
// The returned ProfileInspection and all nested mutable values are
// caller-owned. Filesystem-backed inspection is a point-in-time lookup and
// does not freeze the profile for a later execution. This method does not load
// a prompt, render, read artifacts or schemas, admit backend capacity, contact
// a provider, or generate model output.
//
// A nil Engine returns an error matching ErrInvalidConfig. A blank profile ID
// matches ErrInvalidRequest. An absent exact ID matches ErrProfileNotFound and
// not ErrProfileLoad. Malformed or unreadable profile data, an unknown backend,
// or an invalid resolved target matches ErrProfileLoad. Cancellation during
// profile loading matches ErrProfileLoad while preserving the context error.
// InspectProfile returns no partial result on error.
func (e *Engine) InspectProfile(ctx context.Context, profileID string) (*ProfileInspection, error) {
if e == nil || e.runner == nil {
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
}
inspection, err := e.runner.InspectProfile(ctx, profileID)
if err != nil {
return nil, mapPublicError(err)
}
return fromDomainProfileInspection(inspection), nil
}
// Prepare resolves and renders a prompt request without calling an LLM.
//
// Prepare selects the prompt and profile, resolves any selected backend and
// effective execution settings, resolves the output contract, loads and hashes
// inputs, loads structured-output schema metadata when required, and renders
// the session ID and messages. The returned PreparedRun is owned by the caller
// and never contains a resolved API-key value, model output, or validation
// result.
//
// A nil Engine returns an error matching ErrInvalidConfig. Request and
// preparation failures may match ErrInvalidRequest, ErrPromptNotFound,
// ErrPromptLoad, ErrProfileNotFound, ErrProfileLoad, ErrProfileRequired,
// ErrAPIKeyEnvMissing, ErrArtifactLoad, ErrPromptRender, or ErrValidation as
// applicable. Cancellation is passed to the active collaborator and is
// reported in the applicable operation category; no general errors.Is
// relationship to ctx.Err is promised. Prepare returns no partial result on
// error.
func (e *Engine) Prepare(ctx context.Context, req RunRequest) (*PreparedRun, error) {
if e == nil || e.runner == nil {
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
@@ -323,7 +601,57 @@ func (e *Engine) Prepare(ctx context.Context, req RunRequest) (*PreparedRun, err
return fromDomainPreparedRun(prepared), nil
}
// Run executes a prompt request and returns the generated artifact and metadata.
// PrepareExecution completely prepares a prompt request without calling the
// configured LLMClient or reserving backend admission capacity.
//
// The returned opaque handle is bound to this Engine and permits one
// [Engine.RunPrepared] invocation. Preparation freezes the selected sources,
// rendered messages, effective settings, inputs, provider structured-output
// metadata, and validation resources needed by that invocation. The handle
// retains a direct RunRequest.APIKey only in private execution state;
// [PreparedExecution.Details] is credential-redacted.
//
// The context governs preparation only. Cancellation after this method
// returns does not invalidate the handle or propagate to RunPrepared.
// PrepareExecution returns the same error categories as [Engine.Prepare] and
// returns no handle on error. A nil Engine returns an error matching
// ErrInvalidConfig.
func (e *Engine) PrepareExecution(ctx context.Context, req RunRequest) (*PreparedExecution, error) {
if e == nil || e.runner == nil {
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
}
domainReq, err := toDomainRunRequest(req)
if err != nil {
return nil, fmt.Errorf("%w: %v", ErrInvalidRequest, err)
}
prepared, err := e.runner.PrepareExecution(ctx, domainReq)
if err != nil {
return nil, mapPublicError(err)
}
return &PreparedExecution{internal: prepared}, nil
}
// Run prepares a request, invokes the configured LLMClient, and validates the
// generated output.
//
// A content-validation failure is a successful run whose
// RunResult.Validation has Status ValidationFailed. An inability to perform
// validation returns an error matching ErrValidation and no partial result.
// The public Engine does not perform output repair, so validation is
// single-pass even when OutputContract.RepairAttempts is positive.
//
// Run can return every error category documented by [Engine.Prepare], plus
// ErrCapacityExceeded and ErrLLMGenerate. An engine admission rejection is
// discoverable as [CapacityError] and still matches ErrCapacityExceeded. It
// occurs before artifacts, schemas, rendering, or model generation because the
// selected backend's admission capacity is full; it does not match
// ErrInvalidRequest or ErrLLMGenerate. Errors from injected clients remain
// available through errors.Is. Cancellation while waiting for model-generation
// capacity matches both ErrLLMGenerate and the context error. Cancellation
// otherwise follows the active collaborator's documented behavior. A nil
// Engine returns ErrInvalidConfig. Run returns no partial result on error.
func (e *Engine) Run(ctx context.Context, req RunRequest) (*RunResult, error) {
if e == nil || e.runner == nil {
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
@@ -340,3 +668,42 @@ func (e *Engine) Run(ctx context.Context, req RunRequest) (*RunResult, error) {
}
return fromDomainRunResult(result), nil
}
// RunPrepared atomically claims and executes a handle created by
// [Engine.PrepareExecution].
//
// A valid owning-Engine invocation consumes the handle's one attempt before
// credential revalidation, backend admission, generation, or validation.
// Cancellation, capacity rejection, generation failure, operational
// validation failure, and success all leave the handle unusable. A nil,
// zero-value, foreign-Engine, discarded, claimed, or used handle returns an
// error matching ErrInvalidRequest; a nil Engine returns ErrInvalidConfig and
// does not claim the handle.
//
// The supplied context governs this execution attempt independently of the
// preparation context. It covers credential revalidation, admission,
// generation, validation, and any internal repair. Result timing begins after
// the claim and excludes preparation and consumer-held delay.
//
// RunPrepared can return ErrInvalidRequest, ErrAPIKeyEnvMissing,
// ErrCapacityExceeded, ErrLLMGenerate, or ErrValidation as applicable while
// preserving documented collaborator and context identities. An engine
// admission rejection is discoverable as [CapacityError] and still matches
// ErrCapacityExceeded. A completed content-validation rejection is returned
// in RunResult, not as an operational error. An operational error returns no
// partial RunResult.
func (e *Engine) RunPrepared(ctx context.Context, prepared *PreparedExecution) (*RunResult, error) {
if e == nil || e.runner == nil {
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
}
var internal *usecase.PreparedExecution
if prepared != nil {
internal = prepared.internal
}
result, err := e.runner.RunPrepared(ctx, internal)
if err != nil {
return nil, mapPublicError(err)
}
return fromDomainRunResult(result), nil
}

View File

@@ -281,6 +281,9 @@ func TestEngineExecutionSettingPrecedence(t *testing.T) {
intPointer := func(value int) *int {
return &value
}
stringPointer := func(value string) *string {
return &value
}
defaultsProfile := executionProfileFixture{
id: "settings-defaults",
@@ -357,14 +360,14 @@ func TestEngineExecutionSettingPrecedence(t *testing.T) {
wantPresence promptkit.ExecutionTargetPresence
}{
{
name: "framework defaults fill zero-valued profile settings",
name: "unspecified provider controls retain framework timeout",
profile: defaultsProfile,
want: promptkit.ExecutionTarget{
Endpoint: defaultsProfile.endpoint,
Model: defaultsProfile.model,
Temperature: 0,
MaxTokens: 0,
TopP: 1,
TopP: 0,
TimeoutSeconds: 600,
ServiceTier: defaultsProfile.serviceTier,
ReasoningEffort: defaultsProfile.reasoningEffort,
@@ -388,7 +391,7 @@ func TestEngineExecutionSettingPrecedence(t *testing.T) {
TopP: floatPointer(requestTarget.TopP),
TimeoutSeconds: intPointer(requestTarget.TimeoutSeconds),
ServiceTier: requestTarget.ServiceTier,
ReasoningEffort: requestTarget.ReasoningEffort,
ReasoningEffort: stringPointer(requestTarget.ReasoningEffort),
APIKeyEnv: requestTarget.APIKeyEnv,
ExtraParams: requestTarget.ExtraParams,
},
@@ -452,6 +455,43 @@ func TestEngineExecutionSettingPrecedence(t *testing.T) {
}
})
}
t.Run("blank request reasoning clears profile setting", func(t *testing.T) {
profile := executionProfileFixture{
id: "settings-reasoning-clear",
endpoint: "http://profile-reasoning.test/v1",
model: "profile-reasoning-model",
reasoningEffort: "medium",
}
profileDir := t.TempDir()
writeExecutionProfileFixture(t, profileDir, profile)
engine, err := promptkit.NewEngine(promptkit.Config{
PromptDir: frameworkPromptDir,
ProfileDir: profileDir,
SchemaDir: frameworkSchemaDir,
})
if err != nil {
t.Fatalf("construct engine: %v", err)
}
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
PromptID: frameworkMarkdownSummaryPromptID,
ProfileID: profile.id,
Inputs: map[string]promptkit.ArtifactRef{
"transcript": promptkit.Inline("Nia labels the archive."),
"glossary": promptkit.Inline("archive: A catalogued collection."),
},
Execution: &promptkit.ExecutionTargetOverride{
ReasoningEffort: stringPointer(" \t "),
},
})
if err != nil {
t.Fatalf("prepare engine: %v", err)
}
if prepared.EffectiveModelParams.ReasoningEffort != "" {
t.Fatalf("expected blank request reasoning to clear profile value, got %q", prepared.EffectiveModelParams.ReasoningEffort)
}
})
}
func TestRunSucceedsWithInjectedLLMClient(t *testing.T) {
@@ -571,19 +611,26 @@ func TestEngineRunWithDirectorySourcesAndFileInputs(t *testing.T) {
func TestRunPassesPreparedRequestToInjectedLLMClient(t *testing.T) {
const directKey = "direct-injected-key"
const directSession = "assembled-session"
fake := &fakeLLMClient{
response: &promptkit.GenerateResponse{Content: `{"events":[{"title":"Archive labelled"}]}`},
}
engine := newContractEngineWithOptions(t, frameworkSchemaDir, promptkit.WithLLMClient(fake))
_, err := engine.Run(context.Background(), promptkit.RunRequest{
PromptID: frameworkStructuredEventsPromptID,
APIKey: directKey,
runRequest := promptkit.RunRequest{
PromptID: frameworkStructuredEventsPromptID,
SessionID: " " + directSession + " ",
APIKey: directKey,
Inputs: map[string]promptkit.ArtifactRef{
"transcript": promptkit.Inline("Rin opens the gate."),
"glossary": promptkit.Inline("gate: A guarded passage."),
},
})
}
prepared, err := engine.Prepare(context.Background(), runRequest)
if err != nil {
t.Fatalf("expected prepare to succeed, got %v", err)
}
result, err := engine.Run(context.Background(), runRequest)
if err != nil {
t.Fatalf("expected run to succeed, got %v", err)
}
@@ -594,6 +641,16 @@ func TestRunPassesPreparedRequestToInjectedLLMClient(t *testing.T) {
if len(req.Prompt.Messages) != 2 || !strings.Contains(req.Prompt.Messages[1].Content, "Rin opens the gate.") {
t.Fatalf("expected rendered prompt in generate request, got %+v", req.Prompt)
}
if prepared.SessionID != directSession ||
req.Prompt.SessionID != directSession ||
result.SessionID != directSession {
t.Fatalf(
"direct session did not propagate consistently: prepared=%q generated=%q result=%q",
prepared.SessionID,
req.Prompt.SessionID,
result.SessionID,
)
}
if req.StructuredOutput == nil || req.StructuredOutput.Type != promptkit.StructuredOutputJSONSchema || req.StructuredOutput.JSONSchema == nil {
t.Fatalf("expected structured output handoff, got %+v", req.StructuredOutput)
}
@@ -755,6 +812,92 @@ func TestRunUsesDirectAPIKeyWithDefaultLLMClient(t *testing.T) {
}
}
func TestRunUsesResolvedBackendWithBuiltInLLMClient(t *testing.T) {
const (
backendID = "local-test"
envName = "PROMPTKIT_BACKEND_TRANSPORT_KEY"
apiKey = "synthetic-backend-key"
)
t.Setenv(envName, apiKey)
var (
gotAuth string
gotBody map[string]any
)
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
gotAuth = r.Header.Get("Authorization")
if r.URL.Path != "/v1/chat/completions" {
t.Errorf("unexpected path: %s", r.URL.Path)
}
if err := json.NewDecoder(r.Body).Decode(&gotBody); err != nil {
t.Errorf("decode request body: %v", err)
}
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{
"choices": [{"message": {"role": "assistant", "content": "# Summary\n\nDone."}}],
"usage": {"prompt_tokens": 3, "completion_tokens": 4, "total_tokens": 7}
}`))
}))
defer server.Close()
engine, err := promptkit.NewEngine(promptkit.Config{
PromptDir: frameworkPromptDir,
SchemaDir: frameworkSchemaDir,
},
promptkit.WithBackend(promptkit.Backend{
ID: backendID,
Endpoint: server.URL + "/v1",
APIKeyEnv: envName,
ExtraParams: map[string]any{
"provider": "synthetic",
},
}),
promptkit.WithProfiles(promptkit.Profile{
ID: "backend-transport",
BackendID: backendID,
Model: "test-model",
}),
)
if err != nil {
t.Fatalf("construct engine: %v", err)
}
result, err := engine.Run(context.Background(), promptkit.RunRequest{
PromptID: frameworkMarkdownSummaryPromptID,
ProfileID: "backend-transport",
Inputs: map[string]promptkit.ArtifactRef{
"transcript": promptkit.Inline("Rin opens the gate."),
"glossary": promptkit.Inline("gate: A guarded passage."),
},
})
if err != nil {
t.Fatalf("run with resolved backend: %v", err)
}
if gotAuth != "Bearer "+apiKey {
t.Fatalf("unexpected Authorization header: %q", gotAuth)
}
if gotBody["model"] != "test-model" || gotBody["provider"] != "synthetic" {
t.Fatalf("backend defaults did not reach provider payload: %#v", gotBody)
}
for _, field := range []string{"backend_id", "api_key_env"} {
if _, ok := gotBody[field]; ok {
t.Fatalf("internal metadata field %q was serialized to provider payload: %#v", field, gotBody)
}
}
bodyJSON, err := json.Marshal(gotBody)
if err != nil {
t.Fatalf("marshal captured provider payload: %v", err)
}
if strings.Contains(string(bodyJSON), apiKey) {
t.Fatalf("credential value was serialized to provider payload: %s", bodyJSON)
}
if result.SelectedBackendID != backendID ||
result.EffectiveModelParams.Endpoint != server.URL+"/v1" ||
result.EffectiveModelParams.APIKeyEnv != envName {
t.Fatalf("unexpected resolved backend metadata: %+v", result)
}
}
func TestPrepareDirectAPIKeyBypassesMissingEnvWithoutLeakingOrHashing(t *testing.T) {
const missingEnv = "PROMPTKIT_PUBLIC_PREPARE_MISSING"
const firstKey = "first-direct-key"
@@ -1020,6 +1163,7 @@ func TestPublicErrorsSupportErrorsIs(t *testing.T) {
client promptkit.LLMClient
schemaDir string
want error
notWant error
}{
{
name: "invalid request",
@@ -1028,10 +1172,11 @@ func TestPublicErrorsSupportErrorsIs(t *testing.T) {
want: promptkit.ErrInvalidRequest,
},
{
name: "prompt not found",
req: promptkit.RunRequest{PromptID: "missing.prompt"},
client: &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}},
want: promptkit.ErrPromptNotFound,
name: "prompt not found",
req: promptkit.RunRequest{PromptID: "missing.prompt"},
client: &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}},
want: promptkit.ErrPromptNotFound,
notWant: promptkit.ErrPromptLoad,
},
{
name: "profile not found",
@@ -1042,8 +1187,9 @@ func TestPublicErrorsSupportErrorsIs(t *testing.T) {
"transcript": promptkit.Inline("Rin opens the gate."),
},
},
client: &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}},
want: promptkit.ErrProfileNotFound,
client: &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}},
want: promptkit.ErrProfileNotFound,
notWant: promptkit.ErrProfileLoad,
},
{
name: "artifact load",
@@ -1114,6 +1260,9 @@ func TestPublicErrorsSupportErrorsIs(t *testing.T) {
if !errors.Is(err, tc.want) {
t.Fatalf("expected errors.Is(%v), got %v", tc.want, err)
}
if tc.notWant != nil && errors.Is(err, tc.notWant) {
t.Fatalf("did not expect errors.Is(%v), got %v", tc.notWant, err)
}
})
}
}
@@ -1265,6 +1414,18 @@ func TestPrepareUsesBuiltInProfileWithoutProfileDir(t *testing.T) {
if prepared.SelectedProfileID != "mistral-small-3" {
t.Fatalf("unexpected selected profile: %q", prepared.SelectedProfileID)
}
if prepared.SelectedBackendID != promptkit.BackendOpenRouter {
t.Fatalf("unexpected selected backend: %q", prepared.SelectedBackendID)
}
if prepared.EffectiveModelParams.BackendID != promptkit.BackendOpenRouter {
t.Fatalf("unexpected effective backend: %q", prepared.EffectiveModelParams.BackendID)
}
if prepared.EffectiveModelParams.Endpoint != "https://openrouter.ai/api/v1" {
t.Fatalf("unexpected built-in endpoint: %q", prepared.EffectiveModelParams.Endpoint)
}
if prepared.EffectiveModelParams.APIKeyEnv != "OPENROUTER_API_KEY" {
t.Fatalf("unexpected built-in api key environment name: %q", prepared.EffectiveModelParams.APIKeyEnv)
}
if prepared.EffectiveModelParams.Model != "mistralai/mistral-small-3.2-24b-instruct" {
t.Fatalf("unexpected built-in model: %q", prepared.EffectiveModelParams.Model)
}
@@ -1635,6 +1796,7 @@ func TestOpenAICompatibleProfileRunsThroughNormalProfilePath(t *testing.T) {
fake := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}}
prof := promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
ID: "template-profile",
BackendID: " openrouter ",
Endpoint: "http://template/v1",
Model: "template-model",
APIKeyRequired: true,
@@ -1666,7 +1828,9 @@ func TestOpenAICompatibleProfileRunsThroughNormalProfilePath(t *testing.T) {
if len(fake.requests) != 1 {
t.Fatalf("expected one request, got %d", len(fake.requests))
}
if fake.requests[0].Target.Model != "template-model" || fake.requests[0].APIKey != "template-key" {
if fake.requests[0].Target.BackendID != promptkit.BackendOpenRouter ||
fake.requests[0].Target.Model != "template-model" ||
fake.requests[0].APIKey != "template-key" {
t.Fatalf("unexpected generated request: %+v", fake.requests[0])
}
if !reflect.DeepEqual(fake.requests[0].Target.ExtraParams, map[string]any{"provider": "template"}) {
@@ -1699,6 +1863,16 @@ func TestEngineRunLayersTransportAndGenerationTimeouts(t *testing.T) {
configTimeout: 5 * time.Second,
wantRemainingAtRequest: 5 * time.Second,
},
{
name: "zero configuration uses ten minute transport default",
wantRemainingAtRequest: 10 * time.Minute,
},
{
name: "negative configuration uses ten minute transport default",
configTimeout: -2 * time.Second,
suppliedClientTimeout: -3 * time.Second,
wantRemainingAtRequest: 10 * time.Minute,
},
{
name: "profile deadline is shorter than transport cap",
suppliedClientTimeout: 6 * time.Second,
@@ -2127,6 +2301,8 @@ func TestSourceOptionsRejectInvalidInputs(t *testing.T) {
{name: "profile fs nil", opt: promptkit.WithProfileFS(nil, "profiles")},
{name: "profile fs empty root", opt: promptkit.WithProfileFS(fstest.MapFS{}, "")},
{name: "profile file empty", opt: promptkit.WithProfileFile("")},
{name: "fallback profile fs nil", opt: promptkit.WithFallbackProfileFS(nil, "profiles")},
{name: "fallback profile fs empty root", opt: promptkit.WithFallbackProfileFS(fstest.MapFS{}, "")},
{name: "schema fs nil", opt: promptkit.WithSchemaFS(nil, "schemas")},
{name: "schema fs empty root", opt: promptkit.WithSchemaFS(fstest.MapFS{}, "")},
{name: "schema file empty", opt: promptkit.WithSchemaFile("")},
@@ -2240,6 +2416,11 @@ func TestExtraParamsTypedNestedValuesAreCopiedAcrossPublicBoundary(t *testing.T)
}
func TestRunRejectsInvalidExtraParams(t *testing.T) {
cyclicMap := map[string]any{}
cyclicMap["self"] = cyclicMap
cyclicSlice := []any{nil}
cyclicSlice[0] = cyclicSlice
tests := []struct {
name string
extraParams map[string]any
@@ -2251,42 +2432,10 @@ func TestRunRejectsInvalidExtraParams(t *testing.T) {
{name: "nan", extraParams: map[string]any{"bad": math.NaN()}},
{name: "positive infinity", extraParams: map[string]any{"bad": math.Inf(1)}},
{name: "negative infinity", extraParams: map[string]any{"bad": math.Inf(-1)}},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
fake := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}}
engine := newContractEngineWithOptions(t, frameworkSchemaDir, promptkit.WithLLMClient(fake))
_, err := engine.Run(context.Background(), promptkit.RunRequest{
PromptID: frameworkMarkdownSummaryPromptID,
Inputs: map[string]promptkit.ArtifactRef{
"transcript": promptkit.Inline("Rin opens the gate."),
"glossary": promptkit.Inline("gate: A guarded passage."),
},
Execution: &promptkit.ExecutionTargetOverride{ExtraParams: tc.extraParams},
})
if !errors.Is(err, promptkit.ErrInvalidRequest) {
t.Fatalf("expected ErrInvalidRequest, got %v", err)
}
if len(fake.requests) != 0 {
t.Fatalf("expected invalid request to fail before LLM call, got %d requests", len(fake.requests))
}
})
}
}
func TestRunRejectsCyclicExtraParams(t *testing.T) {
cyclicMap := map[string]any{}
cyclicMap["self"] = cyclicMap
cyclicSlice := []any{nil}
cyclicSlice[0] = cyclicSlice
tests := []struct {
name string
extraParams map[string]any
}{
{name: "map", extraParams: cyclicMap},
{name: "slice", extraParams: map[string]any{"cycle": cyclicSlice}},
{name: "cyclic map", extraParams: cyclicMap},
{name: "cyclic slice", extraParams: map[string]any{"cycle": cyclicSlice}},
{name: "malformed JSON number", extraParams: map[string]any{"value": json.Number("+1")}},
{name: "empty nested key", extraParams: map[string]any{"nested": map[string]any{"": true}}},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {

View File

@@ -3,7 +3,9 @@ package promptkit
import (
"errors"
"fmt"
"strings"
"gitea.maximumdirect.net/eric/promptkit/internal/capacity"
"gitea.maximumdirect.net/eric/promptkit/internal/profile"
"gitea.maximumdirect.net/eric/promptkit/internal/promptdef"
"gitea.maximumdirect.net/eric/promptkit/internal/usecase"
@@ -13,6 +15,11 @@ func mapPublicError(err error) error {
if err == nil {
return nil
}
var internalCapacityError *usecase.CapacityError
if errors.As(err, &internalCapacityError) && internalCapacityError != nil &&
strings.TrimSpace(internalCapacityError.BackendID) != "" {
return &CapacityError{BackendID: internalCapacityError.BackendID}
}
publicErr := publicErrorFor(err)
if publicErr == nil {
return err
@@ -38,6 +45,8 @@ func publicErrorFor(err error) error {
return ErrProfileLoad
case errors.Is(err, usecase.ErrAPIKeyEnvMissing):
return errors.Join(ErrInvalidRequest, ErrAPIKeyEnvMissing)
case errors.Is(err, capacity.ErrCapacityExceeded):
return ErrCapacityExceeded
case errors.Is(err, usecase.ErrArtifactLoad):
return ErrArtifactLoad
case errors.Is(err, usecase.ErrPromptRender):

50
errors_internal_test.go Normal file
View File

@@ -0,0 +1,50 @@
package promptkit
import (
"context"
"errors"
"fmt"
"testing"
"gitea.maximumdirect.net/eric/promptkit/internal/usecase"
)
func TestMapPublicErrorPreservesGenerationCancellation(t *testing.T) {
internalErr := fmt.Errorf("%w: %w", usecase.ErrLLMGenerate, context.Canceled)
err := mapPublicError(internalErr)
if !errors.Is(err, ErrLLMGenerate) {
t.Fatalf("mapped error=%v, want ErrLLMGenerate", err)
}
if !errors.Is(err, context.Canceled) {
t.Fatalf("mapped error=%v, want context.Canceled", err)
}
}
func TestMapPublicErrorTranslatesCapacityError(t *testing.T) {
internalErr := &usecase.CapacityError{BackendID: "limited"}
err := mapPublicError(internalErr)
var publicErr *CapacityError
if !errors.As(err, &publicErr) || publicErr == nil {
t.Fatalf("mapped error=%v, want public CapacityError", err)
}
if publicErr.BackendID != "limited" {
t.Fatalf("mapped backend ID=%q, want limited", publicErr.BackendID)
}
if !errors.Is(err, ErrCapacityExceeded) {
t.Fatalf("mapped error=%v, want ErrCapacityExceeded", err)
}
if errors.Is(err, ErrInvalidRequest) || errors.Is(err, ErrLLMGenerate) {
t.Fatalf("mapped capacity error has an unrelated category: %v", err)
}
var leakedInternalErr *usecase.CapacityError
if errors.As(err, &leakedInternalErr) {
t.Fatalf("mapped error exposes internal CapacityError: %v", err)
}
internalErr.BackendID = "changed"
if publicErr.BackendID != "limited" {
t.Fatalf("mapped backend ID changed with source error: %q", publicErr.BackendID)
}
}

View File

@@ -0,0 +1,81 @@
package main
import (
"context"
"encoding/json"
"fmt"
"os"
"gitea.maximumdirect.net/eric/promptkit"
)
type deterministicClient struct{}
func (deterministicClient) Generate(
ctx context.Context,
_ promptkit.GenerateRequest,
) (*promptkit.GenerateResponse, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
return &promptkit.GenerateResponse{
Content: "Ada finished the migration review.",
Usage: promptkit.TokenUsage{
PromptTokens: 12,
CompletionTokens: 6,
TotalTokens: 18,
},
}, nil
}
type summary struct {
Output string `json:"output"`
ValidationStatus promptkit.ValidationStatus `json:"validation_status"`
IsValid bool `json:"is_valid"`
Model string `json:"model"`
TotalTokens int `json:"total_tokens"`
}
func main() {
engine, err := promptkit.NewEngine(
promptkit.Config{},
promptkit.WithPromptFile("examples/go-library/run/prompt.yaml"),
promptkit.WithProfiles(promptkit.Profile{
ID: "offline-example",
Endpoint: "https://example.invalid/v1",
Model: "offline-model",
}),
promptkit.WithLLMClient(deterministicClient{}),
)
if err != nil {
exit(err)
}
result, err := engine.Run(context.Background(), promptkit.RunRequest{
PromptID: "example.run",
Inputs: map[string]promptkit.ArtifactRef{
"note": promptkit.Inline("Ada finished the migration review."),
},
})
if err != nil {
exit(err)
}
encoder := json.NewEncoder(os.Stdout)
encoder.SetIndent("", " ")
if err := encoder.Encode(summary{
Output: result.RawOutput,
ValidationStatus: result.Validation.Status,
IsValid: result.Validation.IsValid,
Model: result.ModelName,
TotalTokens: result.Usage.TotalTokens,
}); err != nil {
exit(err)
}
}
func exit(err error) {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}

View File

@@ -0,0 +1,16 @@
id: example.run
version: "1.0.0"
default_profile: offline-example
description: Run a prompt with a deterministic injected model client.
inputs:
- name: note
required: true
content_type: text/plain
messages:
- role: system
content: Summarize the note in one sentence.
- role: user
content: '{{input "note"}}'
output:
format: text
validation_mode: basic

View File

@@ -2,19 +2,23 @@ package promptkit
import "fmt"
// String returns a concise request summary without exposing direct API keys.
// String returns a concise request summary without exposing the direct API key
// or input and variable contents. Reflection-based formatting does not carry
// this guarantee.
func (r RunRequest) String() string {
return r.redactedString()
}
// GoString returns a concise request summary without exposing direct API keys.
// GoString returns a concise request summary without exposing the direct API
// key or input and variable contents. Reflection-based formatting does not
// carry this guarantee.
func (r RunRequest) GoString() string {
return r.redactedString()
}
func (r RunRequest) redactedString() string {
return fmt.Sprintf(
"promptkit.RunRequest{PromptID:%q PromptVersion:%q ProfileID:%q APIKeySet:%t Inputs:%d Vars:%d ExecutionSet:%t ValidationSet:%t Metadata:%d}",
"promptkit.RunRequest{PromptID:%q PromptVersion:%q ProfileID:%q APIKeySet:%t Inputs:%d Vars:%d ExecutionSet:%t ValidationSet:%t}",
r.PromptID,
r.PromptVersion,
r.ProfileID,
@@ -23,18 +27,19 @@ func (r RunRequest) redactedString() string {
len(r.Vars),
r.Execution != nil,
r.Validation != nil,
len(r.Metadata),
)
}
// String returns a concise request summary without exposing direct API keys or
// rendered prompt content.
// rendered prompt content. Reflection-based formatting does not carry this
// guarantee.
func (r GenerateRequest) String() string {
return r.redactedString()
}
// GoString returns a concise request summary without exposing direct API keys or
// rendered prompt content.
// rendered prompt content. Reflection-based formatting does not carry this
// guarantee.
func (r GenerateRequest) GoString() string {
return r.redactedString()
}

View File

@@ -0,0 +1,212 @@
// Package backend owns validated, immutable OpenAI-compatible backend
// definitions.
package backend
import (
"errors"
"fmt"
"net/url"
"regexp"
"sort"
"strings"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
"gitea.maximumdirect.net/eric/promptkit/internal/jsonvalue"
"gitea.maximumdirect.net/eric/promptkit/internal/llm"
)
const (
// OpenRouterID is the reserved ID of Promptkit's built-in OpenRouter
// backend.
OpenRouterID = "openrouter"
openRouterEndpoint = "https://openrouter.ai/api/v1"
openRouterAPIKeyEnv = "OPENROUTER_API_KEY"
openRouterConcurrencyLimit = 16
defaultQueueCapacity = 1024
)
// ErrBackendNotFound identifies a registry lookup for an unknown backend ID.
var ErrBackendNotFound = errors.New("backend not found")
var environmentVariableName = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`)
// Registry is an immutable collection of validated backend definitions.
type Registry struct {
backends map[string]domain.Backend
}
// NewRegistry constructs a registry containing the built-in OpenRouter
// definition followed by the supplied additions. Every ID must be unique.
func NewRegistry(additions []domain.Backend) (*Registry, error) {
registry := &Registry{
backends: make(map[string]domain.Backend, len(additions)+1),
}
definitions := make([]domain.Backend, 0, len(additions)+1)
definitions = append(definitions, domain.Backend{
ID: OpenRouterID,
Endpoint: openRouterEndpoint,
APIKeyEnv: openRouterAPIKeyEnv,
ConcurrencyLimit: openRouterConcurrencyLimit,
})
definitions = append(definitions, additions...)
for _, definition := range definitions {
definition.ID = strings.TrimSpace(definition.ID)
if definition.ID == "" {
return nil, errors.New("backend ID must not be blank")
}
if _, exists := registry.backends[definition.ID]; exists {
return nil, fmt.Errorf("backend ID %q is already registered", definition.ID)
}
normalized, err := normalizeBackend(definition)
if err != nil {
return nil, err
}
registry.backends[normalized.ID] = normalized
}
return registry, nil
}
// GetBackend returns a defensive copy of the backend registered with id.
func (r *Registry) GetBackend(id string) (domain.Backend, error) {
if r == nil {
return domain.Backend{}, fmt.Errorf("%w: %q", ErrBackendNotFound, id)
}
definition, ok := r.backends[id]
if !ok {
return domain.Backend{}, fmt.Errorf("%w: %q", ErrBackendNotFound, id)
}
extraParams, err := jsonvalue.CopyMap(definition.ExtraParams)
if err != nil {
return domain.Backend{}, fmt.Errorf("copy backend %q: %w", id, err)
}
definition.ExtraParams = extraParams
return definition, nil
}
// CapacityPolicies returns a copy of the normalized policies for limited
// backends.
func (r *Registry) CapacityPolicies() map[string]domain.BackendCapacityPolicy {
policies := make(map[string]domain.BackendCapacityPolicy)
if r == nil {
return policies
}
for id, definition := range r.backends {
if definition.ConcurrencyLimit == 0 {
continue
}
policies[id] = domain.BackendCapacityPolicy{
ConcurrencyLimit: definition.ConcurrencyLimit,
QueueCapacity: definition.QueueCapacity,
}
}
return policies
}
func normalizeBackend(definition domain.Backend) (domain.Backend, error) {
definition.Endpoint = strings.TrimSpace(definition.Endpoint)
if err := validateEndpoint(definition.Endpoint); err != nil {
return domain.Backend{}, fmt.Errorf("backend %q endpoint: %w", definition.ID, err)
}
definition.APIKeyEnv = strings.TrimSpace(definition.APIKeyEnv)
if definition.APIKeyEnv != "" && !environmentVariableName.MatchString(definition.APIKeyEnv) {
return domain.Backend{}, fmt.Errorf(
"backend %q api key environment variable %q is invalid",
definition.ID,
definition.APIKeyEnv,
)
}
if definition.ConcurrencyLimit < 0 {
return domain.Backend{}, fmt.Errorf(
"backend %q concurrency limit must not be negative",
definition.ID,
)
}
if definition.QueueCapacity < 0 {
return domain.Backend{}, fmt.Errorf(
"backend %q queue capacity must not be negative",
definition.ID,
)
}
if definition.ConcurrencyLimit == 0 {
if definition.QueueCapacitySet {
return domain.Backend{}, fmt.Errorf(
"backend %q queue capacity requires a positive concurrency limit",
definition.ID,
)
}
definition.QueueCapacity = 0
} else {
if !definition.QueueCapacitySet {
definition.QueueCapacity = defaultQueueCapacity
definition.QueueCapacitySet = true
}
maxInt := int(^uint(0) >> 1)
if definition.QueueCapacity > maxInt-definition.ConcurrencyLimit {
return domain.Backend{}, fmt.Errorf(
"backend %q total capacity overflows int",
definition.ID,
)
}
}
keys := make([]string, 0, len(definition.ExtraParams))
for key := range definition.ExtraParams {
keys = append(keys, key)
}
sort.Strings(keys)
for _, key := range keys {
if key == "" {
return domain.Backend{}, fmt.Errorf("backend %q extra parameter key must not be empty", definition.ID)
}
if llm.IsReservedOpenAIChatRequestField(key) {
return domain.Backend{}, fmt.Errorf(
"backend %q extra parameter %q collides with a reserved request field",
definition.ID,
key,
)
}
}
extraParams, err := jsonvalue.CopyMap(definition.ExtraParams)
if err != nil {
return domain.Backend{}, fmt.Errorf("backend %q extra parameters: %w", definition.ID, err)
}
definition.ExtraParams = extraParams
return definition, nil
}
func validateEndpoint(endpoint string) error {
if endpoint == "" {
return errors.New("must not be blank")
}
if strings.Contains(endpoint, "#") {
return errors.New("must not contain a fragment")
}
parsed, err := url.Parse(endpoint)
if err != nil {
return fmt.Errorf("must be a valid URL: %w", err)
}
scheme := strings.ToLower(parsed.Scheme)
if scheme != "http" && scheme != "https" {
return errors.New("must use http or https")
}
if !parsed.IsAbs() || parsed.Hostname() == "" {
return errors.New("must be absolute and include a host")
}
if parsed.User != nil {
return errors.New("must not contain user information")
}
if parsed.RawQuery != "" || parsed.ForceQuery {
return errors.New("must not contain a query string")
}
return nil
}

View File

@@ -0,0 +1,368 @@
package backend_test
import (
"errors"
"strings"
"testing"
"gitea.maximumdirect.net/eric/promptkit/internal/backend"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
)
const validEndpoint = "https://backend.example/v1"
func TestRegistryIncludesExactOpenRouterDefinition(t *testing.T) {
registry, err := backend.NewRegistry(nil)
if err != nil {
t.Fatalf("construct registry: %v", err)
}
definition, err := registry.GetBackend(backend.OpenRouterID)
if err != nil {
t.Fatalf("look up OpenRouter: %v", err)
}
if definition.ID != "openrouter" ||
definition.Endpoint != "https://openrouter.ai/api/v1" ||
definition.APIKeyEnv != "OPENROUTER_API_KEY" ||
definition.ConcurrencyLimit != 16 ||
definition.QueueCapacity != 1024 ||
!definition.QueueCapacitySet ||
definition.ExtraParams != nil {
t.Fatalf("unexpected OpenRouter definition: %#v", definition)
}
policies := registry.CapacityPolicies()
if len(policies) != 1 ||
policies["openrouter"] != (domain.BackendCapacityPolicy{
ConcurrencyLimit: 16,
QueueCapacity: 1024,
}) {
t.Fatalf("unexpected OpenRouter capacity policies: %#v", policies)
}
}
func TestRegistryNormalizesUniqueAdditionsAndIsolatesMutations(t *testing.T) {
nested := map[string]int{"limit": 2}
extraParams := map[string]any{
"count": int64(7),
"nested": nested,
}
registry, err := backend.NewRegistry([]domain.Backend{
{
ID: " custom ",
Endpoint: " https://custom.example/openai/v1 ",
APIKeyEnv: " CUSTOM_API_KEY ",
ExtraParams: extraParams,
ConcurrencyLimit: 3,
QueueCapacity: 2,
QueueCapacitySet: true,
},
{
ID: "Custom",
Endpoint: validEndpoint,
},
})
if err != nil {
t.Fatalf("construct registry: %v", err)
}
nested["limit"] = 99
extraParams["added"] = true
got, err := registry.GetBackend("custom")
if err != nil {
t.Fatalf("look up custom backend: %v", err)
}
if got.ID != "custom" ||
got.Endpoint != "https://custom.example/openai/v1" ||
got.APIKeyEnv != "CUSTOM_API_KEY" ||
got.ConcurrencyLimit != 3 ||
got.QueueCapacity != 2 ||
!got.QueueCapacitySet {
t.Fatalf("unexpected normalized definition: %#v", got)
}
if count, ok := got.ExtraParams["count"].(int64); !ok || count != 7 {
t.Fatalf("integer type or value changed: %#v", got.ExtraParams["count"])
}
gotNested, ok := got.ExtraParams["nested"].(map[string]int)
if !ok || gotNested["limit"] != 2 {
t.Fatalf("container type or value changed: %#v", got.ExtraParams["nested"])
}
if _, exists := got.ExtraParams["added"]; exists {
t.Fatalf("registry retained caller map: %#v", got.ExtraParams)
}
gotNested["limit"] = 100
got.ExtraParams["added"] = true
again, err := registry.GetBackend("custom")
if err != nil {
t.Fatalf("look up custom backend again: %v", err)
}
if again.ExtraParams["nested"].(map[string]int)["limit"] != 2 {
t.Fatalf("lookup exposed registry nested map: %#v", again.ExtraParams)
}
if _, exists := again.ExtraParams["added"]; exists {
t.Fatalf("lookup exposed registry map: %#v", again.ExtraParams)
}
if _, err := registry.GetBackend("Custom"); err != nil {
t.Fatalf("backend IDs should be case-sensitive: %v", err)
}
policies := registry.CapacityPolicies()
if len(policies) != 2 {
t.Fatalf("unexpected capacity policy count: %#v", policies)
}
policies["custom"] = domain.BackendCapacityPolicy{}
delete(policies, backend.OpenRouterID)
againPolicies := registry.CapacityPolicies()
if againPolicies["custom"] != (domain.BackendCapacityPolicy{
ConcurrencyLimit: 3,
QueueCapacity: 2,
}) {
t.Fatalf("capacity policy map mutated registry state: %#v", againPolicies)
}
if _, ok := againPolicies[backend.OpenRouterID]; !ok {
t.Fatalf("capacity policy deletion mutated registry state: %#v", againPolicies)
}
}
func TestNewRegistryNormalizesCapacityPolicy(t *testing.T) {
maxInt := int(^uint(0) >> 1)
tests := []struct {
name string
definition domain.Backend
want domain.BackendCapacityPolicy
wantSet bool
wantError bool
}{
{
name: "unlimited when omitted",
definition: domain.Backend{},
},
{
name: "default queue",
definition: domain.Backend{
ConcurrencyLimit: 2,
},
want: domain.BackendCapacityPolicy{
ConcurrencyLimit: 2,
QueueCapacity: 1024,
},
wantSet: true,
},
{
name: "explicit zero queue",
definition: domain.Backend{
ConcurrencyLimit: 2,
QueueCapacitySet: true,
},
want: domain.BackendCapacityPolicy{
ConcurrencyLimit: 2,
},
wantSet: true,
},
{
name: "negative concurrency limit",
definition: domain.Backend{
ConcurrencyLimit: -1,
},
wantError: true,
},
{
name: "negative queue capacity",
definition: domain.Backend{
ConcurrencyLimit: 1,
QueueCapacity: -1,
QueueCapacitySet: true,
},
wantError: true,
},
{
name: "queue without limit",
definition: domain.Backend{
QueueCapacitySet: true,
},
wantError: true,
},
{
name: "total overflow",
definition: domain.Backend{
ConcurrencyLimit: maxInt,
QueueCapacity: 1,
QueueCapacitySet: true,
},
wantError: true,
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
tc.definition.ID = "custom"
tc.definition.Endpoint = validEndpoint
registry, err := backend.NewRegistry([]domain.Backend{tc.definition})
if tc.wantError {
if err == nil {
t.Fatal("expected invalid capacity policy error")
}
return
}
if err != nil {
t.Fatalf("construct registry: %v", err)
}
definition, err := registry.GetBackend("custom")
if err != nil {
t.Fatalf("look up custom backend: %v", err)
}
if definition.ConcurrencyLimit != tc.want.ConcurrencyLimit ||
definition.QueueCapacity != tc.want.QueueCapacity ||
definition.QueueCapacitySet != tc.wantSet {
t.Fatalf("normalized capacity=(%d, %d, %t), want (%d, %d, %t)",
definition.ConcurrencyLimit,
definition.QueueCapacity,
definition.QueueCapacitySet,
tc.want.ConcurrencyLimit,
tc.want.QueueCapacity,
tc.wantSet,
)
}
policies := registry.CapacityPolicies()
got, ok := policies["custom"]
if ok != tc.wantSet || got != tc.want {
t.Fatalf("capacity policy=(%#v, %t), want (%#v, %t)", got, ok, tc.want, tc.wantSet)
}
})
}
}
func TestNewRegistryRejectsDuplicateIDs(t *testing.T) {
tests := []struct {
name string
additions []domain.Backend
wantID string
}{
{
name: "built-in collision after normalization",
additions: []domain.Backend{{
ID: " openrouter ",
}},
wantID: "openrouter",
},
{
name: "consumer collision after normalization",
additions: []domain.Backend{
{ID: "custom", Endpoint: validEndpoint},
{ID: " custom ", Endpoint: "https://other.example/v1"},
},
wantID: "custom",
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
_, err := backend.NewRegistry(tc.additions)
if err == nil {
t.Fatal("expected duplicate ID error")
}
if !strings.Contains(err.Error(), tc.wantID) {
t.Fatalf("expected error to identify %q, got %v", tc.wantID, err)
}
})
}
}
func TestNewRegistryValidatesIDs(t *testing.T) {
for _, id := range []string{"", " \t\n "} {
t.Run(id, func(t *testing.T) {
_, err := backend.NewRegistry([]domain.Backend{{
ID: id,
Endpoint: validEndpoint,
}})
if err == nil {
t.Fatal("expected blank ID error")
}
})
}
}
func TestNewRegistryValidatesEndpoints(t *testing.T) {
tests := []struct {
name string
endpoint string
}{
{name: "blank", endpoint: ""},
{name: "relative", endpoint: "/v1"},
{name: "missing host", endpoint: "https:///v1"},
{name: "unsupported scheme", endpoint: "ftp://backend.example/v1"},
{name: "user information", endpoint: "https://user@backend.example/v1"},
{name: "query", endpoint: "https://backend.example/v1?mode=chat"},
{name: "empty query", endpoint: "https://backend.example/v1?"},
{name: "fragment", endpoint: "https://backend.example/v1#chat"},
{name: "empty fragment", endpoint: "https://backend.example/v1#"},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
_, err := backend.NewRegistry([]domain.Backend{{
ID: "custom",
Endpoint: tc.endpoint,
}})
if err == nil {
t.Fatal("expected invalid endpoint error")
}
})
}
}
func TestNewRegistryValidatesEnvironmentVariableNames(t *testing.T) {
for _, name := range []string{"1API_KEY", "API-KEY", "API KEY", "ÅPI_KEY"} {
t.Run(name, func(t *testing.T) {
_, err := backend.NewRegistry([]domain.Backend{{
ID: "custom",
Endpoint: validEndpoint,
APIKeyEnv: name,
}})
if err == nil {
t.Fatal("expected invalid environment-variable name error")
}
})
}
}
func TestNewRegistryRejectsInvalidAndReservedExtraParameters(t *testing.T) {
tests := []struct {
name string
extraParams map[string]any
}{
{name: "unsupported value", extraParams: map[string]any{"value": make(chan int)}},
{name: "reserved key", extraParams: map[string]any{"model": "override"}},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
_, err := backend.NewRegistry([]domain.Backend{{
ID: "custom",
Endpoint: validEndpoint,
ExtraParams: tc.extraParams,
}})
if err == nil {
t.Fatal("expected invalid extra parameters error")
}
})
}
}
func TestRegistryLookupReportsNotFound(t *testing.T) {
registry, err := backend.NewRegistry(nil)
if err != nil {
t.Fatalf("construct registry: %v", err)
}
_, err = registry.GetBackend("missing")
if !errors.Is(err, backend.ErrBackendNotFound) {
t.Fatalf("expected ErrBackendNotFound, got %v", err)
}
if !strings.Contains(err.Error(), "missing") {
t.Fatalf("expected error to identify backend, got %v", err)
}
}

View File

@@ -0,0 +1,40 @@
package capacity
import (
"context"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
"gitea.maximumdirect.net/eric/promptkit/internal/llm"
)
type client struct {
manager *Manager
next llm.Client
}
// NewClient wraps next with configured active-generation limits. A nil manager
// leaves next unchanged.
func NewClient(manager *Manager, next llm.Client) llm.Client {
if manager == nil {
return next
}
return &client{
manager: manager,
next: next,
}
}
func (c *client) Generate(
ctx context.Context,
req domain.GenerateRequest,
) (*domain.GenerateResponse, error) {
pool := c.manager.getPool(req.Target.BackendID)
if pool == nil {
return c.next.Generate(ctx, req)
}
if err := pool.acquire(ctx); err != nil {
return nil, err
}
defer pool.releaseActive()
return c.next.Generate(ctx, req)
}

View File

@@ -0,0 +1,517 @@
package capacity
import (
"context"
"errors"
"reflect"
"runtime"
"sync"
"sync/atomic"
"testing"
"time"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
"gitea.maximumdirect.net/eric/promptkit/internal/llm"
)
type generateResult struct {
response *domain.GenerateResponse
err error
}
type clientFunc func(
context.Context,
domain.GenerateRequest,
) (*domain.GenerateResponse, error)
func (f clientFunc) Generate(
ctx context.Context,
req domain.GenerateRequest,
) (*domain.GenerateResponse, error) {
return f(ctx, req)
}
type blockingClient struct {
mu sync.Mutex
active int
peak int
calls map[string]int
started chan string
releases map[string]chan struct{}
}
func newBlockingClient(releases map[string]chan struct{}) *blockingClient {
return &blockingClient{
calls: make(map[string]int),
started: make(chan string, 64),
releases: releases,
}
}
func (c *blockingClient) Generate(
ctx context.Context,
req domain.GenerateRequest,
) (*domain.GenerateResponse, error) {
id := req.Prompt.SessionID
c.mu.Lock()
c.active++
if c.active > c.peak {
c.peak = c.active
}
c.calls[id]++
c.mu.Unlock()
defer func() {
c.mu.Lock()
c.active--
c.mu.Unlock()
}()
c.started <- id
if release := c.releases[id]; release != nil {
select {
case <-release:
case <-ctx.Done():
return nil, ctx.Err()
}
}
return &domain.GenerateResponse{Content: id}, nil
}
func (c *blockingClient) callCount(id string) int {
c.mu.Lock()
defer c.mu.Unlock()
return c.calls[id]
}
func (c *blockingClient) peakConcurrency() int {
c.mu.Lock()
defer c.mu.Unlock()
return c.peak
}
func generateAsync(
client llm.Client,
ctx context.Context,
backendID string,
id string,
) <-chan generateResult {
result := make(chan generateResult, 1)
go func() {
response, err := client.Generate(ctx, domain.GenerateRequest{
Prompt: domain.RenderedPrompt{SessionID: id},
Target: domain.ExecutionTarget{BackendID: backendID},
})
result <- generateResult{response: response, err: err}
}()
return result
}
func waitForWaiterCount(t *testing.T, manager *Manager, backendID string, want int) {
t.Helper()
pool := manager.pools[backendID]
deadline := time.Now().Add(2 * time.Second)
for {
pool.mu.Lock()
got := pool.waiters.Len()
pool.mu.Unlock()
if got == want {
return
}
if time.Now().After(deadline) {
t.Fatalf("waiter count=%d, want %d", got, want)
}
runtime.Gosched()
}
}
func receiveStarted(t *testing.T, started <-chan string) string {
t.Helper()
select {
case id := <-started:
return id
case <-time.After(2 * time.Second):
t.Fatal("timed out waiting for wrapped client invocation")
return ""
}
}
func receiveResult(t *testing.T, result <-chan generateResult) generateResult {
t.Helper()
select {
case got := <-result:
return got
case <-time.After(2 * time.Second):
t.Fatal("timed out waiting for generation result")
return generateResult{}
}
}
func newTestManager(t *testing.T, policies map[string]domain.BackendCapacityPolicy) *Manager {
t.Helper()
manager, err := NewManager(policies)
if err != nil {
t.Fatalf("construct manager: %v", err)
}
return manager
}
func TestClientLimitsPeakConcurrencyAndServesWaitersFIFO(t *testing.T) {
manager := newTestManager(t, map[string]domain.BackendCapacityPolicy{
"limited": {ConcurrencyLimit: 1},
})
firstRelease := make(chan struct{})
secondRelease := make(chan struct{})
thirdRelease := make(chan struct{})
next := newBlockingClient(map[string]chan struct{}{
"first": firstRelease,
"second": secondRelease,
"third": thirdRelease,
})
client := NewClient(manager, next)
first := generateAsync(client, context.Background(), "limited", "first")
if got := receiveStarted(t, next.started); got != "first" {
t.Fatalf("first invocation=%q, want first", got)
}
second := generateAsync(client, context.Background(), "limited", "second")
waitForWaiterCount(t, manager, "limited", 1)
third := generateAsync(client, context.Background(), "limited", "third")
waitForWaiterCount(t, manager, "limited", 2)
close(firstRelease)
if got := receiveResult(t, first); got.err != nil {
t.Fatalf("first generation: %v", got.err)
}
if got := receiveStarted(t, next.started); got != "second" {
t.Fatalf("second invocation=%q, want second", got)
}
close(secondRelease)
if got := receiveResult(t, second); got.err != nil {
t.Fatalf("second generation: %v", got.err)
}
if got := receiveStarted(t, next.started); got != "third" {
t.Fatalf("third invocation=%q, want third", got)
}
close(thirdRelease)
if got := receiveResult(t, third); got.err != nil {
t.Fatalf("third generation: %v", got.err)
}
if peak := next.peakConcurrency(); peak != 1 {
t.Fatalf("peak concurrency=%d, want 1", peak)
}
}
func TestClientPeakConcurrencyDoesNotExceedConfiguredLimit(t *testing.T) {
const limit = 2
manager := newTestManager(t, map[string]domain.BackendCapacityPolicy{
"limited": {ConcurrencyLimit: limit},
})
gate := make(chan struct{})
releases := make(map[string]chan struct{})
for i := range 5 {
releases[string(rune('a'+i))] = gate
}
next := newBlockingClient(releases)
client := NewClient(manager, next)
results := make([]<-chan generateResult, 0, len(releases))
for id := range releases {
results = append(results, generateAsync(client, context.Background(), "limited", id))
}
for range limit {
receiveStarted(t, next.started)
}
waitForWaiterCount(t, manager, "limited", len(releases)-limit)
close(gate)
for _, result := range results {
if got := receiveResult(t, result); got.err != nil {
t.Fatalf("generation: %v", got.err)
}
}
if peak := next.peakConcurrency(); peak != limit {
t.Fatalf("peak concurrency=%d, want %d", peak, limit)
}
}
func TestClientRemovesCanceledWaiters(t *testing.T) {
tests := []struct {
name string
cancelID string
wantOrder []string
}{
{name: "first waiter", cancelID: "one", wantOrder: []string{"two", "three"}},
{name: "middle waiter", cancelID: "two", wantOrder: []string{"one", "three"}},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
manager := newTestManager(t, map[string]domain.BackendCapacityPolicy{
"limited": {ConcurrencyLimit: 1},
})
holderRelease := make(chan struct{})
releases := map[string]chan struct{}{
"holder": holderRelease,
"one": make(chan struct{}),
"two": make(chan struct{}),
"three": make(chan struct{}),
}
next := newBlockingClient(releases)
client := NewClient(manager, next)
holder := generateAsync(client, context.Background(), "limited", "holder")
if got := receiveStarted(t, next.started); got != "holder" {
t.Fatalf("initial invocation=%q, want holder", got)
}
contexts := make(map[string]context.Context)
cancels := make(map[string]context.CancelFunc)
results := make(map[string]<-chan generateResult)
for _, id := range []string{"one", "two", "three"} {
contexts[id], cancels[id] = context.WithCancel(context.Background())
results[id] = generateAsync(client, contexts[id], "limited", id)
waitForWaiterCount(t, manager, "limited", len(results))
}
cancels[tc.cancelID]()
if got := receiveResult(t, results[tc.cancelID]); !errors.Is(got.err, context.Canceled) {
t.Fatalf("canceled waiter error=%v, want context.Canceled", got.err)
}
waitForWaiterCount(t, manager, "limited", 2)
close(holderRelease)
if got := receiveResult(t, holder); got.err != nil {
t.Fatalf("holder generation: %v", got.err)
}
for _, id := range tc.wantOrder {
if got := receiveStarted(t, next.started); got != id {
t.Fatalf("next invocation=%q, want %q", got, id)
}
close(releases[id])
if got := receiveResult(t, results[id]); got.err != nil {
t.Fatalf("%s generation: %v", id, got.err)
}
}
if calls := next.callCount(tc.cancelID); calls != 0 {
t.Fatalf("canceled waiter invoked wrapped client %d times", calls)
}
for _, cancel := range cancels {
cancel()
}
})
}
}
func TestClientGrantCancellationRaceDoesNotLeakPermit(t *testing.T) {
const iterations = 200
for i := range iterations {
manager := newTestManager(t, map[string]domain.BackendCapacityPolicy{
"limited": {ConcurrencyLimit: 1},
})
holderRelease := make(chan struct{})
var waiterCalls atomic.Int64
next := clientFunc(func(
_ context.Context,
req domain.GenerateRequest,
) (*domain.GenerateResponse, error) {
if req.Prompt.SessionID == "holder" {
<-holderRelease
} else if req.Prompt.SessionID == "waiter" {
waiterCalls.Add(1)
}
return &domain.GenerateResponse{Content: req.Prompt.SessionID}, nil
})
client := NewClient(manager, next)
holder := generateAsync(client, context.Background(), "limited", "holder")
waitForActiveCount(t, manager, "limited", 1)
ctx, cancel := context.WithCancel(context.Background())
waiterResult := generateAsync(client, ctx, "limited", "waiter")
waitForWaiterCount(t, manager, "limited", 1)
start := make(chan struct{})
var race sync.WaitGroup
race.Add(2)
go func() {
defer race.Done()
<-start
cancel()
}()
go func() {
defer race.Done()
<-start
close(holderRelease)
}()
close(start)
race.Wait()
if got := receiveResult(t, holder); got.err != nil {
t.Fatalf("iteration %d holder generation: %v", i, got.err)
}
got := receiveResult(t, waiterResult)
switch calls := waiterCalls.Load(); {
case calls == 0 && errors.Is(got.err, context.Canceled):
case calls == 1 && got.err == nil:
default:
t.Fatalf("iteration %d waiter calls=%d error=%v", i, calls, got.err)
}
probe := generateAsync(client, context.Background(), "limited", "probe")
if got := receiveResult(t, probe); got.err != nil {
t.Fatalf("iteration %d probe generation: %v", i, got.err)
}
waitForActiveCount(t, manager, "limited", 0)
waitForWaiterCount(t, manager, "limited", 0)
}
}
func waitForActiveCount(t *testing.T, manager *Manager, backendID string, want int) {
t.Helper()
pool := manager.pools[backendID]
deadline := time.Now().Add(2 * time.Second)
for {
pool.mu.Lock()
got := pool.active
pool.mu.Unlock()
if got == want {
return
}
if time.Now().After(deadline) {
t.Fatalf("active count=%d, want %d", got, want)
}
runtime.Gosched()
}
}
func TestClientUsesIndependentPoolsAndUnlimitedFastPaths(t *testing.T) {
manager := newTestManager(t, map[string]domain.BackendCapacityPolicy{
"alpha": {ConcurrencyLimit: 1},
"beta": {ConcurrencyLimit: 1},
})
alphaRelease := make(chan struct{})
betaRelease := make(chan struct{})
next := newBlockingClient(map[string]chan struct{}{
"alpha": alphaRelease,
"beta": betaRelease,
})
client := NewClient(manager, next)
alpha := generateAsync(client, context.Background(), "alpha", "alpha")
beta := generateAsync(client, context.Background(), "beta", "beta")
started := map[string]bool{
receiveStarted(t, next.started): true,
receiveStarted(t, next.started): true,
}
if !started["alpha"] || !started["beta"] {
t.Fatalf("independent pools did not both start: %#v", started)
}
close(alphaRelease)
close(betaRelease)
if got := receiveResult(t, alpha); got.err != nil {
t.Fatalf("alpha generation: %v", got.err)
}
if got := receiveResult(t, beta); got.err != nil {
t.Fatalf("beta generation: %v", got.err)
}
for _, backendID := range []string{"", "unknown"} {
response, err := client.Generate(context.Background(), domain.GenerateRequest{
Prompt: domain.RenderedPrompt{SessionID: backendID},
Target: domain.ExecutionTarget{BackendID: backendID},
})
if err != nil || response == nil {
t.Fatalf("unlimited backend %q response=(%#v, %v)", backendID, response, err)
}
}
if got := NewClient(nil, next); got != next {
t.Fatal("nil manager did not return the wrapped client unchanged")
}
}
func TestClientPreservesRequestsResponsesAndErrors(t *testing.T) {
manager := newTestManager(t, map[string]domain.BackendCapacityPolicy{
"limited": {ConcurrencyLimit: 1},
})
request := domain.GenerateRequest{
Prompt: domain.RenderedPrompt{
SessionID: "session",
Messages: []domain.RenderedMessage{
{Role: "user", Content: "content"},
},
},
Target: domain.ExecutionTarget{
BackendID: "limited",
Model: "model",
ExtraParams: map[string]any{"key": "value"},
},
}
response := &domain.GenerateResponse{
Content: "output",
Usage: domain.TokenUsage{TotalTokens: 7},
}
collaboratorErr := errors.New("collaborator failure")
tests := []struct {
name string
response *domain.GenerateResponse
err error
}{
{name: "successful response", response: response},
{name: "nil response"},
{name: "collaborator error", response: response, err: collaboratorErr},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
var captured domain.GenerateRequest
next := clientFunc(func(
_ context.Context,
req domain.GenerateRequest,
) (*domain.GenerateResponse, error) {
captured = req
return tc.response, tc.err
})
gotResponse, gotErr := NewClient(manager, next).Generate(context.Background(), request)
if !reflect.DeepEqual(captured, request) {
t.Fatalf("request changed: %#v", captured)
}
if gotResponse != tc.response || gotErr != tc.err {
t.Fatalf("response=(%p, %v), want (%p, %v)",
gotResponse, gotErr, tc.response, tc.err)
}
})
}
}
func TestClientReleasesPermitDuringPanicUnwinding(t *testing.T) {
manager := newTestManager(t, map[string]domain.BackendCapacityPolicy{
"limited": {ConcurrencyLimit: 1},
})
var calls atomic.Int64
next := clientFunc(func(
_ context.Context,
_ domain.GenerateRequest,
) (*domain.GenerateResponse, error) {
if calls.Add(1) == 1 {
panic("test panic")
}
return &domain.GenerateResponse{Content: "recovered"}, nil
})
client := NewClient(manager, next)
request := domain.GenerateRequest{
Target: domain.ExecutionTarget{BackendID: "limited"},
}
func() {
defer func() {
if recover() == nil {
t.Fatal("expected wrapped client panic")
}
}()
_, _ = client.Generate(context.Background(), request)
}()
response, err := client.Generate(context.Background(), request)
if err != nil || response == nil || response.Content != "recovered" {
t.Fatalf("generation after panic=(%#v, %v)", response, err)
}
}

View File

@@ -0,0 +1,160 @@
// Package capacity coordinates engine-local run admission and model-generation
// concurrency for configured backends.
package capacity
import (
"container/list"
"context"
"errors"
"fmt"
"strings"
"sync"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
)
// ErrCapacityExceeded identifies an admission rejected because a backend's
// configured run capacity is full.
var ErrCapacityExceeded = errors.New("backend capacity exceeded")
// Manager owns independent backend capacity pools with immutable limits.
type Manager struct {
pools map[string]*pool
}
type pool struct {
mu sync.Mutex
concurrencyLimit int
totalCapacity int
admitted int
active int
waiters list.List
}
type waiter struct {
ready chan struct{}
element *list.Element
granted bool
}
// NewManager constructs independent pools from normalized backend policies.
func NewManager(policies map[string]domain.BackendCapacityPolicy) (*Manager, error) {
manager := &Manager{
pools: make(map[string]*pool, len(policies)),
}
maxInt := int(^uint(0) >> 1)
for id, policy := range policies {
if strings.TrimSpace(id) == "" {
return nil, errors.New("backend capacity policy ID must not be blank")
}
if policy.ConcurrencyLimit <= 0 {
return nil, fmt.Errorf(
"backend %q concurrency limit must be positive",
id,
)
}
if policy.QueueCapacity < 0 {
return nil, fmt.Errorf(
"backend %q queue capacity must not be negative",
id,
)
}
if policy.QueueCapacity > maxInt-policy.ConcurrencyLimit {
return nil, fmt.Errorf("backend %q total capacity overflows int", id)
}
manager.pools[id] = &pool{
concurrencyLimit: policy.ConcurrencyLimit,
totalCapacity: policy.ConcurrencyLimit + policy.QueueCapacity,
}
}
return manager, nil
}
// Admit immediately reserves one configured backend run slot. Backends without
// a configured pool are unlimited.
func (m *Manager) Admit(ctx context.Context, backendID string) (func(), error) {
pool := m.getPool(backendID)
if pool == nil {
return releaseNothing, nil
}
pool.mu.Lock()
defer pool.mu.Unlock()
if err := ctx.Err(); err != nil {
return nil, err
}
if pool.admitted >= pool.totalCapacity {
return nil, ErrCapacityExceeded
}
pool.admitted++
var once sync.Once
return func() {
once.Do(func() {
pool.mu.Lock()
pool.admitted--
pool.mu.Unlock()
})
}, nil
}
func releaseNothing() {}
func (m *Manager) getPool(backendID string) *pool {
if m == nil || backendID == "" {
return nil
}
return m.pools[backendID]
}
func (p *pool) acquire(ctx context.Context) error {
p.mu.Lock()
if err := ctx.Err(); err != nil {
p.mu.Unlock()
return err
}
if p.active < p.concurrencyLimit && p.waiters.Len() == 0 {
p.active++
p.mu.Unlock()
return nil
}
waiter := &waiter{ready: make(chan struct{})}
waiter.element = p.waiters.PushBack(waiter)
p.mu.Unlock()
select {
case <-waiter.ready:
return nil
case <-ctx.Done():
p.mu.Lock()
if !waiter.granted {
p.waiters.Remove(waiter.element)
waiter.element = nil
p.mu.Unlock()
return ctx.Err()
}
p.mu.Unlock()
return nil
}
}
func (p *pool) releaseActive() {
var ready chan struct{}
p.mu.Lock()
if element := p.waiters.Front(); element != nil {
waiter := element.Value.(*waiter)
p.waiters.Remove(element)
waiter.element = nil
waiter.granted = true
ready = waiter.ready
} else {
p.active--
}
p.mu.Unlock()
if ready != nil {
close(ready)
}
}

View File

@@ -0,0 +1,163 @@
package capacity
import (
"context"
"errors"
"testing"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
)
func TestNewManagerRejectsInvalidPolicies(t *testing.T) {
maxInt := int(^uint(0) >> 1)
tests := []struct {
name string
id string
policy domain.BackendCapacityPolicy
}{
{
name: "blank ID",
id: " \t ",
policy: domain.BackendCapacityPolicy{ConcurrencyLimit: 1},
},
{
name: "zero concurrency",
id: "backend",
policy: domain.BackendCapacityPolicy{},
},
{
name: "negative concurrency",
id: "backend",
policy: domain.BackendCapacityPolicy{ConcurrencyLimit: -1},
},
{
name: "negative queue",
id: "backend",
policy: domain.BackendCapacityPolicy{
ConcurrencyLimit: 1,
QueueCapacity: -1,
},
},
{
name: "total overflow",
id: "backend",
policy: domain.BackendCapacityPolicy{
ConcurrencyLimit: maxInt,
QueueCapacity: 1,
},
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
_, err := NewManager(map[string]domain.BackendCapacityPolicy{
tc.id: tc.policy,
})
if err == nil {
t.Fatal("expected invalid policy error")
}
})
}
}
func TestManagerAdmissionIsBoundedAndReleaseIsIdempotent(t *testing.T) {
policies := map[string]domain.BackendCapacityPolicy{
"limited": {
ConcurrencyLimit: 2,
QueueCapacity: 1,
},
"independent": {
ConcurrencyLimit: 1,
},
}
manager, err := NewManager(policies)
if err != nil {
t.Fatalf("construct manager: %v", err)
}
policies["limited"] = domain.BackendCapacityPolicy{
ConcurrencyLimit: 100,
QueueCapacity: 100,
}
releases := make([]func(), 0, 3)
for range 3 {
release, err := manager.Admit(context.Background(), "limited")
if err != nil {
t.Fatalf("admit within configured capacity: %v", err)
}
releases = append(releases, release)
}
if release, err := manager.Admit(context.Background(), "limited"); release != nil ||
!errors.Is(err, ErrCapacityExceeded) {
t.Fatalf("admission beyond capacity=(release=%t, err=%v), want ErrCapacityExceeded",
release != nil, err)
}
independentRelease, err := manager.Admit(context.Background(), "independent")
if err != nil {
t.Fatalf("admit independent backend while first is full: %v", err)
}
independentRelease()
releases[0]()
releases[0]()
replacement, err := manager.Admit(context.Background(), "limited")
if err != nil {
t.Fatalf("admit after release: %v", err)
}
replacement()
releases[1]()
releases[2]()
pool := manager.pools["limited"]
pool.mu.Lock()
admitted := pool.admitted
pool.mu.Unlock()
if admitted != 0 {
t.Fatalf("admitted runs after releases=%d, want 0", admitted)
}
}
func TestManagerAdmissionHonorsContextAndUnlimitedBackends(t *testing.T) {
manager, err := NewManager(map[string]domain.BackendCapacityPolicy{
"limited": {ConcurrencyLimit: 1},
})
if err != nil {
t.Fatalf("construct manager: %v", err)
}
release, err := manager.Admit(context.Background(), "limited")
if err != nil {
t.Fatalf("fill limited pool: %v", err)
}
defer release()
ctx, cancel := context.WithCancel(context.Background())
cancel()
if release, err := manager.Admit(ctx, "limited"); release != nil ||
!errors.Is(err, context.Canceled) {
t.Fatalf("canceled limited admission=(release=%t, err=%v), want context cancellation",
release != nil, err)
}
var nilManager *Manager
for _, tc := range []struct {
name string
manager *Manager
backendID string
}{
{name: "nil manager", manager: nilManager, backendID: "limited"},
{name: "blank ID", manager: manager},
{name: "unknown ID", manager: manager, backendID: "unknown"},
} {
t.Run(tc.name, func(t *testing.T) {
release, err := tc.manager.Admit(ctx, tc.backendID)
if err != nil {
t.Fatalf("unlimited admission: %v", err)
}
if release == nil {
t.Fatal("unlimited admission returned nil release")
}
release()
release()
})
}
}

View File

@@ -14,9 +14,6 @@ const (
ContentTypeApplicationJSON = "application/json"
OpenAIChatCompletionsPath = "/chat/completions"
ExecutionDefaultTemperature = 0.0
ExecutionDefaultMaxTokens = 0
ExecutionDefaultTopP = 1.0
ExecutionDefaultTimeoutSeconds = 600
)
@@ -26,9 +23,6 @@ var (
func ExecutionTargetDefault() domain.ExecutionTarget {
return domain.ExecutionTarget{
Temperature: ExecutionDefaultTemperature,
MaxTokens: ExecutionDefaultMaxTokens,
TopP: ExecutionDefaultTopP,
TimeoutSeconds: ExecutionDefaultTimeoutSeconds,
}
}

View File

@@ -63,12 +63,12 @@ type RunRequest struct {
PromptID string
PromptVersion string
ProfileID string
SessionID string
APIKey string `json:"-" yaml:"-"`
Inputs map[string]ArtifactRef
Vars map[string]string
Execution *ExecutionTargetOverride
Validation *OutputContract
Metadata map[string]string
}
// RunResult represents the complete result of a prompt execution run.
@@ -80,8 +80,10 @@ type RunResult struct {
PromptID string
PromptVersion string
PromptHash string
SessionID string
RenderedPromptHash string
SelectedProfileID string
SelectedBackendID string
ModelName string
Endpoint string
EffectiveModelParams ExecutionTarget
@@ -99,6 +101,7 @@ type PreparedRun struct {
PromptVersion string `json:"prompt_version,omitempty"`
PromptHash string `json:"prompt_hash,omitempty"`
SelectedProfileID string `json:"selected_profile_id"`
SelectedBackendID string `json:"selected_backend_id,omitempty"`
EffectiveModelParams ExecutionTarget `json:"effective_model_params"`
TargetPresence ExecutionTargetPresence `json:"-"`
OutputContract OutputContract `json:"output_contract"`
@@ -142,6 +145,16 @@ type PromptDefinition struct {
Validation OutputContract `yaml:"validation"`
}
// PromptInspection is the resolved result of exact prompt inspection.
type PromptInspection struct {
PromptID string
PromptVersion string
PromptHash string
DefaultProfileID string
Inputs []PromptInput
OutputContract OutputContract
}
// PromptInput describes one named input expected by a prompt definition.
type PromptInput struct {
Name string `yaml:"name"`
@@ -158,9 +171,28 @@ type PromptMessageTemplate struct {
CacheControl *CacheControl `yaml:"cache_control,omitempty" json:"cache_control,omitempty"`
}
// Backend describes reusable OpenAI-compatible connection defaults.
type Backend struct {
ID string
Endpoint string
APIKeyEnv string
ExtraParams map[string]any
ConcurrencyLimit int
QueueCapacity int
QueueCapacitySet bool
}
// BackendCapacityPolicy describes normalized run and generation capacity for
// one limited backend.
type BackendCapacityPolicy struct {
ConcurrencyLimit int
QueueCapacity int
}
// ExecutionProfile describes how and where to execute a model.
type ExecutionProfile struct {
ID string `yaml:"id"`
BackendID string `yaml:"backend"`
Endpoint string `yaml:"endpoint"`
Model string `yaml:"model"`
Temperature float64 `yaml:"temperature"`
@@ -183,7 +215,7 @@ type ExecutionTargetOverride struct {
TopP *float64 `json:"top_p,omitempty"`
TimeoutSeconds *int `json:"timeout_seconds,omitempty"`
ServiceTier string `json:"service_tier,omitempty"`
ReasoningEffort string `json:"reasoning_effort,omitempty"`
ReasoningEffort *string `json:"reasoning_effort,omitempty"`
APIKeyEnv string `json:"api_key_env,omitempty"`
ExtraParams map[string]any `json:"extra_params,omitempty"`
}
@@ -199,6 +231,7 @@ type ExecutionTargetPresence struct {
// ExecutionTarget represents effective model runtime settings for a run.
type ExecutionTarget struct {
BackendID string `yaml:"backend" json:"backend_id,omitempty"`
Endpoint string `yaml:"endpoint" json:"endpoint"`
Model string `yaml:"model" json:"model"`
Temperature float64 `yaml:"temperature" json:"temperature"`
@@ -213,6 +246,13 @@ type ExecutionTarget struct {
ExtraParams map[string]any `yaml:"extra_params" json:"extra_params"`
}
// ProfileInspection is the resolved result of exact profile inspection.
type ProfileInspection struct {
ProfileID string
EffectiveModelParams ExecutionTarget
APIKeyRequired bool
}
// OutputContract defines the requirements for the output artifact.
type OutputContract struct {
Format OutputFormat `yaml:"format"`

View File

@@ -0,0 +1,19 @@
package domain
import (
"fmt"
"strings"
"unicode/utf8"
)
// NormalizeSessionID applies the shared session identifier rule.
func NormalizeSessionID(raw string) (string, error) {
normalized := strings.TrimSpace(raw)
if normalized == "" {
return "", nil
}
if length := utf8.RuneCountInString(normalized); length > SessionIDMaxLength {
return "", fmt.Errorf("session_id length %d exceeds maximum %d", length, SessionIDMaxLength)
}
return normalized, nil
}

View File

@@ -0,0 +1,57 @@
package domain
import (
"strings"
"testing"
)
func TestNormalizeSessionID(t *testing.T) {
tests := []struct {
name string
raw string
want string
wantErr bool
}{
{
name: "trims surrounding Unicode whitespace",
raw: "\u2003 session-123 \u2003",
want: "session-123",
},
{
name: "blank input is omitted",
raw: " \t\u2003 ",
want: "",
},
{
name: "maximum Unicode length is accepted",
raw: strings.Repeat("界", SessionIDMaxLength),
want: strings.Repeat("界", SessionIDMaxLength),
},
{
name: "one Unicode code point over maximum is rejected",
raw: strings.Repeat("界", SessionIDMaxLength+1),
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := NormalizeSessionID(tt.raw)
if tt.wantErr {
if err == nil {
t.Fatal("expected normalization error")
}
if !strings.Contains(err.Error(), "exceeds maximum") {
t.Fatalf("expected useful length diagnostic, got %v", err)
}
return
}
if err != nil {
t.Fatalf("normalize session id: %v", err)
}
if got != tt.want {
t.Fatalf("normalized session id = %q, want %q", got, tt.want)
}
})
}
}

View File

@@ -1,25 +1,36 @@
package promptkit
// Package jsonvalue validates and defensively copies JSON-compatible value
// trees used by configuration, request, and prepared-state boundaries.
package jsonvalue
import (
"encoding/json"
"fmt"
"math"
"reflect"
"sort"
"strconv"
)
const maxSafeJSONInteger = 1<<53 - 1
type jsonVisit struct {
type visit struct {
typ reflect.Type
ptr uintptr
}
func copyPublicJSONMap(src map[string]any) (map[string]any, error) {
// Copy validates and deeply copies a JSON-compatible value while preserving
// compatible concrete map, slice, array, scalar, and number types.
func Copy(src any) (any, error) {
return copyValue(reflect.ValueOf(src), "value", make(map[visit]struct{}), true)
}
// CopyMap validates and deeply copies an extra-parameter map while preserving
// compatible concrete map, slice, array, scalar, and number types.
func CopyMap(src map[string]any) (map[string]any, error) {
if src == nil {
return nil, nil
}
copied, err := copyPublicJSONValue(reflect.ValueOf(src), "extra_params", make(map[jsonVisit]struct{}))
copied, err := copyValue(reflect.ValueOf(src), "extra_params", make(map[visit]struct{}), false)
if err != nil {
return nil, err
}
@@ -30,7 +41,12 @@ func copyPublicJSONMap(src map[string]any) (map[string]any, error) {
return out, nil
}
func copyPublicJSONValue(value reflect.Value, path string, seen map[jsonVisit]struct{}) (any, error) {
func copyValue(
value reflect.Value,
path string,
seen map[visit]struct{},
allowEmptyMapKeys bool,
) (any, error) {
if !value.IsValid() {
return nil, nil
}
@@ -38,12 +54,15 @@ func copyPublicJSONValue(value reflect.Value, path string, seen map[jsonVisit]st
if value.IsNil() {
return nil, nil
}
return copyPublicJSONValue(value.Elem(), path, seen)
return copyValue(value.Elem(), path, seen, allowEmptyMapKeys)
}
if !value.CanInterface() {
return nil, fmt.Errorf("%s: value cannot be copied", path)
}
if number, ok := value.Interface().(json.Number); ok {
if _, err := json.Marshal(number); err != nil {
return nil, fmt.Errorf("%s: invalid JSON number", path)
}
f, err := strconv.ParseFloat(number.String(), 64)
if err != nil || math.IsNaN(f) || math.IsInf(f, 0) {
return nil, fmt.Errorf("%s: invalid JSON number", path)
@@ -65,8 +84,8 @@ func copyPublicJSONValue(value reflect.Value, path string, seen map[jsonVisit]st
}
return value.Interface(), nil
case reflect.Float32, reflect.Float64:
f := value.Convert(reflect.TypeOf(float64(0))).Float()
if math.IsNaN(f) || math.IsInf(f, 0) {
number := value.Convert(reflect.TypeOf(float64(0))).Float()
if math.IsNaN(number) || math.IsInf(number, 0) {
return nil, fmt.Errorf("%s: floating-point value must be finite", path)
}
return value.Interface(), nil
@@ -74,28 +93,33 @@ func copyPublicJSONValue(value reflect.Value, path string, seen map[jsonVisit]st
if value.IsNil() {
return nil, nil
}
visit := jsonVisit{typ: value.Type(), ptr: value.Pointer()}
if _, ok := seen[visit]; ok {
current := visit{typ: value.Type(), ptr: value.Pointer()}
if _, ok := seen[current]; ok {
return nil, fmt.Errorf("%s: cyclic value is not supported", path)
}
seen[visit] = struct{}{}
defer delete(seen, visit)
return copyPublicJSONValue(value.Elem(), path, seen)
seen[current] = struct{}{}
defer delete(seen, current)
return copyValue(value.Elem(), path, seen, allowEmptyMapKeys)
case reflect.Map:
return copyPublicJSONMapValue(value, path, seen)
return copyMapValue(value, path, seen, allowEmptyMapKeys)
case reflect.Slice:
if value.IsNil() {
return nil, nil
}
return copyPublicJSONSequenceValue(value, path, seen)
return copySequenceValue(value, path, seen, allowEmptyMapKeys)
case reflect.Array:
return copyPublicJSONSequenceValue(value, path, seen)
return copySequenceValue(value, path, seen, allowEmptyMapKeys)
default:
return nil, fmt.Errorf("%s: unsupported JSON value type %s", path, value.Type())
}
}
func copyPublicJSONMapValue(value reflect.Value, path string, seen map[jsonVisit]struct{}) (any, error) {
func copyMapValue(
value reflect.Value,
path string,
seen map[visit]struct{},
allowEmptyMapKeys bool,
) (any, error) {
if value.IsNil() {
return nil, nil
}
@@ -103,37 +127,43 @@ func copyPublicJSONMapValue(value reflect.Value, path string, seen map[jsonVisit
return nil, fmt.Errorf("%s: map key type %s is not supported", path, value.Type().Key())
}
visit := jsonVisit{typ: value.Type(), ptr: value.Pointer()}
if _, ok := seen[visit]; ok {
current := visit{typ: value.Type(), ptr: value.Pointer()}
if _, ok := seen[current]; ok {
return nil, fmt.Errorf("%s: cyclic value is not supported", path)
}
seen[visit] = struct{}{}
defer delete(seen, visit)
seen[current] = struct{}{}
defer delete(seen, current)
keys := value.MapKeys()
sort.Slice(keys, func(i, j int) bool {
return keys[i].String() < keys[j].String()
})
type entry struct {
key reflect.Value
name string
value any
}
entries := make([]entry, 0, value.Len())
entries := make([]entry, 0, len(keys))
preserveType := true
elemType := value.Type().Elem()
iter := value.MapRange()
for iter.Next() {
key := iter.Key()
elementType := value.Type().Elem()
for _, key := range keys {
name := key.String()
copied, err := copyPublicJSONValue(iter.Value(), path+"."+name, seen)
if name == "" && !allowEmptyMapKeys {
return nil, fmt.Errorf("%s: map key must not be empty", path)
}
copied, err := copyValue(value.MapIndex(key), path+"."+name, seen, allowEmptyMapKeys)
if err != nil {
return nil, err
}
entries = append(entries, entry{key: key, name: name, value: copied})
if copied == nil {
if !canAssignNil(elemType) {
if !canAssignNil(elementType) {
preserveType = false
}
continue
}
if !reflect.TypeOf(copied).AssignableTo(elemType) {
if !reflect.TypeOf(copied).AssignableTo(elementType) {
preserveType = false
}
}
@@ -142,7 +172,7 @@ func copyPublicJSONMapValue(value reflect.Value, path string, seen map[jsonVisit
out := reflect.MakeMapWithSize(value.Type(), len(entries))
for _, entry := range entries {
if entry.value == nil {
out.SetMapIndex(entry.key, reflect.Zero(elemType))
out.SetMapIndex(entry.key, reflect.Zero(elementType))
continue
}
out.SetMapIndex(entry.key, reflect.ValueOf(entry.value))
@@ -157,33 +187,43 @@ func copyPublicJSONMapValue(value reflect.Value, path string, seen map[jsonVisit
return out, nil
}
func copyPublicJSONSequenceValue(value reflect.Value, path string, seen map[jsonVisit]struct{}) (any, error) {
var visit jsonVisit
func copySequenceValue(
value reflect.Value,
path string,
seen map[visit]struct{},
allowEmptyMapKeys bool,
) (any, error) {
var current visit
if value.Kind() == reflect.Slice {
visit = jsonVisit{typ: value.Type(), ptr: value.Pointer()}
if _, ok := seen[visit]; ok {
current = visit{typ: value.Type(), ptr: value.Pointer()}
if _, ok := seen[current]; ok {
return nil, fmt.Errorf("%s: cyclic value is not supported", path)
}
seen[visit] = struct{}{}
defer delete(seen, visit)
seen[current] = struct{}{}
defer delete(seen, current)
}
values := make([]any, value.Len())
preserveType := true
elemType := value.Type().Elem()
elementType := value.Type().Elem()
for i := 0; i < value.Len(); i++ {
copied, err := copyPublicJSONValue(value.Index(i), fmt.Sprintf("%s[%d]", path, i), seen)
copied, err := copyValue(
value.Index(i),
fmt.Sprintf("%s[%d]", path, i),
seen,
allowEmptyMapKeys,
)
if err != nil {
return nil, err
}
values[i] = copied
if copied == nil {
if !canAssignNil(elemType) {
if !canAssignNil(elementType) {
preserveType = false
}
continue
}
if !reflect.TypeOf(copied).AssignableTo(elemType) {
if !reflect.TypeOf(copied).AssignableTo(elementType) {
preserveType = false
}
}
@@ -195,7 +235,7 @@ func copyPublicJSONSequenceValue(value reflect.Value, path string, seen map[json
}
for i, copied := range values {
if copied == nil {
out.Index(i).Set(reflect.Zero(elemType))
out.Index(i).Set(reflect.Zero(elementType))
continue
}
out.Index(i).Set(reflect.ValueOf(copied))

View File

@@ -0,0 +1,112 @@
package jsonvalue_test
import (
"encoding/json"
"math"
"reflect"
"testing"
"gitea.maximumdirect.net/eric/promptkit/internal/jsonvalue"
)
func TestCopyMapPreservesTypesAndIsolatesMutations(t *testing.T) {
nested := map[string]int{"limit": 2}
sequence := []string{"one", "two"}
input := map[string]any{
"count": int64(7),
"number": json.Number("-1.25e+2"),
"nested": nested,
"sequence": sequence,
}
copied, err := jsonvalue.CopyMap(input)
if err != nil {
t.Fatalf("copy map: %v", err)
}
nested["limit"] = 99
sequence[0] = "changed"
input["added"] = true
if got, ok := copied["count"].(int64); !ok || got != 7 {
t.Fatalf("integer type or value changed: %#v", copied["count"])
}
if got, ok := copied["number"].(json.Number); !ok || got != "-1.25e+2" {
t.Fatalf("JSON number type or value changed: %#v", copied["number"])
}
if got := copied["nested"].(map[string]int)["limit"]; got != 2 {
t.Fatalf("nested map was not isolated: %d", got)
}
if got := copied["sequence"].([]string)[0]; got != "one" {
t.Fatalf("sequence was not isolated: %q", got)
}
if _, ok := copied["added"]; ok {
t.Fatalf("top-level map was not isolated: %#v", copied)
}
}
func TestCopyAllowsEmptyObjectKeysAndIsolatesMutations(t *testing.T) {
nested := map[string]any{"": []any{"original"}}
copiedValue, err := jsonvalue.Copy(nested)
if err != nil {
t.Fatalf("copy value: %v", err)
}
nested[""].([]any)[0] = "changed"
copied := copiedValue.(map[string]any)
if got := copied[""].([]any)[0]; got != "original" {
t.Fatalf("copied value was not isolated: %v", got)
}
}
func TestCopyMapRejectsInvalidValues(t *testing.T) {
cyclicMap := map[string]any{}
cyclicMap["self"] = cyclicMap
cyclicSlice := []any{nil}
cyclicSlice[0] = cyclicSlice
tests := []struct {
name string
value any
}{
{name: "empty nested key", value: map[string]int{"": 1}},
{name: "non-string map key", value: map[int]string{1: "one"}},
{name: "unsupported value", value: make(chan int)},
{name: "cyclic map", value: cyclicMap},
{name: "cyclic slice", value: cyclicSlice},
{name: "NaN", value: math.NaN()},
{name: "positive infinity", value: math.Inf(1)},
{name: "unsafe signed integer", value: int64(1 << 53)},
{name: "unsafe unsigned integer", value: uint64(1 << 53)},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
if _, err := jsonvalue.CopyMap(map[string]any{"value": tc.value}); err == nil {
t.Fatal("expected validation error")
}
})
}
}
func TestCopyMapValidatesJSONNumberSyntaxAndRange(t *testing.T) {
for _, number := range []json.Number{"0", "-1", "1.25", "-1.25e+2"} {
t.Run("valid "+number.String(), func(t *testing.T) {
got, err := jsonvalue.CopyMap(map[string]any{"value": number})
if err != nil {
t.Fatalf("copy valid JSON number: %v", err)
}
if !reflect.DeepEqual(got["value"], number) {
t.Fatalf("JSON number changed: got %#v want %#v", got["value"], number)
}
})
}
for _, number := range []json.Number{"", "01", "+1", "1.", ".1", "1e9999", "not-a-number"} {
t.Run("invalid "+number.String(), func(t *testing.T) {
if _, err := jsonvalue.CopyMap(map[string]any{"value": number}); err == nil {
t.Fatal("expected invalid JSON number error")
}
})
}
}

View File

@@ -12,7 +12,6 @@ import (
"os"
"strings"
"time"
"unicode/utf8"
"gitea.maximumdirect.net/eric/promptkit/internal/defaults"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
@@ -177,12 +176,11 @@ func openAIChatRequestFromGenerateRequest(req domain.GenerateRequest, defaultMod
wireReq := openAIChatRequest{
Model: model,
}
if sessionID := strings.TrimSpace(req.Prompt.SessionID); sessionID != "" {
if n := utf8.RuneCountInString(sessionID); n > domain.SessionIDMaxLength {
return openAIChatRequest{}, fmt.Errorf("session_id length %d exceeds maximum %d", n, domain.SessionIDMaxLength)
}
wireReq.SessionID = sessionID
sessionID, err := domain.NormalizeSessionID(req.Prompt.SessionID)
if err != nil {
return openAIChatRequest{}, err
}
wireReq.SessionID = sessionID
wireReq.Messages = make([]openAIChatRequestMessage, 0, len(req.Prompt.Messages))
for _, msg := range req.Prompt.Messages {
@@ -262,7 +260,7 @@ func openAIChatRequestPayload(req openAIChatRequest) (map[string]any, error) {
if key == "" {
return nil, errors.New("extra_params key must not be empty")
}
if _, reserved := reservedOpenAIChatRequestFields[key]; reserved {
if IsReservedOpenAIChatRequestField(key) {
return nil, fmt.Errorf("extra_params key %q collides with reserved request field", key)
}
if _, err := json.Marshal(value); err != nil {
@@ -274,16 +272,23 @@ func openAIChatRequestPayload(req openAIChatRequest) (map[string]any, error) {
return out, nil
}
var reservedOpenAIChatRequestFields = map[string]struct{}{
"model": {},
"session_id": {},
"messages": {},
"temperature": {},
"max_tokens": {},
"top_p": {},
"service_tier": {},
"reasoning_effort": {},
"response_format": {},
// IsReservedOpenAIChatRequestField reports whether name is owned by the
// standard OpenAI-compatible chat request rather than extra parameters.
func IsReservedOpenAIChatRequestField(name string) bool {
switch name {
case "model",
"session_id",
"messages",
"temperature",
"max_tokens",
"top_p",
"service_tier",
"reasoning_effort",
"response_format":
return true
default:
return false
}
}
type openAIChatRequestMessage struct {

View File

@@ -1,9 +1,8 @@
id: aion-2
endpoint: https://openrouter.ai/api/v1
backend: openrouter
model: aion-labs/aion-2.0
temperature: 0.72
reasoning_effort: high
top_p: 0.95
timeout_seconds: 180
api_key_env: OPENROUTER_API_KEY
service_tier: flex

View File

@@ -1,7 +1,6 @@
id: claude-fable-latest
endpoint: https://openrouter.ai/api/v1
backend: openrouter
model: "~anthropic/claude-fable-latest"
reasoning_effort: high
timeout_seconds: 600
api_key_env: OPENROUTER_API_KEY
service_tier: flex

View File

@@ -1,7 +1,6 @@
id: claude-haiku-latest
endpoint: https://openrouter.ai/api/v1
backend: openrouter
model: "~anthropic/claude-haiku-latest"
reasoning_effort: medium
timeout_seconds: 240
api_key_env: OPENROUTER_API_KEY
service_tier: flex

View File

@@ -1,7 +1,6 @@
id: claude-opus-latest
endpoint: https://openrouter.ai/api/v1
backend: openrouter
model: "~anthropic/claude-opus-latest"
reasoning_effort: high
timeout_seconds: 240
api_key_env: OPENROUTER_API_KEY
service_tier: flex

View File

@@ -1,7 +1,6 @@
id: claude-sonnet-latest
endpoint: https://openrouter.ai/api/v1
backend: openrouter
model: "~anthropic/claude-sonnet-latest"
reasoning_effort: high
timeout_seconds: 240
api_key_env: OPENROUTER_API_KEY
service_tier: flex

View File

@@ -1,7 +1,6 @@
id: deepseek-3-2
endpoint: https://openrouter.ai/api/v1
backend: openrouter
model: deepseek/deepseek-v3.2
reasoning_effort: high
timeout_seconds: 180
api_key_env: OPENROUTER_API_KEY
service_tier: flex

View File

@@ -1,7 +1,6 @@
id: deepseek-4-flash
endpoint: https://openrouter.ai/api/v1
backend: openrouter
model: deepseek/deepseek-v4-flash
#reasoning_effort: medium
timeout_seconds: 180
api_key_env: OPENROUTER_API_KEY
service_tier: flex

View File

@@ -1,7 +1,6 @@
id: deepseek-4-pro
endpoint: https://openrouter.ai/api/v1
backend: openrouter
model: deepseek/deepseek-v4-pro
reasoning_effort: high
timeout_seconds: 180
api_key_env: OPENROUTER_API_KEY
service_tier: flex

View File

@@ -1,9 +1,8 @@
id: gemini-2-flash-lite
endpoint: https://openrouter.ai/api/v1
backend: openrouter
model: "google/gemini-2.5-flash-lite"
#temperature: 0.15
reasoning_effort: high
#top_p: 0.98
timeout_seconds: 240
api_key_env: OPENROUTER_API_KEY
service_tier: flex

View File

@@ -1,9 +1,8 @@
id: gemini-2-flash
endpoint: https://openrouter.ai/api/v1
backend: openrouter
model: "google/gemini-2.5-flash"
#temperature: 0.15
reasoning_effort: high
#top_p: 0.98
timeout_seconds: 240
api_key_env: OPENROUTER_API_KEY
service_tier: flex

View File

@@ -1,9 +1,8 @@
id: gemini-2-pro
endpoint: https://openrouter.ai/api/v1
backend: openrouter
model: "google/gemini-2.5-pro"
#temperature: 0.15
reasoning_effort: high
#top_p: 0.98
timeout_seconds: 240
api_key_env: OPENROUTER_API_KEY
service_tier: flex

View File

@@ -1,9 +1,8 @@
id: gemini-3-flash-lite
endpoint: https://openrouter.ai/api/v1
backend: openrouter
model: "google/gemini-3.1-flash-lite"
#temperature: 0.15
reasoning_effort: high
#top_p: 0.98
timeout_seconds: 240
api_key_env: OPENROUTER_API_KEY
service_tier: flex

View File

@@ -1,9 +1,8 @@
id: gemini-flash-latest
endpoint: https://openrouter.ai/api/v1
backend: openrouter
model: "~google/gemini-flash-latest"
#temperature: 0.15
reasoning_effort: high
#top_p: 0.98
timeout_seconds: 240
api_key_env: OPENROUTER_API_KEY
service_tier: flex

View File

@@ -1,9 +1,8 @@
id: gemini-pro-latest
endpoint: https://openrouter.ai/api/v1
backend: openrouter
model: "~google/gemini-pro-latest"
#temperature: 0.15
reasoning_effort: high
#top_p: 0.98
timeout_seconds: 240
api_key_env: OPENROUTER_API_KEY
service_tier: flex

View File

@@ -1,9 +1,8 @@
id: gemma-4-31b
endpoint: https://openrouter.ai/api/v1
backend: openrouter
model: google/gemma-4-31b-it:exacto
temperature: 0.15
reasoning_effort: high
top_p: 0.98
timeout_seconds: 240
api_key_env: OPENROUTER_API_KEY
service_tier: flex

View File

@@ -1,9 +1,8 @@
id: minimax-m2
endpoint: https://openrouter.ai/api/v1
backend: openrouter
model: minimax/minimax-m2.5
temperature: 0.5
reasoning_effort: high
top_p: 0.95
timeout_seconds: 180
api_key_env: OPENROUTER_API_KEY
service_tier: flex

View File

@@ -1,9 +1,8 @@
id: minimax-m3
endpoint: https://openrouter.ai/api/v1
backend: openrouter
model: minimax/minimax-m3
#temperature: 0.5
reasoning_effort: high
#top_p: 0.95
timeout_seconds: 180
api_key_env: OPENROUTER_API_KEY
service_tier: flex

View File

@@ -1,7 +1,6 @@
id: mistral-large-2512
endpoint: https://openrouter.ai/api/v1
backend: openrouter
model: mistralai/mistral-large-2512
temperature: 0.15
top_p: 0.98
timeout_seconds: 180
api_key_env: OPENROUTER_API_KEY

View File

@@ -1,8 +1,7 @@
id: mistral-medium-3-5
endpoint: https://openrouter.ai/api/v1
backend: openrouter
model: mistralai/mistral-medium-3-5
temperature: 0.15
reasoning_effort: high
top_p: 0.98
timeout_seconds: 180
api_key_env: OPENROUTER_API_KEY

View File

@@ -1,7 +1,6 @@
id: mistral-small-3
endpoint: https://openrouter.ai/api/v1
backend: openrouter
model: mistralai/mistral-small-3.2-24b-instruct
temperature: 0.05
top_p: 1.0
timeout_seconds: 180
api_key_env: OPENROUTER_API_KEY

View File

@@ -1,8 +1,7 @@
id: mistral-small-4
endpoint: https://openrouter.ai/api/v1
backend: openrouter
model: mistralai/mistral-small-2603
temperature: 0.1
reasoning_effort: high
top_p: 0.98
timeout_seconds: 180
api_key_env: OPENROUTER_API_KEY

View File

@@ -1,7 +1,6 @@
id: nemotron-3-ultra
endpoint: https://openrouter.ai/api/v1
backend: openrouter
model: nvidia/nemotron-3-ultra-550b-a55b
reasoning_effort: high
timeout_seconds: 180
api_key_env: OPENROUTER_API_KEY
service_tier: flex

View File

@@ -1,7 +1,6 @@
id: gpt-5-mini
endpoint: https://openrouter.ai/api/v1
backend: openrouter
model: "openai/gpt-5.4-mini"
reasoning_effort: high
timeout_seconds: 240
api_key_env: OPENROUTER_API_KEY
service_tier: flex

View File

@@ -1,7 +1,6 @@
id: gpt-5-nano
endpoint: https://openrouter.ai/api/v1
backend: openrouter
model: "openai/gpt-5.4-nano"
reasoning_effort: high
timeout_seconds: 240
api_key_env: OPENROUTER_API_KEY
service_tier: flex

View File

@@ -2,7 +2,6 @@ package builtin
import (
"embed"
"strings"
"gitea.maximumdirect.net/eric/promptkit/internal/profile"
)
@@ -15,17 +14,3 @@ var assets embed.FS
func NewRepository() profile.Repository {
return profile.NewFSRepository(assets, assetRoot)
}
func NewRepositoryWithPrimary(primary profile.Repository) profile.Repository {
if primary == nil {
return NewRepository()
}
return profile.NewOverlayRepository(primary, NewRepository())
}
func NewRepositoryWithDirectory(dir string) profile.Repository {
if strings.TrimSpace(dir) == "" {
return NewRepository()
}
return NewRepositoryWithPrimary(profile.NewFilesystemRepository(dir))
}

View File

@@ -2,13 +2,11 @@ package builtin
import (
"context"
"errors"
"io/fs"
"strings"
"testing"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
"gitea.maximumdirect.net/eric/promptkit/internal/profile"
"gitea.maximumdirect.net/eric/promptkit/internal/backend"
"gopkg.in/yaml.v3"
)
@@ -28,6 +26,12 @@ func TestBuiltInProfilesValidateThroughRepository(t *testing.T) {
if p.ID != id {
t.Fatalf("expected profile id %q, got %q", id, p.ID)
}
if p.BackendID != backend.OpenRouterID {
t.Fatalf("expected profile %q to select %q, got %q", id, backend.OpenRouterID, p.BackendID)
}
if p.Endpoint != "" || p.APIKeyEnv != "" {
t.Fatalf("expected profile %q to inherit backend connection settings, got endpoint=%q api_key_env=%q", id, p.Endpoint, p.APIKeyEnv)
}
})
}
}
@@ -60,6 +64,15 @@ func loadBuiltInProfileIDs(t *testing.T) map[string]string {
if _, ok := raw["api_key"]; ok {
t.Fatalf("built-in profile %s contains raw api_key", name)
}
if raw["backend"] != backend.OpenRouterID {
t.Fatalf("built-in profile %s does not select %q", name, backend.OpenRouterID)
}
if _, ok := raw["endpoint"]; ok {
t.Fatalf("built-in profile %s repeats endpoint", name)
}
if _, ok := raw["api_key_env"]; ok {
t.Fatalf("built-in profile %s repeats api_key_env", name)
}
id, ok := raw["id"].(string)
if !ok || strings.TrimSpace(id) == "" {
t.Fatalf("built-in profile %s has missing id", name)
@@ -75,53 +88,3 @@ func loadBuiltInProfileIDs(t *testing.T) map[string]string {
}
return ids
}
func TestRepositoryWithPrimaryUsesPrimaryBeforeBuiltIns(t *testing.T) {
repo := NewRepositoryWithPrimary(staticProfileRepo{
profiles: map[string]string{"mistral-small-3": "custom-model"},
})
p, err := repo.GetProfile(context.Background(), "mistral-small-3")
if err != nil {
t.Fatalf("expected profile to load, got %v", err)
}
if p.Model != "custom-model" {
t.Fatalf("expected primary profile to override built-in, got %+v", p)
}
}
func TestRepositoryWithPrimaryFallsBackToBuiltIns(t *testing.T) {
repo := NewRepositoryWithPrimary(staticProfileRepo{})
p, err := repo.GetProfile(context.Background(), "mistral-small-3")
if err != nil {
t.Fatalf("expected built-in profile to load, got %v", err)
}
if p.ID != "mistral-small-3" {
t.Fatalf("unexpected profile: %+v", p)
}
}
func TestRepositoryWithPrimaryDoesNotFallBackAfterPrimaryError(t *testing.T) {
repo := NewRepositoryWithPrimary(staticProfileRepo{err: profile.ErrInvalidProfile})
_, err := repo.GetProfile(context.Background(), "mistral-small-3")
if !errors.Is(err, profile.ErrInvalidProfile) {
t.Fatalf("expected primary error, got %v", err)
}
}
type staticProfileRepo struct {
profiles map[string]string
err error
}
func (r staticProfileRepo) GetProfile(_ context.Context, id string) (*domain.ExecutionProfile, error) {
if r.err != nil {
return nil, r.err
}
if model, ok := r.profiles[id]; ok {
return &domain.ExecutionProfile{ID: id, Endpoint: "http://primary/v1", Model: model}, nil
}
return nil, profile.ErrProfileNotFound
}

View File

@@ -121,6 +121,7 @@ func loadProfile(ctx context.Context, fsys fs.FS, root string, id string) (*doma
if prof.ID != id {
continue
}
prof.BackendID = strings.TrimSpace(prof.BackendID)
if err := validateProfile(&prof); err != nil {
if errors.Is(err, ErrRawAPIKeyNotAllowed) {
return nil, fmt.Errorf("%w: %s", err, relPath)
@@ -189,8 +190,8 @@ func validateProfile(p *domain.ExecutionProfile) error {
if strings.TrimSpace(p.ID) == "" {
return errors.New("id is required")
}
if strings.TrimSpace(p.Endpoint) == "" {
return errors.New("endpoint is required")
if strings.TrimSpace(p.BackendID) == "" && strings.TrimSpace(p.Endpoint) == "" {
return errors.New("backend or endpoint is required")
}
if strings.TrimSpace(p.Model) == "" {
return errors.New("model is required")

View File

@@ -52,6 +52,43 @@ func TestFilesystemRepository_GetProfile(t *testing.T) {
}
})
t.Run("backend and endpoint connection matrix", func(t *testing.T) {
tests := []struct {
name string
connection string
wantBackend string
wantEndpoint string
wantErr bool
}{
{name: "backend only", connection: "backend: ' openrouter '", wantBackend: "openrouter"},
{name: "endpoint only", connection: "endpoint: http://localhost:8000/v1", wantEndpoint: "http://localhost:8000/v1"},
{name: "both", connection: "backend: openrouter\nendpoint: http://localhost:8000/v1", wantBackend: "openrouter", wantEndpoint: "http://localhost:8000/v1"},
{name: "neither", wantErr: true},
{name: "blank backend", connection: "backend: ' '", wantErr: true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
id := "connection-" + strings.ReplaceAll(tt.name, " ", "-")
writeProfileTestFile(t, filepath.Join(tmpDir, id+".yaml"), "id: "+id+"\nmodel: model\n"+tt.connection+"\n")
p, err := repo.GetProfile(ctx, id)
if tt.wantErr {
if !errors.Is(err, ErrInvalidProfile) {
t.Fatalf("expected ErrInvalidProfile, got %v", err)
}
return
}
if err != nil {
t.Fatalf("expected profile to load, got %v", err)
}
if p.BackendID != tt.wantBackend || p.Endpoint != tt.wantEndpoint {
t.Fatalf("unexpected connection values: backend=%q endpoint=%q", p.BackendID, p.Endpoint)
}
})
}
})
t.Run("valid profile with api_key_env", func(t *testing.T) {
p, err := repo.GetProfile(ctx, "local-secure")
if err != nil {

View File

@@ -5,10 +5,9 @@ import (
"context"
"errors"
"fmt"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
"strings"
"text/template"
"unicode/utf8"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
)
var (
@@ -95,10 +94,6 @@ func (r *goRenderer) Render(ctx context.Context, definition *domain.PromptDefini
}
func renderSessionID(raw string, funcs template.FuncMap, vars map[string]string) (string, error) {
if strings.TrimSpace(raw) == "" {
return "", nil
}
tmpl, err := template.New("session_id").Funcs(funcs).Option("missingkey=error").Parse(raw)
if err != nil {
return "", fmt.Errorf("%w: session_id: %v", ErrInvalidTemplate, err)
@@ -109,9 +104,9 @@ func renderSessionID(raw string, funcs template.FuncMap, vars map[string]string)
return "", fmt.Errorf("%w: session_id: %w", ErrRenderFailure, err)
}
sessionID := strings.TrimSpace(buf.String())
if n := utf8.RuneCountInString(sessionID); n > domain.SessionIDMaxLength {
return "", fmt.Errorf("%w: session_id length %d exceeds maximum %d", ErrRenderFailure, n, domain.SessionIDMaxLength)
sessionID, err := domain.NormalizeSessionID(buf.String())
if err != nil {
return "", fmt.Errorf("%w: session_id: %v", ErrRenderFailure, err)
}
return sessionID, nil
}

View File

@@ -0,0 +1,24 @@
package usecase
import (
"fmt"
"strings"
"gitea.maximumdirect.net/eric/promptkit/internal/capacity"
)
// CapacityError identifies bounded admission rejected for one selected backend.
type CapacityError struct {
BackendID string
}
func (e *CapacityError) Error() string {
if e == nil || strings.TrimSpace(e.BackendID) == "" {
return capacity.ErrCapacityExceeded.Error()
}
return fmt.Sprintf("backend %q admission: %v", e.BackendID, capacity.ErrCapacityExceeded)
}
func (e *CapacityError) Unwrap() error {
return capacity.ErrCapacityExceeded
}

View File

@@ -0,0 +1,298 @@
package usecase
import (
"context"
"fmt"
"sync"
"time"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
"gitea.maximumdirect.net/eric/promptkit/internal/jsonvalue"
"gitea.maximumdirect.net/eric/promptkit/internal/validate"
)
type preparedExecutionState uint8
const (
preparedExecutionReady preparedExecutionState = iota
preparedExecutionClaimed
preparedExecutionDiscarded
)
// PreparedExecution owns one frozen, single-use runner execution.
type PreparedExecution struct {
owner *Runner
mu sync.Mutex
state preparedExecutionState
details *domain.PreparedRun
payload *preparedExecutionPayload
}
type preparedExecutionPayload struct {
prepared *domain.PreparedRun
validation validate.PreparedValidation
directKey string
}
// PrepareExecution completes preparation without generation or admission and
// returns a runner-bound, single-use execution.
func (r *Runner) PrepareExecution(ctx context.Context, req domain.RunRequest) (*PreparedExecution, error) {
state, err := r.resolvePreparation(ctx, req, time.Now().UTC())
if err != nil {
return nil, err
}
validationPlan, err := r.prepareValidation(ctx, state.effectiveContract)
if err != nil {
return nil, err
}
structuredOutput, err := r.structuredOutputFromValidationPlan(
state.definition,
state.effectiveContract,
validationPlan,
)
if err != nil {
return nil, err
}
prepared, err := r.completePreparationWithStructuredOutput(ctx, req, state, structuredOutput)
if err != nil {
return nil, err
}
executionSnapshot, err := clonePreparedRun(prepared)
if err != nil {
return nil, fmt.Errorf("%w: failed to copy prepared execution: %v", ErrInvalidRequest, err)
}
details, err := clonePreparedRun(executionSnapshot)
if err != nil {
return nil, fmt.Errorf("%w: failed to copy prepared execution details: %v", ErrInvalidRequest, err)
}
return &PreparedExecution{
owner: r,
state: preparedExecutionReady,
details: details,
payload: &preparedExecutionPayload{
prepared: executionSnapshot,
validation: validationPlan,
directKey: state.effectiveModel.APIKey,
},
}, nil
}
func (r *Runner) prepareValidation(
ctx context.Context,
contract domain.OutputContract,
) (validate.PreparedValidation, error) {
if r.validator == nil {
return noOpPreparedValidation{contract: contract}, nil
}
preparer, ok := r.validator.(validate.ValidationPreparer)
if !ok {
return nil, fmt.Errorf("%w: validator does not support prepared validation", ErrValidation)
}
plan, err := preparer.PrepareValidation(ctx, contract)
if err != nil {
return nil, fmt.Errorf("%w: %w", ErrValidation, err)
}
if plan == nil {
return nil, fmt.Errorf("%w: validator returned nil prepared validation", ErrValidation)
}
return plan, nil
}
func (r *Runner) structuredOutputFromValidationPlan(
def *domain.PromptDefinition,
contract domain.OutputContract,
plan validate.PreparedValidation,
) (*domain.StructuredOutputSpec, error) {
if contract.ValidationMode != domain.ValidationJSONSchema {
return nil, nil
}
schemaDocument := plan.SchemaDocument()
if schemaDocument == nil {
if r.validator == nil {
return nil, nil
}
return nil, fmt.Errorf("%w: prepared json_schema validation has no schema document", ErrValidation)
}
return structuredOutputSpec(def, schemaDocument), nil
}
// Details returns a fresh credential-redacted copy of the prepared run.
func (p *PreparedExecution) Details() *domain.PreparedRun {
if p == nil {
return nil
}
p.mu.Lock()
detailsSnapshot := p.details
p.mu.Unlock()
details, err := clonePreparedRun(detailsSnapshot)
if err != nil {
return nil
}
return details
}
// Discard invalidates an unclaimed execution and drops its private payload.
func (p *PreparedExecution) Discard() {
if p == nil {
return
}
p.mu.Lock()
if p.state != preparedExecutionReady {
p.mu.Unlock()
return
}
p.state = preparedExecutionDiscarded
payload := p.payload
p.payload = nil
p.mu.Unlock()
payload.clear()
}
// RunPrepared claims and executes one prepared execution owned by this runner.
func (r *Runner) RunPrepared(ctx context.Context, prepared *PreparedExecution) (*domain.RunResult, error) {
payload, err := prepared.claim(r)
if err != nil {
return nil, err
}
defer payload.clear()
runID, err := newRunID()
if err != nil {
return nil, fmt.Errorf("failed to create run id: %w", err)
}
start := time.Now().UTC()
target := payload.prepared.EffectiveModelParams
if err := validateAPIKey(target.APIKeyEnv, payload.directKey, target.APIKeyRequired); err != nil {
return nil, fmt.Errorf("%w: %w", ErrInvalidRequest, err)
}
release, err := r.admitRun(ctx, target.BackendID)
if err != nil {
return nil, err
}
defer release()
return r.executePreparedRun(ctx, payload.prepared, payload.directKey, runID, start, func(
ctx context.Context,
artifact *domain.Artifact,
attemptsUsed int,
) (domain.ValidationResult, error) {
result, validationErr := payload.validation.Validate(ctx, artifact)
if validationErr != nil {
return domain.ValidationResult{}, validationErr
}
result.RepairAttempts = attemptsUsed
return result, nil
})
}
func (p *PreparedExecution) claim(owner *Runner) (*preparedExecutionPayload, error) {
if p == nil || owner == nil || p.owner != owner {
return nil, fmt.Errorf("%w: prepared execution does not belong to this runner", ErrInvalidRequest)
}
p.mu.Lock()
defer p.mu.Unlock()
if p.state != preparedExecutionReady || p.payload == nil {
return nil, fmt.Errorf("%w: prepared execution is not ready", ErrInvalidRequest)
}
p.state = preparedExecutionClaimed
payload := p.payload
p.payload = nil
return payload, nil
}
func (p *preparedExecutionPayload) clear() {
if p == nil {
return
}
if p.prepared != nil {
p.prepared.EffectiveModelParams.APIKey = ""
}
p.prepared = nil
p.validation = nil
p.directKey = ""
}
type noOpPreparedValidation struct {
contract domain.OutputContract
}
func (p noOpPreparedValidation) Validate(
ctx context.Context,
_ *domain.Artifact,
) (domain.ValidationResult, error) {
if err := ctx.Err(); err != nil {
return domain.ValidationResult{}, err
}
return domain.ValidationResult{
Status: domain.ValidationSkipped,
Mode: p.contract.ValidationMode,
SchemaPath: p.contract.SchemaPath,
RepairAttempts: p.contract.RepairAttempts,
IsValid: true,
}, nil
}
func (noOpPreparedValidation) SchemaDocument() any {
return nil
}
func clonePreparedRun(source *domain.PreparedRun) (*domain.PreparedRun, error) {
if source == nil {
return nil, nil
}
copied := *source
extraParams, err := jsonvalue.CopyMap(source.EffectiveModelParams.ExtraParams)
if err != nil {
return nil, err
}
copied.EffectiveModelParams.ExtraParams = extraParams
if source.InputHashes != nil {
copied.InputHashes = make(map[string]string, len(source.InputHashes))
for name, hash := range source.InputHashes {
copied.InputHashes[name] = hash
}
}
if source.Messages != nil {
copied.Messages = make([]domain.RenderedMessage, len(source.Messages))
for i, message := range source.Messages {
copied.Messages[i] = message
if message.CacheControl != nil {
cacheControl := *message.CacheControl
copied.Messages[i].CacheControl = &cacheControl
}
}
}
if source.StructuredOutput != nil {
structuredOutput := *source.StructuredOutput
copied.StructuredOutput = &structuredOutput
if source.StructuredOutput.JSONSchema != nil {
jsonSchema := *source.StructuredOutput.JSONSchema
copied.StructuredOutput.JSONSchema = &jsonSchema
schema, copyErr := cloneJSONValue(source.StructuredOutput.JSONSchema.Schema)
if copyErr != nil {
return nil, copyErr
}
copied.StructuredOutput.JSONSchema.Schema = schema
}
}
return &copied, nil
}
func cloneJSONValue(source any) (any, error) {
return jsonvalue.Copy(source)
}

View File

@@ -0,0 +1,510 @@
package usecase
import (
"context"
"errors"
"os"
"reflect"
"testing"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
"gitea.maximumdirect.net/eric/promptkit/internal/validate"
)
type recordingPreparedValidation struct {
contract domain.OutputContract
schemaDocument any
results []domain.ValidationResult
errs []error
artifacts []string
}
func (p *recordingPreparedValidation) Validate(
_ context.Context,
artifact *domain.Artifact,
) (domain.ValidationResult, error) {
p.artifacts = append(p.artifacts, string(artifact.Body))
index := len(p.artifacts) - 1
if index < len(p.errs) && p.errs[index] != nil {
return domain.ValidationResult{}, p.errs[index]
}
if len(p.results) == 0 {
return domain.ValidationResult{
Status: domain.ValidationPassed,
Mode: p.contract.ValidationMode,
IsValid: true,
}, nil
}
if index >= len(p.results) {
index = len(p.results) - 1
}
return p.results[index], nil
}
func (p *recordingPreparedValidation) SchemaDocument() any {
return p.schemaDocument
}
type recordingValidationPreparer struct {
plan *recordingPreparedValidation
prepareErr error
prepareCalls int
directValidateCalls int
}
func (v *recordingValidationPreparer) Validate(
context.Context,
*domain.Artifact,
domain.OutputContract,
) (domain.ValidationResult, error) {
v.directValidateCalls++
return domain.ValidationResult{}, errors.New("live validation must not be used")
}
func (v *recordingValidationPreparer) PrepareValidation(
_ context.Context,
contract domain.OutputContract,
) (validate.PreparedValidation, error) {
v.prepareCalls++
if v.prepareErr != nil {
return nil, v.prepareErr
}
v.plan.contract = contract
return v.plan, nil
}
func TestRunnerPrepareExecutionCompletesWithoutAdmissionOrGeneration(t *testing.T) {
schemaDocument := map[string]any{
"type": "object",
"properties": map[string]any{
"value": map[string]any{"type": "string"},
"": map[string]any{"type": "boolean"},
},
}
def := promptDef(domain.FormatJSON, domain.ValidationJSONSchema, 0)
def.Validation.SchemaPath = "schema.json"
reader := defaultArtifactReader()
renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{
SessionID: "prepared-session",
Messages: []domain.RenderedMessage{{
Role: "user",
Content: "original message",
}},
}}
llmClient := &fakeLLM{forbid: true}
validator := &recordingValidationPreparer{
plan: &recordingPreparedValidation{schemaDocument: schemaDocument},
}
admitter := &fakeRunAdmitter{}
profile := defaultExecutionProfile()
profile.ExtraParams = map[string]any{
"metadata": map[string]any{"source": "original"},
}
runner := NewRunner(
&fakePromptRepo{def: def},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": profile}},
nil,
reader,
renderer,
llmClient,
validator,
admitter,
)
prepared, err := runner.PrepareExecution(context.Background(), domain.RunRequest{
PromptID: "p",
ProfileID: "exec",
APIKey: "direct-test-key",
Inputs: singleInputRef(),
})
if err != nil {
t.Fatalf("prepare execution: %v", err)
}
defer prepared.Discard()
if validator.prepareCalls != 1 || validator.directValidateCalls != 0 {
t.Fatalf(
"validation calls=(prepare=%d direct=%d), want (1, 0)",
validator.prepareCalls,
validator.directValidateCalls,
)
}
if reader.calls != 1 || renderer.calls != 1 {
t.Fatalf("completion calls=(artifact=%d render=%d), want (1, 1)", reader.calls, renderer.calls)
}
if len(admitter.backendIDs) != 0 || llmClient.calls != 0 {
t.Fatalf("prepare invoked execution collaborators: admission=%v generation=%d", admitter.backendIDs, llmClient.calls)
}
first := prepared.Details()
if first == nil {
t.Fatal("prepared details are nil")
}
if first.EffectiveModelParams.APIKey != "" {
t.Fatal("prepared details retained the direct API key")
}
if first.StructuredOutput == nil ||
first.StructuredOutput.JSONSchema == nil ||
!reflect.DeepEqual(first.StructuredOutput.JSONSchema.Schema, schemaDocument) {
t.Fatalf("prepared details have unexpected structured output: %#v", first.StructuredOutput)
}
first.Messages[0].Content = "caller mutation"
first.InputHashes["input"] = "caller mutation"
first.EffectiveModelParams.ExtraParams["metadata"].(map[string]any)["source"] = "caller mutation"
first.StructuredOutput.JSONSchema.Schema.(map[string]any)["type"] = "string"
renderer.rendered.Messages[0].Content = "source mutation"
second := prepared.Details()
if second.Messages[0].Content != "original message" ||
second.InputHashes["input"] == "caller mutation" ||
second.EffectiveModelParams.ExtraParams["metadata"].(map[string]any)["source"] != "original" ||
second.StructuredOutput.JSONSchema.Schema.(map[string]any)["type"] != "object" {
t.Fatalf("details did not preserve an independent snapshot: %#v", second)
}
}
func TestRunnerRunPreparedRechecksEnvironmentCredentialBeforeAdmission(t *testing.T) {
const environmentName = "PROMPTKIT_PREPARED_EXECUTION_TEST_KEY"
t.Setenv(environmentName, "available-during-preparation")
profile := defaultExecutionProfile()
profile.APIKeyEnv = environmentName
validator := &recordingValidationPreparer{plan: &recordingPreparedValidation{}}
admitter := &fakeRunAdmitter{}
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "unexpected"}}
runner := NewRunner(
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": profile}},
nil,
defaultArtifactReader(),
defaultRenderer(),
llmClient,
validator,
admitter,
)
prepared, err := runner.PrepareExecution(context.Background(), domain.RunRequest{
PromptID: "p",
ProfileID: "exec",
Inputs: singleInputRef(),
})
if err != nil {
t.Fatalf("prepare execution: %v", err)
}
if err := os.Unsetenv(environmentName); err != nil {
t.Fatalf("unset credential environment: %v", err)
}
result, err := runner.RunPrepared(context.Background(), prepared)
if result != nil {
t.Fatalf("credential failure returned partial result: %+v", result)
}
if !errors.Is(err, ErrInvalidRequest) || !errors.Is(err, ErrAPIKeyEnvMissing) {
t.Fatalf("credential error identities are missing: %v", err)
}
if len(admitter.backendIDs) != 0 || llmClient.calls != 0 {
t.Fatalf("credential failure reached admission or generation: admission=%v generation=%d", admitter.backendIDs, llmClient.calls)
}
if _, err := runner.RunPrepared(context.Background(), prepared); !errors.Is(err, ErrInvalidRequest) {
t.Fatalf("credential failure did not consume execution: %v", err)
}
}
func TestRunnerRunPreparedKeepsDirectCredentialOutOfMetadata(t *testing.T) {
const directKey = "direct-prepared-test-key"
profile := defaultExecutionProfile()
profile.APIKeyRequired = true
client := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
runner := NewRunner(
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": profile}},
nil,
defaultArtifactReader(),
defaultRenderer(),
client,
&recordingValidationPreparer{plan: &recordingPreparedValidation{}},
nil,
)
prepared, err := runner.PrepareExecution(context.Background(), domain.RunRequest{
PromptID: "p",
ProfileID: "exec",
APIKey: directKey,
Inputs: singleInputRef(),
})
if err != nil {
t.Fatalf("prepare execution: %v", err)
}
if details := prepared.Details(); details.EffectiveModelParams.APIKey != "" {
t.Fatal("prepared details retained direct credential")
}
result, err := runner.RunPrepared(context.Background(), prepared)
if err != nil {
t.Fatalf("run prepared: %v", err)
}
if client.lastReq.Target.APIKey != directKey {
t.Fatal("generation did not receive direct credential")
}
if result.EffectiveModelParams.APIKey != "" {
t.Fatal("run result retained direct credential")
}
}
func TestRunnerRunPreparedUsesFrozenValidationForInitialAndRepairOutputs(t *testing.T) {
validator := &recordingValidationPreparer{
plan: &recordingPreparedValidation{
results: []domain.ValidationResult{
{
Status: domain.ValidationFailed,
Mode: domain.ValidationJSON,
Errors: []string{"invalid"},
IsValid: false,
},
{
Status: domain.ValidationPassed,
Mode: domain.ValidationJSON,
IsValid: true,
},
},
},
}
repairer := &fakeRepairer{
responses: []*domain.GenerateResponse{{Content: `{"repaired":true}`}},
}
admitter := &fakeRunAdmitter{}
reader := defaultArtifactReader()
renderer := defaultRenderer()
runner := NewRunnerWithRepairer(
&fakePromptRepo{def: promptDef(domain.FormatJSON, domain.ValidationJSON, 1)},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
nil,
reader,
renderer,
&fakeLLM{resp: &domain.GenerateResponse{Content: `{"broken":true}`}},
validator,
repairer,
admitter,
)
prepared, err := runner.PrepareExecution(context.Background(), domain.RunRequest{
PromptID: "p",
ProfileID: "exec",
Inputs: singleInputRef(),
})
if err != nil {
t.Fatalf("prepare execution: %v", err)
}
result, err := runner.RunPrepared(context.Background(), prepared)
if err != nil {
t.Fatalf("run prepared: %v", err)
}
if !reflect.DeepEqual(validator.plan.artifacts, []string{`{"broken":true}`, `{"repaired":true}`}) {
t.Fatalf("prepared validation artifacts=%#v", validator.plan.artifacts)
}
if validator.directValidateCalls != 0 || repairer.calls != 1 {
t.Fatalf("validation/repair calls=(direct=%d repair=%d), want (0, 1)", validator.directValidateCalls, repairer.calls)
}
if result.Validation.Status != domain.ValidationPassed || result.Validation.RepairAttempts != 1 {
t.Fatalf("unexpected repaired validation result: %+v", result.Validation)
}
if admitter.releaseCalls != 1 {
t.Fatalf("admission releases=%d, want 1", admitter.releaseCalls)
}
if reader.calls != 1 || renderer.calls != 1 {
t.Fatalf("execution reopened preparation sources: artifact=%d render=%d", reader.calls, renderer.calls)
}
}
func TestRunnerRunPreparedReleasesAdmissionAcrossExecutionErrors(t *testing.T) {
generationFailure := errors.New("generation failed")
validationFailure := errors.New("validation failed")
repairFailure := errors.New("repair failed")
tests := []struct {
name string
generationErr error
validation *recordingPreparedValidation
repairer *fakeRepairer
wantError error
}{
{
name: "generation failure",
generationErr: generationFailure,
validation: &recordingPreparedValidation{},
wantError: ErrLLMGenerate,
},
{
name: "validation failure",
validation: &recordingPreparedValidation{
errs: []error{validationFailure},
},
wantError: ErrValidation,
},
{
name: "repair failure",
validation: &recordingPreparedValidation{
results: []domain.ValidationResult{{
Status: domain.ValidationFailed,
Mode: domain.ValidationJSON,
Errors: []string{"invalid"},
IsValid: false,
}},
},
repairer: &fakeRepairer{err: repairFailure},
wantError: ErrValidation,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
def := promptDef(domain.FormatJSON, domain.ValidationJSON, 1)
if test.repairer == nil {
def.Validation.RepairAttempts = 0
}
validator := &recordingValidationPreparer{plan: test.validation}
admitter := &fakeRunAdmitter{}
runner := NewRunnerWithRepairer(
&fakePromptRepo{def: def},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
nil,
defaultArtifactReader(),
defaultRenderer(),
&fakeLLM{
resp: &domain.GenerateResponse{Content: `{"value":true}`},
err: test.generationErr,
},
validator,
test.repairer,
admitter,
)
prepared, err := runner.PrepareExecution(context.Background(), domain.RunRequest{
PromptID: "p",
ProfileID: "exec",
Inputs: singleInputRef(),
})
if err != nil {
t.Fatalf("prepare execution: %v", err)
}
result, err := runner.RunPrepared(context.Background(), prepared)
if result != nil || !errors.Is(err, test.wantError) {
t.Fatalf("run prepared=(%+v, %v), want %v", result, err, test.wantError)
}
if len(admitter.backendIDs) != 1 || admitter.releaseCalls != 1 {
t.Fatalf(
"admission calls=%#v releases=%d, want one each",
admitter.backendIDs,
admitter.releaseCalls,
)
}
})
}
}
func TestRunnerPreparedExecutionOwnershipUseAndDiscard(t *testing.T) {
newRunner := func() *Runner {
return NewRunner(
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
nil,
defaultArtifactReader(),
defaultRenderer(),
&fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}},
&recordingValidationPreparer{plan: &recordingPreparedValidation{}},
nil,
)
}
request := domain.RunRequest{
PromptID: "p",
ProfileID: "exec",
Inputs: singleInputRef(),
}
owner := newRunner()
prepared, err := owner.PrepareExecution(context.Background(), request)
if err != nil {
t.Fatalf("prepare execution: %v", err)
}
if _, err := newRunner().RunPrepared(context.Background(), prepared); !errors.Is(err, ErrInvalidRequest) {
t.Fatalf("foreign runner error=%v, want ErrInvalidRequest", err)
}
if _, err := owner.RunPrepared(context.Background(), prepared); err != nil {
t.Fatalf("owner run prepared: %v", err)
}
if _, err := owner.RunPrepared(context.Background(), prepared); !errors.Is(err, ErrInvalidRequest) {
t.Fatalf("second owner run error=%v, want ErrInvalidRequest", err)
}
if prepared.Details() == nil {
t.Fatal("details unavailable after execution")
}
discarded, err := owner.PrepareExecution(context.Background(), request)
if err != nil {
t.Fatalf("prepare discarded execution: %v", err)
}
discarded.Discard()
discarded.Discard()
if _, err := owner.RunPrepared(context.Background(), discarded); !errors.Is(err, ErrInvalidRequest) {
t.Fatalf("discarded execution error=%v, want ErrInvalidRequest", err)
}
if discarded.Details() == nil {
t.Fatal("details unavailable after discard")
}
}
func TestRunnerPreparedExecutionWithoutValidatorSkipsValidation(t *testing.T) {
runner := NewRunner(
&fakePromptRepo{def: promptDef(domain.FormatJSON, domain.ValidationJSON, 0)},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
nil,
defaultArtifactReader(),
defaultRenderer(),
&fakeLLM{resp: &domain.GenerateResponse{Content: `{}`}},
nil,
nil,
)
prepared, err := runner.PrepareExecution(context.Background(), domain.RunRequest{
PromptID: "p",
ProfileID: "exec",
Inputs: singleInputRef(),
})
if err != nil {
t.Fatalf("prepare execution: %v", err)
}
result, err := runner.RunPrepared(context.Background(), prepared)
if err != nil {
t.Fatalf("run prepared: %v", err)
}
if result.Validation.Status != domain.ValidationSkipped || !result.Validation.IsValid {
t.Fatalf("unexpected no-validator result: %+v", result.Validation)
}
}
func TestRunnerPrepareExecutionRequiresValidationPreparer(t *testing.T) {
reader := defaultArtifactReader()
runner := NewRunner(
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationBasic, 0)},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
nil,
reader,
defaultRenderer(),
&fakeLLM{forbid: true},
&fakeValidator{},
nil,
)
prepared, err := runner.PrepareExecution(context.Background(), domain.RunRequest{
PromptID: "p",
ProfileID: "exec",
Inputs: singleInputRef(),
})
if prepared != nil || !errors.Is(err, ErrValidation) {
t.Fatalf("prepare execution=(%+v, %v), want ErrValidation", prepared, err)
}
if reader.calls != 0 {
t.Fatalf("unsupported validator allowed completion, artifact calls=%d", reader.calls)
}
}

View File

@@ -0,0 +1,103 @@
package usecase
import (
"context"
"errors"
"fmt"
"strings"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
)
type resolvedProfileSelection struct {
id string
profile *domain.ExecutionProfile
backend *domain.Backend
}
func (r *Runner) resolveProfileSelection(
ctx context.Context,
profileID string,
) (*resolvedProfileSelection, error) {
normalizedID := strings.TrimSpace(profileID)
if normalizedID == "" {
return nil, fmt.Errorf("%w: profile id is required", ErrInvalidRequest)
}
if r == nil || r.profiles == nil {
return nil, fmt.Errorf("%w: profile repository is not configured", ErrProfileLoad)
}
selectedProfile, err := r.profiles.GetProfile(ctx, normalizedID)
if err != nil {
return nil, fmt.Errorf("%w: %w", ErrProfileLoad, err)
}
if selectedProfile == nil {
return nil, fmt.Errorf("%w: profile repository returned nil profile", ErrProfileLoad)
}
profileValue := *selectedProfile
profileValue.BackendID = strings.TrimSpace(profileValue.BackendID)
var selectedBackend *domain.Backend
if profileValue.BackendID != "" {
if r.backends == nil {
return nil, fmt.Errorf("%w: backend %q cannot be resolved", ErrProfileLoad, profileValue.BackendID)
}
backendValue, err := r.backends.GetBackend(profileValue.BackendID)
if err != nil {
return nil, fmt.Errorf("%w: backend %q: %w", ErrProfileLoad, profileValue.BackendID, err)
}
selectedBackend = &backendValue
}
return &resolvedProfileSelection{
id: normalizedID,
profile: &profileValue,
backend: selectedBackend,
}, nil
}
func validateResolvedExecutionTarget(target domain.ExecutionTarget) error {
if strings.TrimSpace(target.Endpoint) == "" {
return errors.New("execution endpoint is required")
}
if strings.TrimSpace(target.Model) == "" {
return errors.New("execution model is required")
}
return nil
}
// InspectProfile resolves one explicit profile without prompt or execution work.
func (r *Runner) InspectProfile(
ctx context.Context,
profileID string,
) (*domain.ProfileInspection, error) {
normalizedID := strings.TrimSpace(profileID)
if normalizedID == "" {
return nil, fmt.Errorf("%w: profile id is required", ErrInvalidRequest)
}
select {
case <-ctx.Done():
return nil, fmt.Errorf("%w: %w", ErrProfileLoad, ctx.Err())
default:
}
selection, err := r.resolveProfileSelection(ctx, normalizedID)
if err != nil {
return nil, err
}
target, _, err := resolveExecutionTarget(selection.backend, selection.profile, nil)
if err != nil {
return nil, fmt.Errorf("%w: %w", ErrProfileLoad, err)
}
if err := validateResolvedExecutionTarget(target); err != nil {
return nil, fmt.Errorf("%w: %w", ErrProfileLoad, err)
}
target.APIKey = ""
return &domain.ProfileInspection{
ProfileID: selection.id,
EffectiveModelParams: target,
APIKeyRequired: target.APIKeyRequired,
}, nil
}

View File

@@ -0,0 +1,213 @@
package usecase
import (
"context"
"errors"
"reflect"
"testing"
"gitea.maximumdirect.net/eric/promptkit/internal/defaults"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
"gitea.maximumdirect.net/eric/promptkit/internal/profile"
)
type inspectionProfileRepository struct {
profile *domain.ExecutionProfile
err error
calls int
id string
}
func (r *inspectionProfileRepository) GetProfile(
_ context.Context,
id string,
) (*domain.ExecutionProfile, error) {
r.calls++
r.id = id
if r.err != nil {
return nil, r.err
}
return r.profile, nil
}
type inspectionBackendResolver struct {
backend domain.Backend
err error
calls int
id string
}
func (r *inspectionBackendResolver) GetBackend(id string) (domain.Backend, error) {
r.calls++
r.id = id
if r.err != nil {
return domain.Backend{}, r.err
}
return r.backend, nil
}
func TestRunnerInspectProfileResolvesProfileAndBackendOnce(t *testing.T) {
profiles := &inspectionProfileRepository{profile: &domain.ExecutionProfile{
ID: "profile",
BackendID: " backend ",
Model: "profile-model",
Temperature: 0.4,
MaxTokens: 32,
TimeoutSeconds: 45,
ServiceTier: "priority",
ReasoningEffort: "high",
ExtraParams: map[string]any{
"profile": "value",
},
}}
backends := &inspectionBackendResolver{backend: domain.Backend{
ID: "backend",
Endpoint: "https://backend.example/v1",
APIKeyEnv: "BACKEND_KEY",
ExtraParams: map[string]any{
"backend": "value",
},
}}
runner := &Runner{profiles: profiles, backends: backends}
inspection, err := runner.InspectProfile(context.Background(), " profile ")
if err != nil {
t.Fatalf("inspect profile: %v", err)
}
if profiles.calls != 1 || profiles.id != "profile" {
t.Fatalf("profile lookup=(calls=%d id=%q), want one exact lookup", profiles.calls, profiles.id)
}
if backends.calls != 1 || backends.id != "backend" {
t.Fatalf("backend lookup=(calls=%d id=%q), want one exact lookup", backends.calls, backends.id)
}
if profiles.profile.BackendID != " backend " {
t.Fatalf("inspection mutated repository profile backend: %q", profiles.profile.BackendID)
}
wantTarget := domain.ExecutionTarget{
BackendID: "backend",
Endpoint: "https://backend.example/v1",
Model: "profile-model",
Temperature: 0.4,
MaxTokens: 32,
TopP: defaults.ExecutionTargetDefault().TopP,
TimeoutSeconds: 45,
ServiceTier: "priority",
ReasoningEffort: "high",
APIKeyEnv: "BACKEND_KEY",
ExtraParams: map[string]any{
"profile": "value",
},
}
if inspection.ProfileID != "profile" || inspection.APIKeyRequired ||
!reflect.DeepEqual(inspection.EffectiveModelParams, wantTarget) {
t.Fatalf("inspection=%#v, want profile=%q target=%#v", inspection, "profile", wantTarget)
}
}
func TestRunnerInspectProfileDoesNotNeedExecutionCollaboratorsOrCredentials(t *testing.T) {
t.Setenv("PROMPTKIT_INSPECTION_TEST_KEY", "")
profiles := &inspectionProfileRepository{profile: &domain.ExecutionProfile{
ID: "endpoint-only",
Endpoint: "https://profile.example/v1",
Model: "profile-model",
APIKeyEnv: "PROMPTKIT_INSPECTION_TEST_KEY",
}}
runner := &Runner{profiles: profiles}
inspection, err := runner.InspectProfile(context.Background(), "endpoint-only")
if err != nil {
t.Fatalf("inspect endpoint-only profile: %v", err)
}
if inspection.EffectiveModelParams.BackendID != "" ||
inspection.EffectiveModelParams.APIKeyEnv != "PROMPTKIT_INSPECTION_TEST_KEY" ||
inspection.APIKeyRequired {
t.Fatalf("unexpected endpoint-only inspection: %#v", inspection)
}
}
func TestRunnerInspectProfileDirectCredentialRequirementClearsBackendEnvironment(t *testing.T) {
profiles := &inspectionProfileRepository{profile: &domain.ExecutionProfile{
ID: "direct-key",
BackendID: "backend",
Model: "profile-model",
APIKeyRequired: true,
}}
backends := &inspectionBackendResolver{backend: domain.Backend{
ID: "backend",
Endpoint: "https://backend.example/v1",
APIKeyEnv: "BACKEND_KEY",
}}
inspection, err := (&Runner{profiles: profiles, backends: backends}).InspectProfile(
context.Background(),
"direct-key",
)
if err != nil {
t.Fatalf("inspect direct-key profile: %v", err)
}
if !inspection.APIKeyRequired || inspection.EffectiveModelParams.APIKeyEnv != "" {
t.Fatalf("credential requirement was not resolved exclusively: %#v", inspection)
}
}
func TestRunnerInspectProfileClassifiesFailuresWithoutRepositoryWorkAfterCancellation(t *testing.T) {
t.Run("blank ID", func(t *testing.T) {
profiles := &inspectionProfileRepository{}
_, err := (&Runner{profiles: profiles}).InspectProfile(context.Background(), " \t ")
if !errors.Is(err, ErrInvalidRequest) || profiles.calls != 0 {
t.Fatalf("blank inspection=(%v, calls=%d), want invalid request without lookup", err, profiles.calls)
}
})
t.Run("canceled context", func(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
cancel()
profiles := &inspectionProfileRepository{}
_, err := (&Runner{profiles: profiles}).InspectProfile(ctx, "profile")
if !errors.Is(err, ErrProfileLoad) || !errors.Is(err, context.Canceled) || profiles.calls != 0 {
t.Fatalf("canceled inspection=(%v, calls=%d), want profile load and context identities without lookup", err, profiles.calls)
}
})
t.Run("missing profile", func(t *testing.T) {
profiles := &inspectionProfileRepository{err: profile.ErrProfileNotFound}
_, err := (&Runner{profiles: profiles}).InspectProfile(context.Background(), "missing")
if !errors.Is(err, ErrProfileLoad) || !errors.Is(err, profile.ErrProfileNotFound) {
t.Fatalf("missing profile error=%v, want profile load and not-found identities", err)
}
})
t.Run("unknown backend", func(t *testing.T) {
backendErr := errors.New("unknown backend")
profiles := &inspectionProfileRepository{profile: &domain.ExecutionProfile{
ID: "profile", BackendID: "backend", Model: "profile-model",
}}
backends := &inspectionBackendResolver{err: backendErr}
_, err := (&Runner{profiles: profiles, backends: backends}).InspectProfile(context.Background(), "profile")
if !errors.Is(err, ErrProfileLoad) || !errors.Is(err, backendErr) {
t.Fatalf("unknown backend error=%v, want profile load and backend identities", err)
}
})
t.Run("defensive invalid dependencies", func(t *testing.T) {
cases := []struct {
name string
runner *Runner
}{
{name: "nil repository", runner: &Runner{}},
{name: "nil profile", runner: &Runner{profiles: &inspectionProfileRepository{}}},
{name: "invalid target", runner: &Runner{profiles: &inspectionProfileRepository{
profile: &domain.ExecutionProfile{ID: "profile", Endpoint: "https://profile.example/v1"},
}}},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
_, err := tc.runner.InspectProfile(context.Background(), "profile")
if !errors.Is(err, ErrProfileLoad) {
t.Fatalf("inspection error=%v, want profile load", err)
}
})
}
})
}

View File

@@ -0,0 +1,73 @@
package usecase
import (
"context"
"fmt"
"strings"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
)
type resolvedPromptDefinition struct {
definition *domain.PromptDefinition
hash string
}
func (r *Runner) resolvePromptDefinition(
ctx context.Context,
promptID string,
promptVersion string,
) (*resolvedPromptDefinition, error) {
if strings.TrimSpace(promptID) == "" {
return nil, fmt.Errorf("%w: prompt id is required", ErrInvalidRequest)
}
if r == nil || r.promptDefs == nil {
return nil, fmt.Errorf("%w: prompt repository is not configured", ErrPromptLoad)
}
definition, err := r.promptDefs.GetPromptDefinition(ctx, promptID, promptVersion)
if err != nil {
return nil, fmt.Errorf("%w: %w", ErrPromptLoad, err)
}
if definition == nil {
return nil, fmt.Errorf("%w: prompt repository returned nil definition", ErrPromptLoad)
}
hash, err := hashPromptDefinition(definition)
if err != nil {
return nil, fmt.Errorf("%w: failed to hash prompt definition: %v", ErrPromptLoad, err)
}
return &resolvedPromptDefinition{definition: definition, hash: hash}, nil
}
// InspectPrompt resolves one explicit prompt without execution work.
func (r *Runner) InspectPrompt(
ctx context.Context,
promptID string,
promptVersion string,
) (*domain.PromptInspection, error) {
if strings.TrimSpace(promptID) == "" {
return nil, fmt.Errorf("%w: prompt id is required", ErrInvalidRequest)
}
select {
case <-ctx.Done():
return nil, fmt.Errorf("%w: %w", ErrPromptLoad, ctx.Err())
default:
}
selection, err := r.resolvePromptDefinition(ctx, promptID, promptVersion)
if err != nil {
return nil, err
}
inputs := make([]domain.PromptInput, len(selection.definition.Inputs))
copy(inputs, selection.definition.Inputs)
return &domain.PromptInspection{
PromptID: selection.definition.ID,
PromptVersion: selection.definition.Version,
PromptHash: selection.hash,
DefaultProfileID: selection.definition.DefaultProfile,
Inputs: inputs,
OutputContract: selection.definition.Validation,
}, nil
}

View File

@@ -0,0 +1,157 @@
package usecase
import (
"context"
"errors"
"reflect"
"testing"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
"gitea.maximumdirect.net/eric/promptkit/internal/promptdef"
)
type inspectionPromptRepository struct {
definition *domain.PromptDefinition
err error
calls int
id string
version string
}
func (r *inspectionPromptRepository) GetPromptDefinition(
_ context.Context,
id string,
version string,
) (*domain.PromptDefinition, error) {
r.calls++
r.id = id
r.version = version
if r.err != nil {
return nil, r.err
}
return r.definition, nil
}
func TestRunnerInspectPromptResolvesOneDefinitionWithoutExecutionCollaborators(t *testing.T) {
definition := &domain.PromptDefinition{
ID: "normalized.prompt",
Version: "1.2.3",
DefaultProfile: "not-resolved",
Inputs: []domain.PromptInput{
{Name: "document", Required: true, ContentType: "text/plain", Description: "Source document."},
{Name: "audience", ContentType: "text/plain", Description: "Intended reader."},
},
Validation: domain.OutputContract{
Format: domain.FormatJSON,
ValidationMode: domain.ValidationJSONSchema,
SchemaPath: "schemas/result.json",
RepairAttempts: 2,
},
}
repository := &inspectionPromptRepository{definition: definition}
runner := &Runner{promptDefs: repository}
inspection, err := runner.InspectPrompt(context.Background(), " prompt-id ", " version ")
if err != nil {
t.Fatalf("inspect prompt: %v", err)
}
wantHash, err := hashPromptDefinition(definition)
if err != nil {
t.Fatalf("hash prompt definition: %v", err)
}
if repository.calls != 1 || repository.id != " prompt-id " || repository.version != " version " {
t.Fatalf("prompt lookup=(calls=%d id=%q version=%q), want one unchanged lookup", repository.calls, repository.id, repository.version)
}
if inspection.PromptID != definition.ID ||
inspection.PromptVersion != definition.Version ||
inspection.PromptHash != wantHash ||
inspection.DefaultProfileID != definition.DefaultProfile ||
!reflect.DeepEqual(inspection.Inputs, definition.Inputs) ||
inspection.OutputContract != definition.Validation {
t.Fatalf("inspection=%#v, want definition metadata", inspection)
}
inspection.Inputs[0].Name = "changed"
second, err := runner.InspectPrompt(context.Background(), " prompt-id ", " version ")
if err != nil {
t.Fatalf("inspect prompt again: %v", err)
}
if definition.Inputs[0].Name != "document" || second.Inputs[0].Name != "document" {
t.Fatalf("inspection input mutation escaped caller result: definition=%#v next=%#v", definition.Inputs, second.Inputs)
}
}
func TestRunnerInspectPromptClassifiesFailuresWithoutRepositoryWorkAfterCancellation(t *testing.T) {
t.Run("blank ID", func(t *testing.T) {
repository := &inspectionPromptRepository{}
_, err := (&Runner{promptDefs: repository}).InspectPrompt(context.Background(), " \t ", "version")
if !errors.Is(err, ErrInvalidRequest) || repository.calls != 0 {
t.Fatalf("blank inspection=(%v, calls=%d), want invalid request without lookup", err, repository.calls)
}
})
t.Run("canceled context", func(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
cancel()
repository := &inspectionPromptRepository{}
_, err := (&Runner{promptDefs: repository}).InspectPrompt(ctx, "prompt", "version")
if !errors.Is(err, ErrPromptLoad) || !errors.Is(err, context.Canceled) || repository.calls != 0 {
t.Fatalf("canceled inspection=(%v, calls=%d), want prompt load and context identities without lookup", err, repository.calls)
}
})
t.Run("missing prompt", func(t *testing.T) {
repository := &inspectionPromptRepository{err: promptdef.ErrPromptDefinitionNotFound}
_, err := (&Runner{promptDefs: repository}).InspectPrompt(context.Background(), "missing", "version")
if !errors.Is(err, ErrPromptLoad) || !errors.Is(err, promptdef.ErrPromptDefinitionNotFound) {
t.Fatalf("missing prompt error=%v, want prompt load and not-found identities", err)
}
})
t.Run("defensive prompt dependencies", func(t *testing.T) {
var nilRunner *Runner
cases := []struct {
name string
runner *Runner
}{
{name: "nil runner", runner: nilRunner},
{name: "nil repository", runner: &Runner{}},
{name: "nil definition", runner: &Runner{promptDefs: &inspectionPromptRepository{}}},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
_, err := tc.runner.InspectPrompt(context.Background(), "prompt", "version")
if !errors.Is(err, ErrPromptLoad) {
t.Fatalf("inspection error=%v, want prompt load", err)
}
})
}
})
}
func TestRunnerPrepareUsesThePromptInspectionSelectionAndHash(t *testing.T) {
definition := promptDef(domain.FormatMarkdown, domain.ValidationBasic, 0)
repository := &fakePromptRepo{def: definition}
runner := NewRunner(
repository,
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
nil,
&fakeArtifactReader{},
&fakeRenderer{rendered: &domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hello"}}}},
nil,
nil,
nil,
)
inspection, err := runner.InspectPrompt(context.Background(), definition.ID, definition.Version)
if err != nil {
t.Fatalf("inspect prompt: %v", err)
}
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{PromptID: definition.ID, PromptVersion: definition.Version, ProfileID: "exec"})
if err != nil {
t.Fatalf("prepare prompt: %v", err)
}
if inspection.PromptHash != prepared.PromptHash {
t.Fatalf("inspection hash=%q, preparation hash=%q", inspection.PromptHash, prepared.PromptHash)
}
}

View File

@@ -17,6 +17,7 @@ type OutputRepairer interface {
type RepairRequest struct {
PreviousOutput string
ValidationErrors []string
SessionID string
Target domain.ExecutionTarget
StructuredOutput *domain.StructuredOutputSpec
Attempt int
@@ -42,23 +43,26 @@ func (r *defaultOutputRepairer) Repair(ctx context.Context, req RepairRequest) (
errs = strings.Join(req.ValidationErrors, "\n")
}
prompt := domain.RenderedPrompt{Messages: []domain.RenderedMessage{
{
Role: "system",
Content: "You repair invalid JSON output. Return only corrected JSON. Do not include explanations or markdown code fences.",
prompt := domain.RenderedPrompt{
SessionID: req.SessionID,
Messages: []domain.RenderedMessage{
{
Role: "system",
Content: "You repair invalid JSON output. Return only corrected JSON. Do not include explanations or markdown code fences.",
},
{
Role: "user",
Content: fmt.Sprintf(
"Repair attempt %d of %d for validation mode %s.\n\nValidation errors:\n%s\n\nPrevious output:\n%s\n\nReturn only corrected JSON.",
req.Attempt,
req.MaxAttempts,
req.Mode,
errs,
req.PreviousOutput,
),
},
},
{
Role: "user",
Content: fmt.Sprintf(
"Repair attempt %d of %d for validation mode %s.\n\nValidation errors:\n%s\n\nPrevious output:\n%s\n\nReturn only corrected JSON.",
req.Attempt,
req.MaxAttempts,
req.Mode,
errs,
req.PreviousOutput,
),
},
}}
}
resp, err := r.llm.Generate(ctx, domain.GenerateRequest{
Prompt: prompt,

View File

@@ -14,6 +14,7 @@ import (
"unicode"
"gitea.maximumdirect.net/eric/promptkit/internal/artifact"
"gitea.maximumdirect.net/eric/promptkit/internal/capacity"
"gitea.maximumdirect.net/eric/promptkit/internal/defaults"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
"gitea.maximumdirect.net/eric/promptkit/internal/llm"
@@ -40,41 +41,80 @@ var (
type Runner struct {
promptDefs promptdef.Repository
profiles profile.Repository
backends BackendResolver
artifacts artifact.Reader
renderer prompt.Renderer
llm llm.Client
validator validate.Validator
repairer OutputRepairer
admitter RunAdmitter
}
// BackendResolver resolves one normalized backend ID.
type BackendResolver interface {
GetBackend(string) (domain.Backend, error)
}
// RunAdmitter reserves capacity for one resolved backend run.
type RunAdmitter interface {
Admit(context.Context, string) (func(), error)
}
type preparationState struct {
definition *domain.PromptDefinition
directSessionID string
promptDefinitionHash string
selectedProfileID string
effectiveModel domain.ExecutionTarget
targetPresence domain.ExecutionTargetPresence
effectiveContract domain.OutputContract
start time.Time
}
func NewRunner(
promptDefs promptdef.Repository,
profiles profile.Repository,
backends BackendResolver,
artifacts artifact.Reader,
renderer prompt.Renderer,
llmClient llm.Client,
validator validate.Validator,
admitter RunAdmitter,
) *Runner {
return NewRunnerWithRepairer(promptDefs, profiles, artifacts, renderer, llmClient, validator, nil)
return NewRunnerWithRepairer(
promptDefs,
profiles,
backends,
artifacts,
renderer,
llmClient,
validator,
nil,
admitter,
)
}
func NewRunnerWithRepairer(
promptDefs promptdef.Repository,
profiles profile.Repository,
backends BackendResolver,
artifacts artifact.Reader,
renderer prompt.Renderer,
llmClient llm.Client,
validator validate.Validator,
repairer OutputRepairer,
admitter RunAdmitter,
) *Runner {
return &Runner{
promptDefs: promptDefs,
profiles: profiles,
backends: backends,
artifacts: artifacts,
renderer: renderer,
llm: llmClient,
validator: validator,
repairer: repairer,
admitter: admitter,
}
}
@@ -86,14 +126,51 @@ func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunRes
start := time.Now().UTC()
prepared, err := r.Prepare(ctx, req)
state, err := r.resolvePreparation(ctx, req, time.Now().UTC())
if err != nil {
return nil, err
}
release, err := r.admitRun(ctx, state.effectiveModel.BackendID)
if err != nil {
return nil, err
}
defer release()
prepared, err := r.completePreparation(ctx, req, state)
if err != nil {
return nil, err
}
directAPIKey := state.effectiveModel.APIKey
return r.executePreparedRun(ctx, prepared, directAPIKey, runID, start, func(
ctx context.Context,
artifact *domain.Artifact,
attemptsUsed int,
) (domain.ValidationResult, error) {
return r.validateOutput(ctx, artifact, prepared.OutputContract, attemptsUsed)
})
}
type preparedValidationFunc func(
context.Context,
*domain.Artifact,
int,
) (domain.ValidationResult, error)
func (r *Runner) executePreparedRun(
ctx context.Context,
prepared *domain.PreparedRun,
directAPIKey string,
runID string,
start time.Time,
validateArtifact preparedValidationFunc,
) (*domain.RunResult, error) {
executionTarget := prepared.EffectiveModelParams
executionTarget.APIKey = directAPIKey
genResp, err := r.llm.Generate(ctx, domain.GenerateRequest{
Prompt: domain.RenderedPrompt{SessionID: prepared.SessionID, Messages: prepared.Messages},
Target: prepared.EffectiveModelParams,
Target: executionTarget,
TargetPresence: prepared.TargetPresence,
StructuredOutput: prepared.StructuredOutput,
})
@@ -105,7 +182,7 @@ func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunRes
}
outputArtifact := buildOutputArtifact(genResp.Content, prepared.OutputContract.Format)
validationResult, err := r.validateOutput(ctx, &outputArtifact, prepared.OutputContract, 0)
validationResult, err := validateArtifact(ctx, &outputArtifact, 0)
if err != nil {
return nil, fmt.Errorf("%w: %w", ErrValidation, err)
}
@@ -118,7 +195,8 @@ func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunRes
repairResp, repairErr := r.repairer.Repair(ctx, RepairRequest{
PreviousOutput: genResp.Content,
ValidationErrors: validationResult.Errors,
Target: prepared.EffectiveModelParams,
SessionID: prepared.SessionID,
Target: executionTarget,
StructuredOutput: prepared.StructuredOutput,
Attempt: attemptsUsed,
MaxAttempts: prepared.OutputContract.RepairAttempts,
@@ -134,7 +212,7 @@ func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunRes
genResp = repairResp
outputArtifact = buildOutputArtifact(genResp.Content, prepared.OutputContract.Format)
validationResult, err = r.validateOutput(ctx, &outputArtifact, prepared.OutputContract, attemptsUsed)
validationResult, err = validateArtifact(ctx, &outputArtifact, attemptsUsed)
if err != nil {
return nil, fmt.Errorf("%w: %w", ErrValidation, err)
}
@@ -142,6 +220,7 @@ func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunRes
}
end := time.Now().UTC()
executionTarget.APIKey = ""
return &domain.RunResult{
RunID: runID,
@@ -151,11 +230,13 @@ func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunRes
PromptID: prepared.PromptID,
PromptVersion: prepared.PromptVersion,
PromptHash: prepared.PromptHash,
SessionID: prepared.SessionID,
RenderedPromptHash: prepared.RenderedPromptHash,
SelectedProfileID: prepared.SelectedProfileID,
SelectedBackendID: prepared.SelectedBackendID,
ModelName: prepared.EffectiveModelParams.Model,
Endpoint: prepared.EffectiveModelParams.Endpoint,
EffectiveModelParams: prepared.EffectiveModelParams,
EffectiveModelParams: executionTarget,
InputHashes: prepared.InputHashes,
Usage: genResp.Usage,
StartTime: start,
@@ -165,20 +246,32 @@ func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunRes
}
func (r *Runner) Prepare(ctx context.Context, req domain.RunRequest) (*domain.PreparedRun, error) {
state, err := r.resolvePreparation(ctx, req, time.Now().UTC())
if err != nil {
return nil, err
}
return r.completePreparation(ctx, req, state)
}
func (r *Runner) resolvePreparation(
ctx context.Context,
req domain.RunRequest,
start time.Time,
) (*preparationState, error) {
if strings.TrimSpace(req.PromptID) == "" {
return nil, fmt.Errorf("%w: prompt id is required", ErrInvalidRequest)
}
start := time.Now().UTC()
def, err := r.promptDefs.GetPromptDefinition(ctx, req.PromptID, req.PromptVersion)
directSessionID, err := domain.NormalizeSessionID(req.SessionID)
if err != nil {
return nil, fmt.Errorf("%w: %w", ErrPromptLoad, err)
return nil, fmt.Errorf("%w: session_id: %v", ErrInvalidRequest, err)
}
promptDefinitionHash, err := hashPromptDefinition(def)
promptSelection, err := r.resolvePromptDefinition(ctx, req.PromptID, req.PromptVersion)
if err != nil {
return nil, fmt.Errorf("%w: failed to hash prompt definition: %v", ErrPromptLoad, err)
return nil, err
}
def := promptSelection.definition
promptDefinitionHash := promptSelection.hash
selectedProfileID := strings.TrimSpace(req.ProfileID)
if selectedProfileID == "" {
@@ -188,32 +281,58 @@ func (r *Runner) Prepare(ctx context.Context, req domain.RunRequest) (*domain.Pr
return nil, fmt.Errorf("%w: %w: profile id is required either in request or prompt default_profile", ErrInvalidRequest, ErrProfileRequired)
}
execProfile, err := r.profiles.GetProfile(ctx, selectedProfileID)
selection, err := r.resolveProfileSelection(ctx, selectedProfileID)
if err != nil {
return nil, fmt.Errorf("%w: %w", ErrProfileLoad, err)
return nil, err
}
effectiveModel, targetPresence, err := resolveExecutionTarget(execProfile, req.Execution)
effectiveModel, targetPresence, err := resolveExecutionTarget(selection.backend, selection.profile, req.Execution)
if err != nil {
return nil, fmt.Errorf("%w: %w", ErrInvalidRequest, err)
}
effectiveModel.APIKey = req.APIKey
if strings.TrimSpace(effectiveModel.Endpoint) == "" {
return nil, fmt.Errorf("%w: execution endpoint is required", ErrInvalidRequest)
}
if strings.TrimSpace(effectiveModel.Model) == "" {
return nil, fmt.Errorf("%w: execution model is required", ErrInvalidRequest)
if err := validateResolvedExecutionTarget(effectiveModel); err != nil {
return nil, fmt.Errorf("%w: %w", ErrInvalidRequest, err)
}
if err := validateAPIKey(effectiveModel.APIKeyEnv, effectiveModel.APIKey, effectiveModel.APIKeyRequired); err != nil {
return nil, fmt.Errorf("%w: %w", ErrInvalidRequest, err)
}
effectiveContract := resolveOutputContract(def, req.Validation)
structuredOutput, err := r.resolveStructuredOutput(ctx, def, effectiveContract)
return &preparationState{
definition: def,
directSessionID: directSessionID,
promptDefinitionHash: promptDefinitionHash,
selectedProfileID: selection.id,
effectiveModel: effectiveModel,
targetPresence: targetPresence,
effectiveContract: effectiveContract,
start: start,
}, nil
}
func (r *Runner) completePreparation(
ctx context.Context,
req domain.RunRequest,
state *preparationState,
) (*domain.PreparedRun, error) {
structuredOutput, err := r.resolveStructuredOutput(
ctx,
state.definition,
state.effectiveContract,
)
if err != nil {
return nil, err
}
return r.completePreparationWithStructuredOutput(ctx, req, state, structuredOutput)
}
func (r *Runner) completePreparationWithStructuredOutput(
ctx context.Context,
req domain.RunRequest,
state *preparationState,
structuredOutput *domain.StructuredOutputSpec,
) (*domain.PreparedRun, error) {
resolvedInputs := make(map[string]*domain.Artifact, len(req.Inputs))
inputHashes := make(map[string]string, len(req.Inputs))
for name, ref := range req.Inputs {
@@ -228,31 +347,57 @@ func (r *Runner) Prepare(ctx context.Context, req domain.RunRequest) (*domain.Pr
inputHashes[name] = art.Hash
}
renderedPrompt, err := r.renderer.Render(ctx, def, resolvedInputs, req.Vars)
definitionToRender := state.definition
if state.directSessionID != "" {
definitionCopy := *state.definition
definitionCopy.SessionID = ""
definitionToRender = &definitionCopy
}
renderedPrompt, err := r.renderer.Render(ctx, definitionToRender, resolvedInputs, req.Vars)
if err != nil {
return nil, fmt.Errorf("%w: %w", ErrPromptRender, err)
}
if state.directSessionID != "" {
renderedPrompt.SessionID = state.directSessionID
}
end := time.Now().UTC()
effectiveModel := state.effectiveModel
effectiveModel.APIKey = ""
return &domain.PreparedRun{
PromptID: def.ID,
PromptVersion: def.Version,
PromptHash: promptDefinitionHash,
SelectedProfileID: selectedProfileID,
PromptID: state.definition.ID,
PromptVersion: state.definition.Version,
PromptHash: state.promptDefinitionHash,
SelectedProfileID: state.selectedProfileID,
SelectedBackendID: state.effectiveModel.BackendID,
EffectiveModelParams: effectiveModel,
TargetPresence: targetPresence,
OutputContract: effectiveContract,
TargetPresence: state.targetPresence,
OutputContract: state.effectiveContract,
StructuredOutput: structuredOutput,
InputHashes: inputHashes,
SessionID: renderedPrompt.SessionID,
RenderedPromptHash: hashRenderedPrompt(*renderedPrompt),
Messages: renderedPrompt.Messages,
StartTime: start,
StartTime: state.start,
EndTime: end,
DurationMS: end.Sub(start).Milliseconds(),
DurationMS: end.Sub(state.start).Milliseconds(),
}, nil
}
func (r *Runner) admitRun(ctx context.Context, backendID string) (func(), error) {
if r.admitter == nil {
return func() {}, nil
}
release, err := r.admitter.Admit(ctx, backendID)
if err != nil {
if errors.Is(err, capacity.ErrCapacityExceeded) && strings.TrimSpace(backendID) != "" {
return nil, &CapacityError{BackendID: backendID}
}
return nil, err
}
return release, nil
}
func (r *Runner) resolveStructuredOutput(ctx context.Context, def *domain.PromptDefinition, contract domain.OutputContract) (*domain.StructuredOutputSpec, error) {
if contract.ValidationMode != domain.ValidationJSONSchema {
return nil, nil
@@ -268,14 +413,18 @@ func (r *Runner) resolveStructuredOutput(ctx context.Context, def *domain.Prompt
return nil, fmt.Errorf("%w: failed to load json schema for structured output: %v", ErrValidation, err)
}
return structuredOutputSpec(def, schemaDoc), nil
}
func structuredOutputSpec(def *domain.PromptDefinition, schemaDocument any) *domain.StructuredOutputSpec {
return &domain.StructuredOutputSpec{
Type: domain.StructuredOutputJSONSchema,
JSONSchema: &domain.StructuredOutputJSONSpec{
Name: deriveStructuredSchemaName(def.ID, def.Version),
Strict: true,
Schema: schemaDoc,
Schema: schemaDocument,
},
}, nil
}
}
func deriveStructuredSchemaName(promptID string, promptVersion string) string {
@@ -338,7 +487,10 @@ func (r *Runner) shouldAttemptRepair(contract domain.OutputContract, validationR
func mergeExecutionTarget(base domain.ExecutionTarget, override domain.ExecutionTarget) domain.ExecutionTarget {
out := base
if override.Endpoint != "" {
if strings.TrimSpace(override.BackendID) != "" {
out.BackendID = override.BackendID
}
if strings.TrimSpace(override.Endpoint) != "" {
out.Endpoint = override.Endpoint
}
if override.Model != "" {
@@ -367,6 +519,7 @@ func mergeExecutionTarget(base domain.ExecutionTarget, override domain.Execution
}
if override.APIKeyRequired {
out.APIKeyRequired = true
out.APIKeyEnv = ""
}
if len(override.ExtraParams) > 0 {
out.ExtraParams = copyExtraParams(override.ExtraParams)
@@ -414,8 +567,8 @@ func mergeExecutionTargetOverride(base domain.ExecutionTarget, override domain.E
if strings.TrimSpace(override.ServiceTier) != "" {
out.ServiceTier = override.ServiceTier
}
if strings.TrimSpace(override.ReasoningEffort) != "" {
out.ReasoningEffort = override.ReasoningEffort
if override.ReasoningEffort != nil {
out.ReasoningEffort = strings.TrimSpace(*override.ReasoningEffort)
}
if strings.TrimSpace(override.APIKeyEnv) != "" {
out.APIKeyEnv = override.APIKeyEnv
@@ -426,8 +579,9 @@ func mergeExecutionTargetOverride(base domain.ExecutionTarget, override domain.E
return out, presence, nil
}
func resolveExecutionTarget(profileValue *domain.ExecutionProfile, override *domain.ExecutionTargetOverride) (domain.ExecutionTarget, domain.ExecutionTargetPresence, error) {
func resolveExecutionTarget(backendValue *domain.Backend, profileValue *domain.ExecutionProfile, override *domain.ExecutionTargetOverride) (domain.ExecutionTarget, domain.ExecutionTargetPresence, error) {
out := defaults.ExecutionTargetDefault()
out = mergeExecutionTarget(out, backendToTarget(backendValue))
out = mergeExecutionTarget(out, executionProfileToTarget(profileValue))
var presence domain.ExecutionTargetPresence
if override != nil {
@@ -461,8 +615,13 @@ func executionProfileToTarget(p *domain.ExecutionProfile) domain.ExecutionTarget
if p == nil {
return domain.ExecutionTarget{}
}
endpoint := p.Endpoint
if strings.TrimSpace(endpoint) == "" {
endpoint = ""
}
return domain.ExecutionTarget{
Endpoint: p.Endpoint,
BackendID: p.BackendID,
Endpoint: endpoint,
Model: p.Model,
Temperature: p.Temperature,
MaxTokens: p.MaxTokens,
@@ -476,6 +635,18 @@ func executionProfileToTarget(p *domain.ExecutionProfile) domain.ExecutionTarget
}
}
func backendToTarget(value *domain.Backend) domain.ExecutionTarget {
if value == nil {
return domain.ExecutionTarget{}
}
return domain.ExecutionTarget{
BackendID: value.ID,
Endpoint: value.Endpoint,
APIKeyEnv: value.APIKeyEnv,
ExtraParams: copyExtraParams(value.ExtraParams),
}
}
func copyExtraParams(src map[string]any) map[string]any {
if len(src) == 0 {
return nil

File diff suppressed because it is too large Load Diff

View File

@@ -6,6 +6,7 @@ import (
"errors"
"fmt"
"io/fs"
"net/url"
"os"
"path"
"path/filepath"
@@ -16,6 +17,8 @@ import (
"github.com/santhosh-tekuri/jsonschema/v6"
)
const jsonSchemaDraft2020 = "https://json-schema.org/draft/2020-12/schema"
// StandardValidator provides basic, JSON, and JSON Schema output validation.
type StandardValidator struct {
schemaBaseDir string
@@ -42,6 +45,101 @@ func (v *FSValidator) Validate(ctx context.Context, artifact *domain.Artifact, c
return validateArtifact(ctx, artifact, contract, v.validateJSONSchema)
}
type preparedValidation struct {
contract domain.OutputContract
schemaDocument any
schema *jsonschema.Schema
}
func (p *preparedValidation) Validate(ctx context.Context, artifact *domain.Artifact) (domain.ValidationResult, error) {
return validateArtifact(ctx, artifact, p.contract, p.validateJSONSchema)
}
func (p *preparedValidation) SchemaDocument() any {
return p.schemaDocument
}
func (p *preparedValidation) validateJSONSchema(instance any, _ string) ([]string, error) {
if p.schema == nil {
return nil, errors.New("prepared JSON schema is unavailable")
}
if err := p.schema.Validate(instance); err != nil {
return []string{fmt.Sprintf("json schema validation failed: %v", err)}, nil
}
return nil, nil
}
func (v *StandardValidator) PrepareValidation(ctx context.Context, contract domain.OutputContract) (PreparedValidation, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
prepared := &preparedValidation{contract: contract}
if contract.ValidationMode != domain.ValidationJSONSchema {
return prepared, nil
}
resolvedSchemaPath, err := v.resolveSchemaPath(contract.SchemaPath)
if err != nil {
return nil, err
}
schemaDocument, err := loadJSONSchemaFile(resolvedSchemaPath)
if err != nil {
return nil, fmt.Errorf("failed to compile JSON schema %q: %w", resolvedSchemaPath, err)
}
schemaRoot, err := v.schemaRoot()
if err != nil {
return nil, err
}
compiler := newSchemaCompiler(standardSchemaLoader{root: schemaRoot})
if err := compiler.AddResource(resolvedSchemaPath, schemaDocument); err != nil {
return nil, fmt.Errorf("failed to register JSON schema %q: %w", resolvedSchemaPath, err)
}
schema, err := compiler.Compile(resolvedSchemaPath)
if err != nil {
return nil, fmt.Errorf("failed to compile JSON schema %q: %w", resolvedSchemaPath, err)
}
if err := ctx.Err(); err != nil {
return nil, err
}
prepared.schemaDocument = schemaDocument
prepared.schema = schema
return prepared, nil
}
func (v *FSValidator) PrepareValidation(ctx context.Context, contract domain.OutputContract) (PreparedValidation, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
prepared := &preparedValidation{contract: contract}
if contract.ValidationMode != domain.ValidationJSONSchema {
return prepared, nil
}
schemaName, schemaDocument, err := v.loadSchemaDocument(contract.SchemaPath)
if err != nil {
return nil, err
}
resourceURL := fsSchemaResourceURL(schemaName)
compiler := newSchemaCompiler(fsSchemaLoader{fsys: v.fsys, root: filecatalog.CleanFSRoot(v.root)})
if err := compiler.AddResource(resourceURL, schemaDocument); err != nil {
return nil, fmt.Errorf("failed to register JSON schema %q: %w", schemaName, err)
}
schema, err := compiler.Compile(resourceURL)
if err != nil {
return nil, fmt.Errorf("failed to compile JSON schema %q: %w", schemaName, err)
}
if err := ctx.Err(); err != nil {
return nil, err
}
prepared.schemaDocument = schemaDocument
prepared.schema = schema
return prepared, nil
}
type schemaValidatorFunc func(instance any, schemaPath string) ([]string, error)
func validateArtifact(ctx context.Context, artifact *domain.Artifact, contract domain.OutputContract, validateSchema schemaValidatorFunc) (domain.ValidationResult, error) {
@@ -121,7 +219,11 @@ func (v *StandardValidator) validateJSONSchema(instance any, schemaPath string)
return nil, err
}
compiler := jsonschema.NewCompiler()
schemaRoot, err := v.schemaRoot()
if err != nil {
return nil, err
}
compiler := newSchemaCompiler(standardSchemaLoader{root: schemaRoot})
schema, err := compiler.Compile(resolvedSchemaPath)
if err != nil {
return nil, fmt.Errorf("failed to compile JSON schema %q: %w", resolvedSchemaPath, err)
@@ -140,7 +242,10 @@ func (v *FSValidator) validateJSONSchema(instance any, schemaPath string) ([]str
}
resourceURL := fsSchemaResourceURL(schemaName)
compiler := jsonschema.NewCompiler()
if err := validateSchemaDialect(schemaDoc); err != nil {
return nil, fmt.Errorf("failed to compile JSON schema %q: %w", schemaName, err)
}
compiler := newSchemaCompiler(fsSchemaLoader{fsys: v.fsys, root: filecatalog.CleanFSRoot(v.root)})
if err := compiler.AddResource(resourceURL, schemaDoc); err != nil {
return nil, fmt.Errorf("failed to register JSON schema %q: %w", schemaName, err)
}
@@ -184,6 +289,9 @@ func (v *StandardValidator) LoadSchemaDocument(ctx context.Context, schemaPath s
if err := json.Unmarshal(raw, &doc); err != nil {
return nil, fmt.Errorf("failed to decode JSON schema %q: %w", resolved, err)
}
if err := validateSchemaDialect(doc); err != nil {
return nil, fmt.Errorf("failed to decode JSON schema %q: %w", resolved, err)
}
return doc, nil
}
@@ -206,12 +314,14 @@ func (v *StandardValidator) resolveSchemaPath(schemaPath string) (string, error)
return "", errors.New("schema path is required for json_schema validation")
}
resolved := schemaPath
if !filepath.IsAbs(schemaPath) {
resolved = filepath.Join(v.schemaBaseDir, schemaPath)
root, err := v.schemaRoot()
if err != nil {
return "", err
}
resolved, err := containedFilesystemPath(root, schemaPath)
if err != nil {
return "", err
}
resolved = filepath.Clean(resolved)
if _, err := os.Stat(resolved); err != nil {
return "", fmt.Errorf("failed to access schema file %q: %w", resolved, err)
}
@@ -219,6 +329,22 @@ func (v *StandardValidator) resolveSchemaPath(schemaPath string) (string, error)
return resolved, nil
}
func (v *StandardValidator) schemaRoot() (string, error) {
root := v.schemaBaseDir
if strings.TrimSpace(root) == "" {
root = "."
}
absolute, err := filepath.Abs(root)
if err != nil {
return "", fmt.Errorf("failed to resolve schema source %q: %w", root, err)
}
resolved, err := filepath.EvalSymlinks(absolute)
if err != nil {
return "", fmt.Errorf("failed to access schema source %q: %w", root, err)
}
return resolved, nil
}
func (v *FSValidator) loadSchemaDocument(schemaPath string) (string, any, error) {
resolved, err := v.resolveSchemaPath(schemaPath)
if err != nil {
@@ -234,6 +360,9 @@ func (v *FSValidator) loadSchemaDocument(schemaPath string) (string, any, error)
if err := json.Unmarshal(raw, &doc); err != nil {
return "", nil, fmt.Errorf("failed to decode JSON schema %q: %w", resolved, err)
}
if err := validateSchemaDialect(doc); err != nil {
return "", nil, fmt.Errorf("failed to decode JSON schema %q: %w", resolved, err)
}
return resolved, doc, nil
}
@@ -290,3 +419,131 @@ func cleanSchemaFSPath(schemaPath string) (string, error) {
func fsSchemaResourceURL(schemaName string) string {
return "promptkit-schema:///" + strings.TrimPrefix(path.Clean(schemaName), "/")
}
func newSchemaCompiler(loader jsonschema.URLLoader) *jsonschema.Compiler {
compiler := jsonschema.NewCompiler()
compiler.DefaultDraft(jsonschema.Draft2020)
compiler.UseLoader(loader)
return compiler
}
func validateSchemaDialect(doc any) error {
object, ok := doc.(map[string]any)
if !ok {
return nil
}
value, ok := object["$schema"]
if !ok {
return nil
}
dialect, ok := value.(string)
if !ok {
return errors.New("$schema must be a string")
}
if dialect != jsonSchemaDraft2020 && dialect != jsonSchemaDraft2020+"#" {
return fmt.Errorf("unsupported JSON Schema dialect %q; expected %q", dialect, jsonSchemaDraft2020)
}
return nil
}
type standardSchemaLoader struct {
root string
}
func (l standardSchemaLoader) Load(resourceURL string) (any, error) {
fileName, err := (jsonschema.FileLoader{}).ToFile(resourceURL)
if err != nil {
return nil, fmt.Errorf("schema reference %q is not a contained file reference: %w", resourceURL, err)
}
resolved, err := containedFilesystemPath(l.root, fileName)
if err != nil {
return nil, err
}
return loadJSONSchemaFile(resolved)
}
func containedFilesystemPath(root, name string) (string, error) {
candidate := name
if !filepath.IsAbs(candidate) {
candidate = filepath.Join(root, candidate)
}
candidate, err := filepath.Abs(candidate)
if err != nil {
return "", fmt.Errorf("failed to resolve schema path %q: %w", name, err)
}
candidate, err = filepath.EvalSymlinks(candidate)
if err != nil {
return "", fmt.Errorf("failed to access schema file %q: %w", candidate, err)
}
relative, err := filepath.Rel(root, candidate)
if err != nil {
return "", fmt.Errorf("failed to compare schema path %q with source root: %w", candidate, err)
}
if relative == ".." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) {
return "", fmt.Errorf("schema path %q escapes source root", name)
}
return candidate, nil
}
func loadJSONSchemaFile(name string) (any, error) {
raw, err := os.ReadFile(name)
if err != nil {
return nil, err
}
var doc any
if err := json.Unmarshal(raw, &doc); err != nil {
return nil, err
}
if err := validateSchemaDialect(doc); err != nil {
return nil, err
}
return doc, nil
}
type fsSchemaLoader struct {
fsys fs.FS
root string
}
func (l fsSchemaLoader) Load(resourceURL string) (any, error) {
parsed, err := url.Parse(resourceURL)
if err != nil {
return nil, fmt.Errorf("invalid schema reference %q: %w", resourceURL, err)
}
if parsed.Scheme != "promptkit-schema" || parsed.Host != "" {
return nil, fmt.Errorf("schema reference %q is not allowed", resourceURL)
}
name, err := url.PathUnescape(strings.TrimPrefix(parsed.Path, "/"))
if err != nil {
return nil, fmt.Errorf("invalid schema reference %q: %w", resourceURL, err)
}
name = path.Clean(name)
if l.root == "." {
if strings.HasPrefix(name, "../") || name == ".." {
return nil, fmt.Errorf("schema reference %q escapes source root", resourceURL)
}
} else if name != l.root && !strings.HasPrefix(name, l.root+"/") {
return nil, fmt.Errorf("schema reference %q escapes source root", resourceURL)
}
rootInfo, err := fs.Stat(l.fsys, l.root)
if err != nil {
return nil, err
}
if !rootInfo.IsDir() && name != l.root {
return nil, fmt.Errorf("schema reference %q is outside the configured schema file", resourceURL)
}
raw, err := fs.ReadFile(l.fsys, name)
if err != nil {
return nil, err
}
var doc any
if err := json.Unmarshal(raw, &doc); err != nil {
return nil, err
}
if err := validateSchemaDialect(doc); err != nil {
return nil, err
}
return doc, nil
}

View File

@@ -2,8 +2,11 @@ package validate
import (
"context"
"encoding/json"
"os"
"path/filepath"
"reflect"
"strconv"
"strings"
"testing"
"testing/fstest"
@@ -118,6 +121,68 @@ func TestStandardValidatorJSONSchemaSuccess(t *testing.T) {
}
}
func TestStandardValidatorPreparedSchemaSurvivesSourceRemoval(t *testing.T) {
tmp := t.TempDir()
rootSchema := []byte(`{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"title": "original root",
"type": "object",
"required": ["value"],
"properties": {
"value": {"$ref": "value.json"}
}
}`)
rootPath := filepath.Join(tmp, "schema.json")
referencePath := filepath.Join(tmp, "value.json")
if err := os.WriteFile(rootPath, rootSchema, 0o644); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(referencePath, []byte(`{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"type": "integer",
"minimum": 2
}`), 0o644); err != nil {
t.Fatal(err)
}
validator := NewStandardValidator(tmp)
preparer, ok := validator.(ValidationPreparer)
if !ok {
t.Fatal("standard validator does not support validation preparation")
}
prepared, err := preparer.PrepareValidation(context.Background(), domain.OutputContract{
ValidationMode: domain.ValidationJSONSchema,
SchemaPath: "schema.json",
})
if err != nil {
t.Fatalf("prepare validation: %v", err)
}
assertSchemaDocument(t, prepared.SchemaDocument(), rootSchema)
if err := os.Remove(rootPath); err != nil {
t.Fatal(err)
}
if err := os.Remove(referencePath); err != nil {
t.Fatal(err)
}
valid, err := prepared.Validate(context.Background(), &domain.Artifact{Body: []byte(`{"value":3}`)})
if err != nil {
t.Fatalf("validate prepared artifact: %v", err)
}
if valid.Status != domain.ValidationPassed || !valid.IsValid {
t.Fatalf("expected passed/valid, got status=%q valid=%v errors=%v", valid.Status, valid.IsValid, valid.Errors)
}
invalid, err := prepared.Validate(context.Background(), &domain.Artifact{Body: []byte(`{"value":"changed"}`)})
if err != nil {
t.Fatalf("validate prepared artifact: %v", err)
}
if invalid.Status != domain.ValidationFailed || invalid.IsValid {
t.Fatalf("expected failed/invalid, got status=%q valid=%v", invalid.Status, invalid.IsValid)
}
}
func TestStandardValidatorJSONSchemaNestedSchemaPathSuccess(t *testing.T) {
tmp := t.TempDir()
nestedDir := filepath.Join(tmp, "dnd")
@@ -293,6 +358,64 @@ func TestFSValidatorJSONSchemaSuccess(t *testing.T) {
}
}
func TestFSValidatorPreparedSchemaSurvivesSourceMutation(t *testing.T) {
rootSchema := []byte(`{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"title": "original root",
"type": "object",
"required": ["value"],
"properties": {
"value": {"$ref": "value.json"}
}
}`)
fsys := fstest.MapFS{
"schema.json": &fstest.MapFile{Data: rootSchema},
"value.json": &fstest.MapFile{Data: []byte(`{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"type": "integer",
"minimum": 2
}`)},
}
validator := NewFSValidator(fsys, ".")
preparer, ok := validator.(ValidationPreparer)
if !ok {
t.Fatal("filesystem validator does not support validation preparation")
}
prepared, err := preparer.PrepareValidation(context.Background(), domain.OutputContract{
ValidationMode: domain.ValidationJSONSchema,
SchemaPath: "schema.json",
})
if err != nil {
t.Fatalf("prepare validation: %v", err)
}
assertSchemaDocument(t, prepared.SchemaDocument(), rootSchema)
fsys["schema.json"] = &fstest.MapFile{Data: []byte(`{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"type": "string"
}`)}
fsys["value.json"] = &fstest.MapFile{Data: []byte(`{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"type": "string"
}`)}
valid, err := prepared.Validate(context.Background(), &domain.Artifact{Body: []byte(`{"value":3}`)})
if err != nil {
t.Fatalf("validate prepared artifact: %v", err)
}
if valid.Status != domain.ValidationPassed || !valid.IsValid {
t.Fatalf("expected passed/valid, got status=%q valid=%v errors=%v", valid.Status, valid.IsValid, valid.Errors)
}
invalid, err := prepared.Validate(context.Background(), &domain.Artifact{Body: []byte(`{"value":"changed"}`)})
if err != nil {
t.Fatalf("validate prepared artifact: %v", err)
}
if invalid.Status != domain.ValidationFailed || invalid.IsValid {
t.Fatalf("expected failed/invalid, got status=%q valid=%v", invalid.Status, invalid.IsValid)
}
}
func TestFSValidatorJSONSchemaRegistrationError(t *testing.T) {
v := NewFSValidator(fstest.MapFS{
"schemas/%zz.json": &fstest.MapFile{Data: []byte(`{"type":"object"}`)},
@@ -411,3 +534,150 @@ func TestFSValidatorLoadSchemaDocument(t *testing.T) {
t.Fatalf("unexpected schema document: %#v", doc)
}
}
func TestStandardValidatorJSONSchemaReferenceBoundaries(t *testing.T) {
root := t.TempDir()
if err := os.WriteFile(filepath.Join(root, "child.json"), []byte(`{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"type": "string"
}`), 0o644); err != nil {
t.Fatal(err)
}
tests := []struct {
name string
reference string
wantError string
writeOuter bool
}{
{name: "contained relative reference", reference: "child.json"},
{name: "remote reference", reference: "https://example.test/schema.json", wantError: "not a contained file reference"},
{name: "escaping reference", reference: "../outside.json", wantError: "escapes source root", writeOuter: true},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
if tc.writeOuter {
if err := os.WriteFile(filepath.Join(filepath.Dir(root), "outside.json"), []byte(`{"type":"string"}`), 0o644); err != nil {
t.Fatal(err)
}
}
schema := `{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$ref": ` + strconv.Quote(tc.reference) + `
}`
if err := os.WriteFile(filepath.Join(root, "root.json"), []byte(schema), 0o644); err != nil {
t.Fatal(err)
}
v := NewStandardValidator(root)
result, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte(`"value"`)}, domain.OutputContract{
ValidationMode: domain.ValidationJSONSchema,
SchemaPath: "root.json",
})
if tc.wantError == "" {
if err != nil || !result.IsValid {
t.Fatalf("expected contained reference to validate, got result=%#v error=%v", result, err)
}
return
}
if err == nil || !strings.Contains(err.Error(), tc.wantError) {
t.Fatalf("expected error containing %q, got %v", tc.wantError, err)
}
})
}
}
func TestFSValidatorJSONSchemaReferenceBoundaries(t *testing.T) {
tests := []struct {
name string
reference string
wantError string
}{
{name: "same document fragment", reference: "#/$defs/value"},
{name: "contained relative reference", reference: "child.json"},
{name: "remote reference", reference: "https://example.test/schema.json", wantError: "is not allowed"},
{name: "escaping reference", reference: "../outside.json", wantError: "escapes source root"},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
rootSchema := `{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$defs": {"value": {"type": "string"}},
"$ref": ` + strconv.Quote(tc.reference) + `
}`
v := NewFSValidator(fstest.MapFS{
"schemas/root.json": &fstest.MapFile{Data: []byte(rootSchema)},
"schemas/child.json": &fstest.MapFile{Data: []byte(`{"type":"string"}`)},
"outside.json": &fstest.MapFile{Data: []byte(`{"type":"string"}`)},
}, "schemas")
result, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte(`"value"`)}, domain.OutputContract{
ValidationMode: domain.ValidationJSONSchema,
SchemaPath: "root.json",
})
if tc.wantError == "" {
if err != nil || !result.IsValid {
t.Fatalf("expected supported reference to validate, got result=%#v error=%v", result, err)
}
return
}
if err == nil || !strings.Contains(err.Error(), tc.wantError) {
t.Fatalf("expected error containing %q, got %v", tc.wantError, err)
}
})
}
}
func TestJSONSchemaDialectIsDraft2020(t *testing.T) {
tests := []struct {
name string
dialect string
wantError bool
}{
{name: "omitted uses supported default"},
{name: "draft 2020-12", dialect: "https://json-schema.org/draft/2020-12/schema"},
{name: "draft 7 rejected", dialect: "http://json-schema.org/draft-07/schema#", wantError: true},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
schema := map[string]any{"type": "object"}
if tc.dialect != "" {
schema["$schema"] = tc.dialect
}
data, err := json.Marshal(schema)
if err != nil {
t.Fatal(err)
}
v := NewFSValidator(fstest.MapFS{
"schema.json": &fstest.MapFile{Data: data},
}, ".")
_, err = v.Validate(context.Background(), &domain.Artifact{Body: []byte(`{}`)}, domain.OutputContract{
ValidationMode: domain.ValidationJSONSchema,
SchemaPath: "schema.json",
})
if tc.wantError {
if err == nil || !strings.Contains(err.Error(), "unsupported JSON Schema dialect") {
t.Fatalf("expected unsupported-dialect error, got %v", err)
}
return
}
if err != nil {
t.Fatalf("expected supported dialect, got %v", err)
}
})
}
}
func assertSchemaDocument(t *testing.T, got any, expectedJSON []byte) {
t.Helper()
var expected any
if err := json.Unmarshal(expectedJSON, &expected); err != nil {
t.Fatalf("decode expected schema document: %v", err)
}
if !reflect.DeepEqual(got, expected) {
t.Fatalf("schema document mismatch:\n got: %#v\nwant: %#v", got, expected)
}
}

View File

@@ -2,6 +2,7 @@ package validate
import (
"context"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
)
@@ -10,6 +11,20 @@ type Validator interface {
Validate(ctx context.Context, artifact *domain.Artifact, contract domain.OutputContract) (domain.ValidationResult, error)
}
// PreparedValidation validates artifacts against one frozen output contract.
type PreparedValidation interface {
Validate(ctx context.Context, artifact *domain.Artifact) (domain.ValidationResult, error)
// SchemaDocument returns the root JSON Schema document used for provider
// structured output, or nil for non-schema modes. Returned internal
// immutable state must not be mutated.
SchemaDocument() any
}
// ValidationPreparer freezes validation resources for one output contract.
type ValidationPreparer interface {
PrepareValidation(ctx context.Context, contract domain.OutputContract) (PreparedValidation, error)
}
// SchemaDocumentLoader loads JSON schema documents using validator path semantics.
type SchemaDocumentLoader interface {
LoadSchemaDocument(ctx context.Context, schemaPath string) (any, error)

162
json.go Normal file
View File

@@ -0,0 +1,162 @@
package promptkit
import (
"encoding/json"
"time"
)
// MarshalJSON implements json.Marshaler for PreparedRun. It uses RFC 3339
// timestamps, integer duration_ms, and omits zero timing values.
func (r PreparedRun) MarshalJSON() ([]byte, error) {
var startTime, endTime *time.Time
if !r.StartTime.IsZero() {
startTime = &r.StartTime
}
if !r.EndTime.IsZero() {
endTime = &r.EndTime
}
var durationMS *int64
if r.DurationMS != 0 {
durationMS = &r.DurationMS
}
return json.Marshal(struct {
PromptID string `json:"prompt_id"`
PromptVersion string `json:"prompt_version,omitempty"`
PromptHash string `json:"prompt_hash,omitempty"`
SelectedProfileID string `json:"selected_profile_id"`
SelectedBackendID string `json:"selected_backend_id,omitempty"`
EffectiveModelParams ExecutionTarget `json:"effective_model_params"`
OutputContract OutputContract `json:"output_contract"`
StructuredOutput *StructuredOutputSpec `json:"structured_output,omitempty"`
InputHashes map[string]string `json:"input_hashes,omitempty"`
SessionID string `json:"session_id,omitempty"`
RenderedPromptHash string `json:"rendered_prompt_hash"`
Messages []RenderedMessage `json:"messages"`
StartTime *time.Time `json:"start_time,omitempty"`
EndTime *time.Time `json:"end_time,omitempty"`
DurationMS *int64 `json:"duration_ms,omitempty"`
}{
PromptID: r.PromptID,
PromptVersion: r.PromptVersion,
PromptHash: r.PromptHash,
SelectedProfileID: r.SelectedProfileID,
SelectedBackendID: r.SelectedBackendID,
EffectiveModelParams: r.EffectiveModelParams,
OutputContract: r.OutputContract,
StructuredOutput: r.StructuredOutput,
InputHashes: r.InputHashes,
SessionID: r.SessionID,
RenderedPromptHash: r.RenderedPromptHash,
Messages: r.Messages,
StartTime: startTime,
EndTime: endTime,
DurationMS: durationMS,
})
}
// MarshalJSON implements json.Marshaler for RunResult. It encodes Duration as
// integer milliseconds in duration_ms and omits zero timing values.
func (r RunResult) MarshalJSON() ([]byte, error) {
var startTime, endTime *time.Time
if !r.StartTime.IsZero() {
startTime = &r.StartTime
}
if !r.EndTime.IsZero() {
endTime = &r.EndTime
}
var durationMS *int64
if r.Duration != 0 {
value := r.Duration.Milliseconds()
durationMS = &value
}
return json.Marshal(runResultJSON{
RunID: r.RunID,
Artifact: r.Artifact,
RawOutput: r.RawOutput,
Validation: r.Validation,
PromptID: r.PromptID,
PromptVersion: r.PromptVersion,
PromptHash: r.PromptHash,
SessionID: r.SessionID,
RenderedPromptHash: r.RenderedPromptHash,
SelectedProfileID: r.SelectedProfileID,
SelectedBackendID: r.SelectedBackendID,
ModelName: r.ModelName,
Endpoint: r.Endpoint,
EffectiveModelParams: r.EffectiveModelParams,
InputHashes: r.InputHashes,
Usage: r.Usage,
StartTime: startTime,
EndTime: endTime,
DurationMS: durationMS,
})
}
// UnmarshalJSON implements json.Unmarshaler for RunResult. It decodes
// duration_ms into Duration with millisecond precision.
func (r *RunResult) UnmarshalJSON(data []byte) error {
var wire runResultJSON
if err := json.Unmarshal(data, &wire); err != nil {
return err
}
*r = RunResult{
RunID: wire.RunID,
Artifact: wire.Artifact,
RawOutput: wire.RawOutput,
Validation: wire.Validation,
PromptID: wire.PromptID,
PromptVersion: wire.PromptVersion,
PromptHash: wire.PromptHash,
SessionID: wire.SessionID,
RenderedPromptHash: wire.RenderedPromptHash,
SelectedProfileID: wire.SelectedProfileID,
SelectedBackendID: wire.SelectedBackendID,
ModelName: wire.ModelName,
Endpoint: wire.Endpoint,
EffectiveModelParams: wire.EffectiveModelParams,
InputHashes: wire.InputHashes,
Usage: wire.Usage,
Duration: time.Duration(valueOrZero(wire.DurationMS)) * time.Millisecond,
}
if wire.StartTime != nil {
r.StartTime = *wire.StartTime
}
if wire.EndTime != nil {
r.EndTime = *wire.EndTime
}
return nil
}
type runResultJSON struct {
RunID string `json:"run_id"`
Artifact Artifact `json:"artifact"`
RawOutput string `json:"raw_output"`
Validation ValidationResult `json:"validation"`
PromptID string `json:"prompt_id"`
PromptVersion string `json:"prompt_version,omitempty"`
PromptHash string `json:"prompt_hash,omitempty"`
SessionID string `json:"session_id,omitempty"`
RenderedPromptHash string `json:"rendered_prompt_hash"`
SelectedProfileID string `json:"selected_profile_id"`
SelectedBackendID string `json:"selected_backend_id,omitempty"`
ModelName string `json:"model_name"`
Endpoint string `json:"endpoint"`
EffectiveModelParams ExecutionTarget `json:"effective_model_params"`
InputHashes map[string]string `json:"input_hashes,omitempty"`
Usage TokenUsage `json:"usage"`
StartTime *time.Time `json:"start_time,omitempty"`
EndTime *time.Time `json:"end_time,omitempty"`
DurationMS *int64 `json:"duration_ms,omitempty"`
}
func valueOrZero(value *int64) int64 {
if value == nil {
return 0
}
return *value
}

55
prepared_execution.go Normal file
View File

@@ -0,0 +1,55 @@
package promptkit
import "gitea.maximumdirect.net/eric/promptkit/internal/usecase"
const preparedExecutionString = "promptkit.PreparedExecution{opaque}"
// PreparedExecution is an opaque, in-process handle for one completely
// prepared execution. A handle is bound to the [Engine] that created it and
// permits one [Engine.RunPrepared] invocation.
//
// PreparedExecution contains no supported serializable state and cannot be
// used as a restartable job. Copying the value preserves the same shared
// lifecycle; it does not create another execution attempt.
type PreparedExecution struct {
internal *usecase.PreparedExecution
}
// Details returns a fresh caller-owned, credential-redacted copy of the
// prepared request details. Mutating the result cannot affect execution or a
// later Details call. Details remains available after execution or discard.
//
// A nil receiver or zero-value PreparedExecution returns a zero [PreparedRun].
func (p *PreparedExecution) Details() PreparedRun {
if p == nil || p.internal == nil {
return PreparedRun{}
}
details := fromDomainPreparedRun(p.internal.Details())
if details == nil {
return PreparedRun{}
}
return *details
}
// Discard invalidates an unclaimed handle and drops Promptkit's references to
// its execution-only state. Discard is nil-safe and idempotent. It does not
// cancel an execution that has already claimed the handle; use the
// [Engine.RunPrepared] context for cancellation.
func (p *PreparedExecution) Discard() {
if p == nil || p.internal == nil {
return
}
p.internal.Discard()
}
// String returns a constant representation that exposes no retained request,
// rendered content, or credential data.
func (p *PreparedExecution) String() string {
return preparedExecutionString
}
// GoString returns a constant Go-syntax representation that exposes no
// retained request, rendered content, or credential data.
func (p *PreparedExecution) GoString() string {
return preparedExecutionString
}

View File

@@ -0,0 +1,773 @@
package promptkit_test
import (
"context"
"encoding/json"
"errors"
"fmt"
"os"
"reflect"
"strings"
"sync"
"testing"
"testing/fstest"
"time"
"gitea.maximumdirect.net/eric/promptkit"
)
func TestPreparedExecutionFreezesSourcesAndReturnsIndependentDetails(t *testing.T) {
promptSource := preparedPromptSource("original")
profileSource := preparedProfileSource("original-model")
schemaSource := preparedSchemaSource()
reader := &mutablePreparedArtifactReader{
body: "original artifact",
hash: "original-input-hash",
}
client := &preparedRecordingClient{
response: &promptkit.GenerateResponse{Content: `{"value":3}`},
}
engine, err := promptkit.NewEngine(
promptkit.Config{},
promptkit.WithPromptFS(promptSource, "."),
promptkit.WithProfileFS(profileSource, "."),
promptkit.WithSchemaFS(schemaSource, "."),
promptkit.WithArtifactReader(reader),
promptkit.WithLLMClient(client),
)
if err != nil {
t.Fatalf("construct engine: %v", err)
}
temperature := 0.25
extraParams := map[string]any{
"nested": map[string]any{"source": "original"},
}
request := promptkit.RunRequest{
PromptID: "prepared",
Inputs: map[string]promptkit.ArtifactRef{
"input": promptkit.Inline("original request input"),
},
Vars: map[string]string{"label": "original variable"},
Execution: &promptkit.ExecutionTargetOverride{
Temperature: &temperature,
ExtraParams: extraParams,
},
}
preparationContext, cancelPreparation := context.WithCancel(context.Background())
prepared, err := engine.PrepareExecution(preparationContext, request)
if err != nil {
t.Fatalf("prepare execution: %v", err)
}
cancelPreparation()
request.PromptID = "changed"
request.Inputs["input"] = promptkit.Inline("changed request input")
request.Vars["label"] = "changed variable"
temperature = 1.5
extraParams["nested"].(map[string]any)["source"] = "changed"
promptSource["prompt.yaml"] = &fstest.MapFile{Data: []byte(`id: changed`)}
profileSource["profile.yaml"] = &fstest.MapFile{Data: []byte(`id: changed`)}
schemaSource["schema.json"] = &fstest.MapFile{Data: []byte(`{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"title": "changed root",
"type": "string"
}`)}
schemaSource["value.json"] = &fstest.MapFile{Data: []byte(`{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"type": "string"
}`)}
reader.set("changed artifact", "changed-input-hash")
first := prepared.Details()
first.Messages[0].Content = "changed details"
first.InputHashes["input"] = "changed-details-hash"
first.EffectiveModelParams.ExtraParams["nested"].(map[string]any)["source"] = "changed details"
first.StructuredOutput.JSONSchema.Schema.(map[string]any)["title"] = "changed details"
second := prepared.Details()
if second.Messages[0].Content != "Input=original artifact Label=original variable" {
t.Fatalf("details message changed: %q", second.Messages[0].Content)
}
if second.InputHashes["input"] != "original-input-hash" {
t.Fatalf("details input hash changed: %q", second.InputHashes["input"])
}
if second.EffectiveModelParams.Model != "original-model" ||
second.EffectiveModelParams.Temperature != 0.25 ||
second.EffectiveModelParams.ExtraParams["nested"].(map[string]any)["source"] != "original" {
t.Fatalf("details target changed: %+v", second.EffectiveModelParams)
}
schema := second.StructuredOutput.JSONSchema.Schema.(map[string]any)
if schema["title"] != "original root" {
t.Fatalf("details schema changed: %#v", schema)
}
result, err := engine.RunPrepared(context.Background(), prepared)
if err != nil {
t.Fatalf("run prepared after preparation-context cancellation: %v", err)
}
if result.Validation.Status != promptkit.ValidationPassed || !result.Validation.IsValid {
t.Fatalf("frozen schema did not validate original output: %+v", result.Validation)
}
if reader.callCount() != 1 {
t.Fatalf("execution reopened artifact source: calls=%d", reader.callCount())
}
requests := client.snapshot()
if len(requests) != 1 {
t.Fatalf("generation calls=%d, want 1", len(requests))
}
generated := requests[0]
if generated.Prompt.Messages[0].Content != second.Messages[0].Content ||
generated.Target.Model != second.EffectiveModelParams.Model ||
!reflect.DeepEqual(generated.Target.ExtraParams, second.EffectiveModelParams.ExtraParams) ||
!reflect.DeepEqual(generated.StructuredOutput, second.StructuredOutput) {
t.Fatalf("generation did not use frozen details:\nrequest=%+v\ndetails=%+v", generated, second)
}
if result.PromptID != second.PromptID ||
result.PromptVersion != second.PromptVersion ||
result.PromptHash != second.PromptHash ||
result.SessionID != second.SessionID ||
result.RenderedPromptHash != second.RenderedPromptHash ||
result.SelectedProfileID != second.SelectedProfileID ||
result.SelectedBackendID != second.SelectedBackendID ||
!reflect.DeepEqual(result.EffectiveModelParams, second.EffectiveModelParams) ||
!reflect.DeepEqual(result.InputHashes, second.InputHashes) {
t.Fatalf("result provenance does not match details:\nresult=%+v\ndetails=%+v", result, second)
}
}
func TestPreparedExecutionLifecycleAndEngineBinding(t *testing.T) {
ownerClient := &preparedRecordingClient{
response: &promptkit.GenerateResponse{Content: "ok"},
}
owner := newPreparedContractEngine(t, ownerClient, "owner content")
foreign := newPreparedContractEngine(t, &preparedRecordingClient{
response: &promptkit.GenerateResponse{Content: "unexpected"},
}, "foreign content")
prepared, err := owner.PrepareExecution(context.Background(), promptkit.RunRequest{PromptID: "prepared"})
if err != nil {
t.Fatalf("prepare execution: %v", err)
}
copied := *prepared
var nilEngine *promptkit.Engine
if result, err := nilEngine.RunPrepared(context.Background(), prepared); result != nil ||
!errors.Is(err, promptkit.ErrInvalidConfig) {
t.Fatalf("nil engine result=(%+v, %v), want ErrInvalidConfig", result, err)
}
if result, err := foreign.RunPrepared(context.Background(), prepared); result != nil ||
!errors.Is(err, promptkit.ErrInvalidRequest) {
t.Fatalf("foreign engine result=(%+v, %v), want ErrInvalidRequest", result, err)
}
if result, err := owner.RunPrepared(context.Background(), nil); result != nil ||
!errors.Is(err, promptkit.ErrInvalidRequest) {
t.Fatalf("nil handle result=(%+v, %v), want ErrInvalidRequest", result, err)
}
if result, err := owner.RunPrepared(context.Background(), &promptkit.PreparedExecution{}); result != nil ||
!errors.Is(err, promptkit.ErrInvalidRequest) {
t.Fatalf("zero handle result=(%+v, %v), want ErrInvalidRequest", result, err)
}
result, err := owner.RunPrepared(context.Background(), &copied)
if err != nil || result == nil {
t.Fatalf("owner run prepared=(%+v, %v), want success", result, err)
}
for name, handle := range map[string]*promptkit.PreparedExecution{
"original": prepared,
"copy": &copied,
} {
if result, err := owner.RunPrepared(context.Background(), handle); result != nil ||
!errors.Is(err, promptkit.ErrInvalidRequest) {
t.Fatalf("%s reused handle result=(%+v, %v), want ErrInvalidRequest", name, result, err)
}
if handle.Details().PromptID != "prepared" {
t.Fatalf("%s details unavailable after execution", name)
}
}
if len(ownerClient.snapshot()) != 1 {
t.Fatalf("owner generation calls=%d, want 1", len(ownerClient.snapshot()))
}
collaboratorFailure := errors.New("prepared collaborator failure")
failingClient := &preparedRecordingClient{err: collaboratorFailure}
failingEngine := newPreparedContractEngine(t, failingClient, "failure content")
failing, err := failingEngine.PrepareExecution(context.Background(), promptkit.RunRequest{PromptID: "prepared"})
if err != nil {
t.Fatalf("prepare failing execution: %v", err)
}
if result, err := failingEngine.RunPrepared(context.Background(), failing); result != nil ||
!errors.Is(err, promptkit.ErrLLMGenerate) ||
!errors.Is(err, collaboratorFailure) {
t.Fatalf("generation failure result=(%+v, %v), want public and collaborator identities", result, err)
}
if result, err := failingEngine.RunPrepared(context.Background(), failing); result != nil ||
!errors.Is(err, promptkit.ErrInvalidRequest) {
t.Fatalf("failed execution was reusable: result=(%+v, %v)", result, err)
}
cancellationRelease := make(chan struct{})
cancellationStarted := make(chan struct{}, 1)
cancelingEngine := newPreparedContractEngine(t, &preparedRecordingClient{
response: &promptkit.GenerateResponse{Content: "unexpected"},
started: cancellationStarted,
release: cancellationRelease,
}, "cancellation content")
canceling, err := cancelingEngine.PrepareExecution(
context.Background(),
promptkit.RunRequest{PromptID: "prepared"},
)
if err != nil {
t.Fatalf("prepare canceled execution: %v", err)
}
executionContext, cancelExecution := context.WithCancel(context.Background())
type canceledOutcome struct {
result *promptkit.RunResult
err error
}
canceledResult := make(chan canceledOutcome, 1)
go func() {
result, runErr := cancelingEngine.RunPrepared(executionContext, canceling)
canceledResult <- canceledOutcome{result: result, err: runErr}
}()
select {
case <-cancellationStarted:
case <-time.After(5 * time.Second):
t.Fatal("timed out waiting for cancelable generation")
}
cancelExecution()
select {
case outcome := <-canceledResult:
if outcome.result != nil ||
!errors.Is(outcome.err, promptkit.ErrLLMGenerate) ||
!errors.Is(outcome.err, context.Canceled) {
t.Fatalf(
"canceled execution=(%+v, %v), want generation and context identities",
outcome.result,
outcome.err,
)
}
case <-time.After(5 * time.Second):
t.Fatal("timed out waiting for canceled execution")
}
if result, err := cancelingEngine.RunPrepared(context.Background(), canceling); result != nil ||
!errors.Is(err, promptkit.ErrInvalidRequest) {
t.Fatalf("canceled execution was reusable: result=(%+v, %v)", result, err)
}
}
func TestPreparedExecutionConcurrentClaimAllowsOneGeneration(t *testing.T) {
release := make(chan struct{})
client := &preparedRecordingClient{
response: &promptkit.GenerateResponse{Content: "ok"},
started: make(chan struct{}, 1),
release: release,
}
engine := newPreparedContractEngine(t, client, "concurrent content")
prepared, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{PromptID: "prepared"})
if err != nil {
t.Fatalf("prepare execution: %v", err)
}
type outcome struct {
result *promptkit.RunResult
err error
}
outcomes := make(chan outcome, 2)
for i := 0; i < 2; i++ {
go func() {
result, runErr := engine.RunPrepared(context.Background(), prepared)
outcomes <- outcome{result: result, err: runErr}
}()
}
select {
case <-client.started:
case <-time.After(5 * time.Second):
t.Fatal("timed out waiting for generation")
}
select {
case loser := <-outcomes:
if loser.result != nil || !errors.Is(loser.err, promptkit.ErrInvalidRequest) {
t.Fatalf("concurrent loser=(%+v, %v), want ErrInvalidRequest", loser.result, loser.err)
}
case <-time.After(5 * time.Second):
t.Fatal("timed out waiting for rejected concurrent claim")
}
close(release)
select {
case winner := <-outcomes:
if winner.err != nil || winner.result == nil {
t.Fatalf("concurrent winner=(%+v, %v), want success", winner.result, winner.err)
}
case <-time.After(5 * time.Second):
t.Fatal("timed out waiting for successful concurrent claim")
}
if len(client.snapshot()) != 1 {
t.Fatalf("generation calls=%d, want 1", len(client.snapshot()))
}
}
func TestPreparedExecutionRunAndDiscardRaceHasOneWinner(t *testing.T) {
const attempts = 32
for i := 0; i < attempts; i++ {
client := &preparedRecordingClient{
response: &promptkit.GenerateResponse{Content: "ok"},
}
engine := newPreparedContractEngine(t, client, "race content")
prepared, err := engine.PrepareExecution(
context.Background(),
promptkit.RunRequest{PromptID: "prepared"},
)
if err != nil {
t.Fatalf("attempt %d prepare execution: %v", i, err)
}
start := make(chan struct{})
type outcome struct {
result *promptkit.RunResult
err error
}
runOutcome := make(chan outcome, 1)
discardDone := make(chan struct{})
go func() {
<-start
result, runErr := engine.RunPrepared(context.Background(), prepared)
runOutcome <- outcome{result: result, err: runErr}
}()
go func() {
<-start
prepared.Discard()
close(discardDone)
}()
close(start)
runResult := <-runOutcome
<-discardDone
calls := len(client.snapshot())
switch {
case runResult.err == nil:
if runResult.result == nil || calls != 1 {
t.Fatalf(
"attempt %d run won with outcome=(%+v, %v), generation calls=%d",
i,
runResult.result,
runResult.err,
calls,
)
}
case errors.Is(runResult.err, promptkit.ErrInvalidRequest):
if runResult.result != nil || calls != 0 {
t.Fatalf(
"attempt %d discard won with outcome=(%+v, %v), generation calls=%d",
i,
runResult.result,
runResult.err,
calls,
)
}
default:
t.Fatalf("attempt %d unexpected run outcome=(%+v, %v)", i, runResult.result, runResult.err)
}
if prepared.Details().PromptID != "prepared" {
t.Fatalf("attempt %d details unavailable after race", i)
}
}
}
func TestPreparedExecutionDiscardAndFormattingDoNotExposePrivateState(t *testing.T) {
const (
directCredential = "pk-test-direct-credential-41f7"
renderedContent = "rendered-content-sentinel-98d2"
)
client := &preparedRecordingClient{
response: &promptkit.GenerateResponse{Content: "generated output"},
}
engine, err := promptkit.NewEngine(
promptkit.Config{},
promptkit.WithPromptFS(contractPromptFS("prepared", "profile", renderedContent), "."),
promptkit.WithProfiles(promptkit.Profile{
ID: "profile",
Endpoint: "http://example.test/v1",
Model: "model",
APIKeyRequired: true,
}),
promptkit.WithLLMClient(client),
)
if err != nil {
t.Fatalf("construct engine: %v", err)
}
prepared, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{
PromptID: "prepared",
APIKey: directCredential,
})
if err != nil {
t.Fatalf("prepare execution: %v", err)
}
formattedValues := []string{
fmt.Sprint(prepared),
fmt.Sprintf("%+v", prepared),
fmt.Sprintf("%#v", prepared),
}
for _, formatted := range formattedValues {
if formatted != "promptkit.PreparedExecution{opaque}" {
t.Fatalf("unexpected opaque formatting: %q", formatted)
}
assertPreparedPrivateValuesAbsent(t, formatted, directCredential, renderedContent)
}
payload, err := json.Marshal(prepared)
if err != nil {
t.Fatalf("marshal opaque handle: %v", err)
}
assertPreparedPrivateValuesAbsent(t, string(payload), directCredential, renderedContent)
detailsBefore := prepared.Details()
detailsJSON, err := json.Marshal(detailsBefore)
if err != nil {
t.Fatalf("marshal prepared details: %v", err)
}
assertPreparedPrivateValuesAbsent(t, string(detailsJSON), directCredential)
prepared.Discard()
prepared.Discard()
result, lifecycleErr := engine.RunPrepared(context.Background(), prepared)
if result != nil || !errors.Is(lifecycleErr, promptkit.ErrInvalidRequest) {
t.Fatalf("discarded execution result=(%+v, %v), want ErrInvalidRequest", result, lifecycleErr)
}
assertPreparedPrivateValuesAbsent(t, lifecycleErr.Error(), directCredential, renderedContent)
if !reflect.DeepEqual(prepared.Details(), detailsBefore) {
t.Fatal("details changed after discard")
}
executed, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{
PromptID: "prepared",
APIKey: directCredential,
})
if err != nil {
t.Fatalf("prepare execution for request inspection: %v", err)
}
executionResult, err := engine.RunPrepared(context.Background(), executed)
if err != nil {
t.Fatalf("run execution for request inspection: %v", err)
}
requests := client.snapshot()
if len(requests) != 1 || requests[0].APIKey != directCredential {
t.Fatalf("direct credential did not reach only the client credential field: %#v", requests)
}
requestJSON, err := json.Marshal(requests[0])
if err != nil {
t.Fatalf("marshal captured generate request: %v", err)
}
for _, value := range []string{
fmt.Sprint(requests[0]),
fmt.Sprintf("%+v", requests[0]),
fmt.Sprintf("%#v", requests[0]),
string(requestJSON),
fmt.Sprint(executionResult),
} {
assertPreparedPrivateValuesAbsent(t, value, directCredential)
}
resultJSON, err := json.Marshal(executionResult)
if err != nil {
t.Fatalf("marshal execution result: %v", err)
}
assertPreparedPrivateValuesAbsent(t, string(resultJSON), directCredential)
var nilHandle *promptkit.PreparedExecution
nilHandle.Discard()
if !reflect.DeepEqual(nilHandle.Details(), promptkit.PreparedRun{}) {
t.Fatalf("nil handle details=%+v, want zero value", nilHandle.Details())
}
zeroHandle := &promptkit.PreparedExecution{}
zeroHandle.Discard()
if !reflect.DeepEqual(zeroHandle.Details(), promptkit.PreparedRun{}) {
t.Fatalf("zero handle details=%+v, want zero value", zeroHandle.Details())
}
}
func TestPreparedExecutionCredentialCapacityAndTimingBoundaries(t *testing.T) {
t.Run("credential is rechecked before generation", func(t *testing.T) {
const (
environmentName = "PROMPTKIT_PREPARED_CONTRACT_KEY"
environmentKey = "environment-credential-sentinel"
)
t.Setenv(environmentName, environmentKey)
client := &preparedRecordingClient{
response: &promptkit.GenerateResponse{Content: "unexpected"},
}
engine, err := promptkit.NewEngine(
promptkit.Config{},
promptkit.WithPromptFS(contractPromptFS("prepared", "profile", "content"), "."),
promptkit.WithProfileFS(preparedCredentialProfileSource(environmentName), "."),
promptkit.WithLLMClient(client),
)
if err != nil {
t.Fatalf("construct credential engine: %v", err)
}
prepared, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{PromptID: "prepared"})
if err != nil {
t.Fatalf("prepare credential execution: %v", err)
}
if err := os.Unsetenv(environmentName); err != nil {
t.Fatalf("unset credential environment: %v", err)
}
result, err := engine.RunPrepared(context.Background(), prepared)
if result != nil ||
!errors.Is(err, promptkit.ErrInvalidRequest) ||
!errors.Is(err, promptkit.ErrAPIKeyEnvMissing) {
t.Fatalf("credential execution=(%+v, %v), want credential identities", result, err)
}
if len(client.snapshot()) != 0 {
t.Fatalf("credential failure reached generation: %d calls", len(client.snapshot()))
}
assertPreparedPrivateValuesAbsent(t, err.Error(), environmentKey)
if result, err := engine.RunPrepared(context.Background(), prepared); result != nil ||
!errors.Is(err, promptkit.ErrInvalidRequest) {
t.Fatalf("credential failure did not consume handle: result=(%+v, %v)", result, err)
}
})
t.Run("preparation does not admit and execution timing starts after retention", func(t *testing.T) {
release := make(chan struct{})
client := newCapacityGateClient(release, 4)
engine := newBackendCapacityEngine(t, client, 1, capacityInt(0), nil)
activeRun := make(chan capacityRunResult, 1)
go runCapacityRequest(
engine,
context.Background(),
promptkit.RunRequest{PromptID: "prompt"},
activeRun,
)
awaitCapacityRequest(t, client.started)
prepared, err := engine.PrepareExecution(
context.Background(),
promptkit.RunRequest{PromptID: "prompt"},
)
if err != nil {
t.Fatalf("prepare while capacity is full: %v", err)
}
if _, _, calls := client.snapshot(); calls != 1 {
t.Fatalf("preparation invoked generation: calls=%d", calls)
}
if result, err := engine.RunPrepared(context.Background(), prepared); result != nil ||
!errors.Is(err, promptkit.ErrCapacityExceeded) {
t.Fatalf("capacity execution=(%+v, %v), want ErrCapacityExceeded", result, err)
} else {
var capacityErr *promptkit.CapacityError
if !errors.As(err, &capacityErr) || capacityErr == nil || capacityErr.BackendID != "limited" {
t.Fatalf("capacity execution=%v, want limited CapacityError", err)
}
}
if result, err := engine.RunPrepared(context.Background(), prepared); result != nil ||
!errors.Is(err, promptkit.ErrInvalidRequest) {
t.Fatalf("capacity rejection did not consume handle: result=(%+v, %v)", result, err)
}
if prepared.Details().PromptID != "prompt" {
t.Fatal("details unavailable after capacity rejection")
}
close(release)
activeOutcome := awaitCapacityRun(t, activeRun)
if activeOutcome.err != nil || activeOutcome.result == nil {
t.Fatalf("active run outcome=(%+v, %v), want success", activeOutcome.result, activeOutcome.err)
}
timed, err := engine.PrepareExecution(
context.Background(),
promptkit.RunRequest{PromptID: "prompt"},
)
if err != nil {
t.Fatalf("prepare timed execution: %v", err)
}
details := timed.Details()
time.Sleep(25 * time.Millisecond)
executionFloor := time.Now().UTC()
result, err := engine.RunPrepared(context.Background(), timed)
if err != nil {
t.Fatalf("run timed execution: %v", err)
}
if result.StartTime.Before(executionFloor) ||
!result.StartTime.After(details.EndTime) ||
result.EndTime.Before(result.StartTime) ||
result.Duration != result.EndTime.Sub(result.StartTime) {
t.Fatalf(
"execution timing includes preparation or retention: details_end=%s floor=%s result=%+v",
details.EndTime,
executionFloor,
result,
)
}
if _, _, calls := client.snapshot(); calls != 2 {
t.Fatalf("generation calls=%d, want active and timed executions only", calls)
}
})
}
type mutablePreparedArtifactReader struct {
mu sync.Mutex
body string
hash string
calls int
}
func (r *mutablePreparedArtifactReader) Read(
_ context.Context,
_ promptkit.ArtifactRef,
) (*promptkit.Artifact, error) {
r.mu.Lock()
defer r.mu.Unlock()
r.calls++
return &promptkit.Artifact{
Body: []byte(r.body),
Hash: r.hash,
}, nil
}
func (r *mutablePreparedArtifactReader) set(body, hash string) {
r.mu.Lock()
defer r.mu.Unlock()
r.body = body
r.hash = hash
}
func (r *mutablePreparedArtifactReader) callCount() int {
r.mu.Lock()
defer r.mu.Unlock()
return r.calls
}
type preparedRecordingClient struct {
mu sync.Mutex
response *promptkit.GenerateResponse
err error
requests []promptkit.GenerateRequest
started chan struct{}
release <-chan struct{}
}
func (c *preparedRecordingClient) Generate(
ctx context.Context,
request promptkit.GenerateRequest,
) (*promptkit.GenerateResponse, error) {
c.mu.Lock()
c.requests = append(c.requests, request)
c.mu.Unlock()
if c.started != nil {
c.started <- struct{}{}
}
if c.release != nil {
select {
case <-c.release:
case <-ctx.Done():
return nil, ctx.Err()
}
}
if c.err != nil {
return nil, c.err
}
return c.response, nil
}
func (c *preparedRecordingClient) snapshot() []promptkit.GenerateRequest {
c.mu.Lock()
defer c.mu.Unlock()
return append([]promptkit.GenerateRequest(nil), c.requests...)
}
func newPreparedContractEngine(
t *testing.T,
client promptkit.LLMClient,
message string,
) *promptkit.Engine {
t.Helper()
engine, err := promptkit.NewEngine(
promptkit.Config{},
promptkit.WithPromptFS(contractPromptFS("prepared", "profile", message), "."),
promptkit.WithProfiles(promptkit.Profile{
ID: "profile",
Endpoint: "http://example.test/v1",
Model: "model",
}),
promptkit.WithLLMClient(client),
)
if err != nil {
t.Fatalf("construct prepared execution engine: %v", err)
}
return engine
}
func preparedPromptSource(label string) fstest.MapFS {
return fstest.MapFS{
"prompt.yaml": &fstest.MapFile{Data: []byte(`id: prepared
version: "1"
default_profile: profile
inputs:
- name: input
required: true
messages:
- role: user
content: 'Input={{input "input"}} Label={{.label}}'
description: ` + label + `
output:
format: json
validation_mode: json_schema
schema_path: schema.json
`)},
}
}
func preparedProfileSource(model string) fstest.MapFS {
return fstest.MapFS{
"profile.yaml": &fstest.MapFile{Data: []byte(`id: profile
endpoint: http://example.test/v1
model: ` + model + `
`)},
}
}
func preparedCredentialProfileSource(environmentName string) fstest.MapFS {
return fstest.MapFS{
"profile.yaml": &fstest.MapFile{Data: []byte(`id: profile
endpoint: http://example.test/v1
model: model
api_key_env: ` + environmentName + `
`)},
}
}
func preparedSchemaSource() fstest.MapFS {
return fstest.MapFS{
"schema.json": &fstest.MapFile{Data: []byte(`{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"title": "original root",
"type": "object",
"required": ["value"],
"properties": {
"value": {"$ref": "value.json"}
}
}`)},
"value.json": &fstest.MapFile{Data: []byte(`{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"type": "integer",
"minimum": 2
}`)},
}
}
func assertPreparedPrivateValuesAbsent(t *testing.T, value string, privateValues ...string) {
t.Helper()
for _, privateValue := range privateValues {
if strings.Contains(value, privateValue) {
t.Fatalf("value exposed private data %q: %s", privateValue, value)
}
}
}

View File

@@ -7,6 +7,7 @@ import (
"strings"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
"gitea.maximumdirect.net/eric/promptkit/internal/jsonvalue"
"gitea.maximumdirect.net/eric/promptkit/internal/profile"
)
@@ -15,10 +16,16 @@ import (
//
// It does not register global state, maintain a model catalog, or resolve
// credentials. If APIKeyRequired is true, callers satisfy it with
// RunRequest.APIKey. Raw API keys do not belong in profiles.
// RunRequest.APIKey or an explicit request ExecutionTargetOverride.APIKeyEnv.
// Raw API keys do not belong in profiles.
//
// The function copies the ExtraParams map itself but does not recursively copy
// nested values. Validation and a deep copy occur when NewEngine applies a
// WithProfiles option containing the returned Profile.
func OpenAICompatibleProfile(cfg OpenAICompatibleProfileConfig) Profile {
return Profile{
ID: cfg.ID,
BackendID: cfg.BackendID,
Endpoint: cfg.Endpoint,
Model: cfg.Model,
Temperature: cfg.Temperature,
@@ -75,12 +82,13 @@ func (r *memoryProfileRepository) GetProfile(_ context.Context, id string) (*dom
}
func toDomainProfile(publicProfile Profile) (domain.ExecutionProfile, error) {
extraParams, err := copyPublicJSONMap(publicProfile.ExtraParams)
extraParams, err := jsonvalue.CopyMap(publicProfile.ExtraParams)
if err != nil {
return domain.ExecutionProfile{}, err
}
prof := domain.ExecutionProfile{
ID: strings.TrimSpace(publicProfile.ID),
BackendID: strings.TrimSpace(publicProfile.BackendID),
Endpoint: publicProfile.Endpoint,
Model: publicProfile.Model,
Temperature: publicProfile.Temperature,
@@ -102,8 +110,8 @@ func validatePublicProfile(prof domain.ExecutionProfile) error {
if strings.TrimSpace(prof.ID) == "" {
return errors.New("id is required")
}
if strings.TrimSpace(prof.Endpoint) == "" {
return errors.New("endpoint is required")
if strings.TrimSpace(prof.BackendID) == "" && strings.TrimSpace(prof.Endpoint) == "" {
return errors.New("backend or endpoint is required")
}
if strings.TrimSpace(prof.Model) == "" {
return errors.New("model is required")

1411
public_contract_test.go Normal file

File diff suppressed because it is too large Load Diff

734
types.go
View File

@@ -5,302 +5,714 @@ import (
"time"
)
// ArtifactRefType defines how an artifact is referenced.
// ArtifactRefType identifies how an [ArtifactRef] supplies content.
type ArtifactRefType string
const (
// ArtifactRefInline selects ArtifactRef.Body as the content.
ArtifactRefInline ArtifactRefType = "inline"
ArtifactRefFile ArtifactRefType = "file"
// ArtifactRefFile selects the filesystem path in ArtifactRef.URI.
ArtifactRefFile ArtifactRefType = "file"
)
// OutputFormat defines the desired output format.
// OutputFormat identifies the media format of generated output.
// OutputFormat has a stable JSON string representation.
type OutputFormat string
const (
FormatText OutputFormat = "text"
// FormatText identifies plain-text output.
FormatText OutputFormat = "text"
// FormatMarkdown identifies Markdown output.
FormatMarkdown OutputFormat = "markdown"
FormatJSON OutputFormat = "json"
// FormatJSON identifies JSON output.
FormatJSON OutputFormat = "json"
)
// ValidationMode defines the output validation strategy.
// ValidationMode identifies how generated output is checked.
// ValidationMode has a stable JSON string representation.
type ValidationMode string
const (
ValidationNone ValidationMode = "none"
ValidationBasic ValidationMode = "basic"
ValidationJSON ValidationMode = "json"
// ValidationNone skips content validation.
ValidationNone ValidationMode = "none"
// ValidationBasic requires non-empty output.
ValidationBasic ValidationMode = "basic"
// ValidationJSON requires syntactically valid JSON.
ValidationJSON ValidationMode = "json"
// ValidationJSONSchema requires JSON that satisfies OutputContract.SchemaPath.
ValidationJSONSchema ValidationMode = "json_schema"
)
// ValidationStatus defines the result of a validation check.
// ValidationStatus identifies the completed state of an output check.
// ValidationStatus has a stable JSON string representation.
type ValidationStatus string
const (
ValidationPassed ValidationStatus = "passed"
ValidationFailed ValidationStatus = "failed"
// ValidationPassed means the generated output satisfied its contract.
ValidationPassed ValidationStatus = "passed"
// ValidationFailed means validation completed and rejected the generated
// output. Engine.Run and Engine.RunPrepared return this status in a result,
// not as an error.
ValidationFailed ValidationStatus = "failed"
// ValidationSkipped means ValidationNone selected no content check.
ValidationSkipped ValidationStatus = "skipped"
)
// CacheControlType defines provider cache behavior for prompt content.
// CacheControlType identifies provider cache behavior for prompt content.
// CacheControlType has a stable JSON string representation.
type CacheControlType string
const (
// CacheControlEphemeral requests provider-defined ephemeral caching.
CacheControlEphemeral CacheControlType = "ephemeral"
)
// StructuredOutputType identifies provider-level structured output modes.
// StructuredOutputType has a stable JSON string representation.
type StructuredOutputType string
const (
// StructuredOutputJSONSchema supplies JSON Schema response constraints.
StructuredOutputJSONSchema StructuredOutputType = "json_schema"
)
// RunRequest represents a request to prepare or run a single prompt.
// RunRequest selects one prompt execution. It has no stable JSON
// representation.
//
// Prepare, PrepareExecution, and Run copy the request's maps, pointers, and
// nested JSON-compatible values before using them. The caller may mutate the
// request after any method returns. A successful PrepareExecution retains its
// own private execution snapshot for RunPrepared.
type RunRequest struct {
PromptID string
// PromptID is the required non-empty prompt identifier.
PromptID string
// PromptVersion optionally selects one version of PromptID. When empty, the
// prompt source must contain exactly one matching version.
PromptVersion string
ProfileID string
APIKey string `json:"-"`
Inputs map[string]ArtifactRef
Vars map[string]string
Execution *ExecutionTargetOverride
Validation *OutputContract
Metadata map[string]string
// ProfileID selects an execution profile. When empty, the prompt's default
// profile is used; if both are empty, the error matches ErrProfileRequired
// and ErrInvalidRequest.
ProfileID string
// SessionID optionally supplies a direct per-run session identifier. A
// nonblank value is trimmed and overrides the prompt definition's
// session_id template. A blank value supplies no direct override. The
// maximum is 256 Unicode code points after trimming. A direct value is
// opaque consumer metadata, not a credential, and may be exposed in
// prepared values, results, collaborator requests, provider requests, and
// provider observability. Callers should use stable, non-sensitive
// identifiers. An overlong direct value makes Prepare, PrepareExecution, or
// Run return an error matching ErrInvalidRequest.
SessionID string
// APIKey is a request-scoped direct credential. It takes precedence over
// APIKeyEnv, is passed to the selected LLMClient, and is never included in
// prepared values, results, hashes, JSON, String, or GoString output. A
// successful PrepareExecution retains it only in the opaque handle until
// RunPrepared claims the handle or Discard invalidates it.
APIKey string `json:"-"`
// Inputs maps prompt input names to references. A nil or empty map is valid
// only when the selected prompt and its templates require no inputs.
Inputs map[string]ArtifactRef
// Vars supplies Go-template data for messages and the session ID. Nil and
// empty maps are equivalent.
Vars map[string]string
// Execution optionally overrides individual execution settings. Nil uses
// the selected profile over its backend, when any, and the framework
// baseline.
Execution *ExecutionTargetOverride
// Validation optionally replaces the prompt's complete output contract. It
// does not merge individual fields. Nil uses the prompt contract.
Validation *OutputContract
}
// PreparedRun contains prepared prompt execution state. It does not include
// resolved API key values, model output, validation results, or internal target
// presence metadata.
// PreparedRun contains prepared prompt execution state returned by
// [Engine.Prepare] or [PreparedExecution.Details]. It does not include resolved
// API key values, model output, validation results, or internal target presence
// metadata. PreparedRun has a stable JSON representation.
//
// All maps, slices, pointers, and schema values are caller-owned copies. JSON
// timestamps use RFC 3339 and zero timing values are omitted. Hash formats are
// opaque.
type PreparedRun struct {
PromptID string `json:"prompt_id"`
PromptVersion string `json:"prompt_version,omitempty"`
PromptHash string `json:"prompt_hash,omitempty"`
SelectedProfileID string `json:"selected_profile_id"`
EffectiveModelParams ExecutionTarget `json:"effective_model_params"`
OutputContract OutputContract `json:"output_contract"`
StructuredOutput *StructuredOutputSpec `json:"structured_output,omitempty"`
InputHashes map[string]string `json:"input_hashes,omitempty"`
SessionID string `json:"session_id,omitempty"`
RenderedPromptHash string `json:"rendered_prompt_hash"`
Messages []RenderedMessage `json:"messages"`
StartTime time.Time `json:"start_time,omitempty"`
EndTime time.Time `json:"end_time,omitempty"`
DurationMS int64 `json:"duration_ms,omitempty"`
// PromptID is the selected prompt identifier.
PromptID string `json:"prompt_id"`
// PromptVersion is the selected prompt version.
PromptVersion string `json:"prompt_version,omitempty"`
// PromptHash is an opaque equality value for the selected definition.
PromptHash string `json:"prompt_hash,omitempty"`
// SelectedProfileID is the explicit request profile or prompt default that
// supplied execution settings.
SelectedProfileID string `json:"selected_profile_id"`
// SelectedBackendID equals EffectiveModelParams.BackendID. It is empty for
// an endpoint-only profile.
SelectedBackendID string `json:"selected_backend_id,omitempty"`
// EffectiveModelParams contains settings resolved from the framework timeout
// baseline, selected backend, profile, and then request overrides. Unset
// optional provider controls remain zero rather than reporting a provider
// default. It excludes resolved API-key values.
EffectiveModelParams ExecutionTarget `json:"effective_model_params"`
// OutputContract is the complete effective output contract.
OutputContract OutputContract `json:"output_contract"`
// StructuredOutput is non-nil for JSON Schema validation and contains the
// provider-facing response constraint passed to an LLM client.
StructuredOutput *StructuredOutputSpec `json:"structured_output,omitempty"`
// InputHashes maps every supplied input name to its opaque artifact hash.
InputHashes map[string]string `json:"input_hashes,omitempty"`
// SessionID is the effective direct or rendered session identifier, if any.
SessionID string `json:"session_id,omitempty"`
// RenderedPromptHash is an opaque equality value for SessionID and Messages.
RenderedPromptHash string `json:"rendered_prompt_hash"`
// Messages are the rendered messages that Run or RunPrepared passes to the
// LLM client.
Messages []RenderedMessage `json:"messages"`
// StartTime is the UTC time at which preparation began.
StartTime time.Time `json:"start_time,omitempty"`
// EndTime is the UTC time at which preparation completed.
EndTime time.Time `json:"end_time,omitempty"`
// DurationMS is preparation elapsed time in integer milliseconds. JSON uses
// duration_ms and omits a zero value.
DurationMS int64 `json:"duration_ms,omitempty"`
}
// RunResult contains generated output, validation state, and run metadata.
// RunResult has a stable JSON representation and round-trips its Duration
// through the duration_ms JSON field.
//
// All maps, slices, and nested values are caller-owned copies. JSON timestamps
// use RFC 3339 and zero timing values are omitted. Run IDs and hash formats are
// opaque.
type RunResult struct {
RunID string `json:"run_id"`
Artifact Artifact `json:"artifact"`
RawOutput string `json:"raw_output"`
Validation ValidationResult `json:"validation"`
PromptID string `json:"prompt_id"`
PromptVersion string `json:"prompt_version,omitempty"`
PromptHash string `json:"prompt_hash,omitempty"`
RenderedPromptHash string `json:"rendered_prompt_hash"`
SelectedProfileID string `json:"selected_profile_id"`
ModelName string `json:"model_name"`
Endpoint string `json:"endpoint"`
EffectiveModelParams ExecutionTarget `json:"effective_model_params"`
InputHashes map[string]string `json:"input_hashes,omitempty"`
Usage TokenUsage `json:"usage"`
StartTime time.Time `json:"start_time,omitempty"`
EndTime time.Time `json:"end_time,omitempty"`
Duration time.Duration `json:"duration,omitempty"`
// RunID is an opaque identifier for this invocation.
RunID string `json:"run_id"`
// Artifact contains the generated output and derived metadata.
Artifact Artifact `json:"artifact"`
// RawOutput is the exact generated content before artifact classification
// and validation.
RawOutput string `json:"raw_output"`
// Validation records the completed content check.
Validation ValidationResult `json:"validation"`
// PromptID is the selected prompt identifier.
PromptID string `json:"prompt_id"`
// PromptVersion is the selected prompt version.
PromptVersion string `json:"prompt_version,omitempty"`
// PromptHash is the same opaque definition equality value exposed by
// PreparedRun.
PromptHash string `json:"prompt_hash,omitempty"`
// SessionID is the effective direct or rendered session identifier, if any.
// JSON omits an empty value.
SessionID string `json:"session_id,omitempty"`
// RenderedPromptHash is the same opaque rendered-prompt equality value
// computed during preparation.
RenderedPromptHash string `json:"rendered_prompt_hash"`
// SelectedProfileID identifies the profile used for execution.
SelectedProfileID string `json:"selected_profile_id"`
// SelectedBackendID equals EffectiveModelParams.BackendID. It is empty for
// an endpoint-only profile.
SelectedBackendID string `json:"selected_backend_id,omitempty"`
// ModelName is the effective model name and equals
// EffectiveModelParams.Model.
ModelName string `json:"model_name"`
// Endpoint is the effective base endpoint and equals
// EffectiveModelParams.Endpoint.
Endpoint string `json:"endpoint"`
// EffectiveModelParams contains the settings supplied to the LLM client,
// excluding resolved API-key values.
EffectiveModelParams ExecutionTarget `json:"effective_model_params"`
// InputHashes are the opaque input equality values computed during
// preparation.
InputHashes map[string]string `json:"input_hashes,omitempty"`
// Usage is the token accounting reported by the LLM client.
Usage TokenUsage `json:"usage"`
// StartTime is the UTC time immediately before ordinary Run preparation or
// after RunPrepared claims its handle.
StartTime time.Time `json:"start_time,omitempty"`
// EndTime is the UTC time after generation and validation complete.
EndTime time.Time `json:"end_time,omitempty"`
// Duration covers preparation, generation, and validation for Run. For
// RunPrepared it covers only the execution attempt after claim and excludes
// preparation and consumer-held delay. JSON represents it as integer
// milliseconds in duration_ms and omits a zero value.
Duration time.Duration `json:"-"`
}
// ArtifactRef represents a reference to prompt input content.
// ArtifactRef identifies prompt input content. It has no stable JSON
// representation. Prefer [File], [Inline], or [InlineWithURI] to construct one.
type ArtifactRef struct {
// Type must be ArtifactRefInline or ArtifactRefFile.
Type ArtifactRefType
URI string
// URI is the file path for ArtifactRefFile and optional provenance metadata
// for ArtifactRefInline.
URI string
// Body is the content for ArtifactRefInline and is ignored for
// ArtifactRefFile.
Body string
}
// Artifact represents loaded artifact content.
// Artifact represents loaded or generated content and has a stable JSON
// representation. Body uses encoding/json's base64 representation for []byte.
type Artifact struct {
Name string
ContentType string
Body []byte
URI string
Size int64
Hash string
// Name is artifact metadata. During input preparation the engine fills an
// empty reader-supplied name with the request input-map key.
Name string `json:"name"`
// ContentType is the media type reported by the reader or derived for
// generated output.
ContentType string `json:"content_type"`
// Body is the artifact content. Engine boundaries copy this slice.
Body []byte `json:"body"`
// URI is optional source or result provenance metadata.
URI string `json:"uri"`
// Size is content-size metadata in bytes.
Size int64 `json:"size"`
// Hash is an opaque content equality value when the producing reader
// supplies one. Its format and algorithm are not API contracts.
Hash string `json:"hash"`
}
// ArtifactReader resolves a prompt input reference into its content.
//
// Readers are responsible for supplying artifact metadata. The engine assigns
// an input-map name only when the returned artifact name is empty.
// Read may be called concurrently. It must honor ctx cancellation to make
// Prepare, PrepareExecution, and Run responsive to cancellation. The engine
// passes a copied ref and immediately copies the returned Artifact.Body; it
// does not retain either value. Readers supply artifact metadata, and the
// engine assigns an input-map name only when the returned artifact name is
// empty.
//
// An injected reader owns any application-specific path containment,
// authorization, content-size, and content-type policy. It must protect
// sensitive references and bodies in its logging and in any copies it retains.
// It may reuse or mutate the returned artifact and body after Read returns.
//
// Returning a non-nil error makes the engine return an error matching
// ErrArtifactLoad while preserving the reader error through errors.Is.
// Returning a nil artifact with a nil error also produces ErrArtifactLoad.
type ArtifactReader interface {
Read(context.Context, ArtifactRef) (*Artifact, error)
}
// ExecutionTarget represents effective model runtime settings.
// ExecutionTarget represents effective model runtime settings and has a stable
// JSON representation. It never exposes a resolved API-key value.
type ExecutionTarget struct {
Endpoint string `json:"endpoint"`
Model string `json:"model"`
Temperature float64 `json:"temperature"`
MaxTokens int `json:"max_tokens"`
TopP float64 `json:"top_p"`
TimeoutSeconds int `json:"timeout_seconds"`
ServiceTier string `json:"service_tier"`
ReasoningEffort string `json:"reasoning_effort"`
APIKeyEnv string `json:"api_key_env"`
ExtraParams map[string]any `json:"extra_params"`
// BackendID is the effective routing identity selected by the profile. It
// remains unchanged when a profile or request overrides Endpoint and is
// empty for endpoint-only profiles. It is supplied to injected LLMClient
// implementations as part of the effective target.
BackendID string `json:"backend_id,omitempty"`
// Endpoint is the model-provider base URL.
Endpoint string `json:"endpoint"`
// Model is the provider model identifier.
Model string `json:"model"`
// Temperature is the resolved sampling temperature from 0 through 2. Zero
// leaves the field unspecified to compatible providers unless the
// corresponding ExecutionTargetPresence bit is true.
Temperature float64 `json:"temperature"`
// MaxTokens is the non-negative resolved output-token limit. Zero leaves
// the limit unspecified to compatible providers unless the corresponding
// ExecutionTargetPresence bit is true.
MaxTokens int `json:"max_tokens"`
// TopP is the resolved nucleus-sampling value from 0 through 1. Zero leaves
// the field unspecified to compatible providers unless the corresponding
// ExecutionTargetPresence bit is true.
TopP float64 `json:"top_p"`
// TimeoutSeconds is the non-negative per-generation deadline. Zero disables
// this deadline without disabling caller cancellation or the transport cap.
TimeoutSeconds int `json:"timeout_seconds"`
// ServiceTier is an optional provider-specific request tier.
ServiceTier string `json:"service_tier"`
// ReasoningEffort is the effective opaque provider-specific reasoning
// setting. An empty value instructs model clients to omit reasoning.
ReasoningEffort string `json:"reasoning_effort"`
// APIKeyEnv is an environment-variable name, not its credential value.
APIKeyEnv string `json:"api_key_env"`
// ExtraParams contains copied JSON-compatible provider parameters.
ExtraParams map[string]any `json:"extra_params"`
}
// ExecutionTargetOverride represents per-request runtime setting overrides.
// ProfileInspection is the caller-owned result of [Engine.InspectProfile].
// It has no stable JSON representation.
//
// EffectiveModelParams contains a copied effective target. APIKeyRequired is
// separate from that target to preserve ExecutionTarget's general execution
// and stable JSON contracts.
type ProfileInspection struct {
// ProfileID is the trimmed, exact profile ID inspected by the engine.
ProfileID string
// EffectiveModelParams contains settings resolved from the framework timeout
// baseline, selected backend, and then profile, without a request override.
// Unset optional provider controls remain zero rather than reporting a
// provider default. APIKeyEnv is an environment-variable name, never its
// credential value.
EffectiveModelParams ExecutionTarget
// APIKeyRequired reports that a later execution must supply a direct API
// key or an explicit request environment override. It is mutually exclusive
// with a nonblank EffectiveModelParams.APIKeyEnv.
APIKeyRequired bool
}
// PromptInputDefinition describes one declared prompt input.
// It has no stable JSON representation.
type PromptInputDefinition struct {
// Name is the normalized prompt input name.
Name string
// Required is the prompt definition's declared required flag. When true,
// preparation fails if the input is omitted. A false value does not account
// for input references in message or session-ID templates.
Required bool
// ContentType is the declared input media-type metadata.
ContentType string
// Description is the declared human-readable input description.
Description string
}
// PromptInspection is the caller-owned result of [Engine.InspectPrompt].
// It has no stable JSON representation.
//
// Inputs contains copied declared input metadata in definition order.
// OutputContract is the normalized contract declared by the prompt definition,
// rather than a request-level effective override. PromptHash is opaque.
type PromptInspection struct {
// PromptID is the normalized ID of the selected prompt definition.
PromptID string
// PromptVersion is the normalized version of the selected prompt definition.
PromptVersion string
// PromptHash is the opaque equality value for the selected definition.
PromptHash string
// DefaultProfileID is declared metadata and is not resolved by inspection.
DefaultProfileID string
// Inputs contains caller-owned declared input metadata in definition order.
Inputs []PromptInputDefinition
// OutputContract is the normalized contract declared by the definition.
OutputContract OutputContract
}
// ExecutionTargetOverride represents per-request runtime setting overrides and
// has no stable JSON representation.
//
// Non-empty string fields replace profile and backend values. Non-nil pointer
// fields replace profile values and preserve explicit zero or empty values. A
// non-empty ExtraParams map replaces the complete profile or backend map
// rather than merging keys. Empty string fields, nil pointers, and a nil or
// empty ExtraParams map inherit lower-precedence values. An optional provider
// control that remains zero is unspecified; TimeoutSeconds retains its
// framework deadline when no higher-precedence value is present.
type ExecutionTargetOverride struct {
Endpoint string
Model string
Temperature *float64
MaxTokens *int
TopP *float64
TimeoutSeconds *int
ServiceTier string
ReasoningEffort string
APIKeyEnv string
ExtraParams map[string]any
// Endpoint replaces the profile or backend endpoint when non-empty without
// changing the effective BackendID.
Endpoint string
// Model replaces the profile model when non-empty.
Model string
// Temperature, when non-nil, must point to a value from 0 through 2. A
// pointed-to zero is explicitly present; nil inherits a lower-precedence
// value and otherwise leaves the provider control unspecified.
Temperature *float64
// MaxTokens, when non-nil, must point to a non-negative value. A pointed-to
// zero is explicitly present; nil inherits a lower-precedence value and
// otherwise leaves the provider control unspecified.
MaxTokens *int
// TopP, when non-nil, must point to a value from 0 through 1. A pointed-to
// zero is explicitly present; nil inherits a lower-precedence value and
// otherwise leaves the provider control unspecified.
TopP *float64
// TimeoutSeconds, when non-nil, must point to a non-negative value. A
// pointed-to zero disables the per-generation deadline.
TimeoutSeconds *int
// ServiceTier replaces the profile value when non-blank.
ServiceTier string
// ReasoningEffort controls the per-run reasoning setting. Nil inherits the
// profile value. A pointer to a non-blank string trims and replaces the
// profile value. A pointer to an empty or whitespace-only string clears the
// inherited value and disables reasoning for this run. Non-blank values
// are opaque and are not validated against a fixed vocabulary.
ReasoningEffort *string
// APIKeyEnv replaces the profile or backend environment-variable name when
// non-blank. A direct RunRequest.APIKey still takes precedence over
// environment lookup.
APIKeyEnv string
// ExtraParams, when non-empty, replaces the complete profile or backend map.
// Values must be JSON-compatible: nil, booleans, finite numbers, strings,
// arrays or slices, and maps with non-empty string keys. Cycles are invalid.
ExtraParams map[string]any
}
// Profile is an in-memory execution profile for library consumers.
//
// It is equivalent to a loaded profile file after validation. Raw API keys do
// not belong in profiles; use APIKeyRequired to require callers to provide
// RunRequest.APIKey for each request, or use profile YAML api_key_env with file
// and FS profile sources.
// not belong in profiles; use APIKeyRequired to require callers to provide a
// RunRequest.APIKey or explicit request ExecutionTargetOverride.APIKeyEnv, or
// use profile YAML api_key_env with file and FS profile sources. Profile has no
// stable JSON representation.
//
// WithProfiles validates and copies Profile values during NewEngine. Zero
// Temperature, MaxTokens, and TopP values and blank ServiceTier and
// ReasoningEffort values leave those provider controls unspecified. A zero
// TimeoutSeconds retains the framework deadline, while an empty ExtraParams map
// inherits backend request defaults. Use ExecutionTargetOverride pointer fields
// to request an explicit numeric zero.
type Profile struct {
ID string
Endpoint string
Model string
Temperature float64
MaxTokens int
TopP float64
TimeoutSeconds int
ServiceTier string
// ID is the required non-blank profile identifier. WithProfiles trims it.
ID string
// BackendID optionally selects an engine backend. WithProfiles trims it.
// Backend membership is checked when a request selects the profile; an
// unknown ID makes preparation fail with ErrProfileLoad.
BackendID string
// Endpoint is the model-provider base URL. It is required only when
// BackendID is blank and otherwise overrides the backend endpoint when
// non-blank.
Endpoint string
// Model is the required non-blank provider model identifier.
Model string
// Temperature is from 0 through 2. Zero leaves the provider control
// unspecified.
Temperature float64
// MaxTokens is non-negative. Zero leaves the provider control unspecified.
MaxTokens int
// TopP is from 0 through 1. Zero leaves the provider control unspecified
// rather than selecting an explicit zero.
TopP float64
// TimeoutSeconds is non-negative. Zero retains the framework deadline.
TimeoutSeconds int
// ServiceTier is optional; a blank value leaves it unspecified.
ServiceTier string
// ReasoningEffort is optional; a blank value leaves it unspecified.
ReasoningEffort string
APIKeyRequired bool
ExtraParams map[string]any
// APIKeyRequired clears a backend's inherited API-key environment name and
// requires a non-blank RunRequest.APIKey unless the request explicitly
// supplies ExecutionTargetOverride.APIKeyEnv. It does not store a credential.
APIKeyRequired bool
// ExtraParams contains provider-specific JSON-compatible values. An empty
// map inherits backend request defaults, when any. WithProfiles validates
// and deeply copies it during NewEngine.
ExtraParams map[string]any
}
// OpenAICompatibleProfileConfig configures an OpenAI-compatible in-memory
// profile.
//
// It contains ordinary profile fields for OpenAI-compatible chat-completions
// endpoints. APIKeyRequired is satisfied by RunRequest.APIKey. Raw API keys do
// not belong in this config.
// endpoints. APIKeyRequired follows Profile.APIKeyRequired. Raw API keys do not
// belong in this config. OpenAICompatibleProfileConfig has no stable JSON
// representation and is not validated until its resulting Profile is supplied
// through WithProfiles to NewEngine.
type OpenAICompatibleProfileConfig struct {
ID string
Endpoint string
Model string
APIKeyRequired bool
Temperature float64
MaxTokens int
TopP float64
TimeoutSeconds int
ServiceTier string
// ID becomes Profile.ID.
ID string
// BackendID becomes Profile.BackendID.
BackendID string
// Endpoint becomes Profile.Endpoint.
Endpoint string
// Model becomes Profile.Model.
Model string
// APIKeyRequired becomes Profile.APIKeyRequired.
APIKeyRequired bool
// Temperature becomes Profile.Temperature.
Temperature float64
// MaxTokens becomes Profile.MaxTokens.
MaxTokens int
// TopP becomes Profile.TopP.
TopP float64
// TimeoutSeconds becomes Profile.TimeoutSeconds.
TimeoutSeconds int
// ServiceTier becomes Profile.ServiceTier.
ServiceTier string
// ReasoningEffort becomes Profile.ReasoningEffort.
ReasoningEffort string
ExtraParams map[string]any
// ExtraParams becomes a shallow-copied Profile.ExtraParams map. NewEngine
// performs validation and a deep copy when WithProfiles applies the result.
ExtraParams map[string]any
}
// ExecutionTargetPresence tracks which numeric runtime settings were explicit
// request overrides.
// request overrides, including explicit zero values. It has a stable JSON
// representation and is supplied to injected LLM clients so they can preserve
// omission semantics.
type ExecutionTargetPresence struct {
Temperature bool
MaxTokens bool
TopP bool
TimeoutSeconds bool
// Temperature reports a non-nil ExecutionTargetOverride.Temperature.
Temperature bool `json:"temperature"`
// MaxTokens reports a non-nil ExecutionTargetOverride.MaxTokens.
MaxTokens bool `json:"max_tokens"`
// TopP reports a non-nil ExecutionTargetOverride.TopP.
TopP bool `json:"top_p"`
// TimeoutSeconds reports a non-nil ExecutionTargetOverride.TimeoutSeconds.
TimeoutSeconds bool `json:"timeout_seconds"`
}
// OutputContract defines output and validation requirements.
// OutputContract defines output and validation requirements and has a stable
// JSON representation.
//
// A non-nil RunRequest.Validation replaces the complete prompt contract. It
// does not merge fields. The public Engine validates generated output once and
// does not install an output repairer.
type OutputContract struct {
Format OutputFormat `json:"format"`
// Format selects generated artifact metadata. An empty effective value
// defaults to FormatText.
Format OutputFormat `json:"format"`
// ValidationMode selects the content check. Use one of the declared
// ValidationMode constants.
ValidationMode ValidationMode `json:"validation_mode"`
SchemaPath string `json:"schema_path"`
RepairAttempts int `json:"repair_attempts"`
// SchemaPath is required when ValidationMode is ValidationJSONSchema and is
// ignored by other modes.
SchemaPath string `json:"schema_path"`
// RepairAttempts is a requested repair limit. A non-positive value requests
// no repairs. The public Engine performs no repairs even when this value is
// positive, so its runs report zero attempts used.
RepairAttempts int `json:"repair_attempts"`
}
// ValidationResult represents output validation state.
// ValidationResult represents a completed output check and has a stable JSON
// representation. An operational inability to perform validation is returned
// as ErrValidation instead of a ValidationResult.
type ValidationResult struct {
Status ValidationStatus `json:"status"`
Mode ValidationMode `json:"mode"`
Errors []string `json:"errors,omitempty"`
SchemaPath string `json:"schema_path,omitempty"`
RepairAttempts int `json:"repair_attempts"`
IsValid bool `json:"is_valid"`
// Status is Passed, Failed, or Skipped.
Status ValidationStatus `json:"status"`
// Mode is the effective validation mode.
Mode ValidationMode `json:"mode"`
// Errors contains validation diagnostics when Status is ValidationFailed.
Errors []string `json:"errors,omitempty"`
// SchemaPath is the effective schema path for JSON Schema validation.
SchemaPath string `json:"schema_path,omitempty"`
// RepairAttempts is the number of repairs actually attempted. It is always
// zero for the public Engine.
RepairAttempts int `json:"repair_attempts"`
// IsValid is true for ValidationPassed and ValidationSkipped and false for
// ValidationFailed.
IsValid bool `json:"is_valid"`
}
// TokenUsage tracks token consumption.
// TokenUsage contains model-client token accounting and has a stable JSON
// representation. Promptkit preserves values reported by the client and does
// not derive or reconcile them.
type TokenUsage struct {
PromptTokens int `json:"prompt_tokens"`
// PromptTokens is the reported input-token count.
PromptTokens int `json:"prompt_tokens"`
// CompletionTokens is the reported generated-token count.
CompletionTokens int `json:"completion_tokens"`
TotalTokens int `json:"total_tokens"`
CachedTokens int `json:"cached_tokens"`
// TotalTokens is the reported total-token count.
TotalTokens int `json:"total_tokens"`
// CachedTokens is the reported cached-input-token count.
CachedTokens int `json:"cached_tokens"`
// CacheWriteTokens is the reported cache-write-token count.
CacheWriteTokens int `json:"cache_write_tokens"`
}
// RenderedPrompt is the fully rendered prompt passed to an LLM client.
// RenderedPrompt is the fully rendered prompt passed to an LLM client and has
// a stable JSON representation.
type RenderedPrompt struct {
SessionID string `json:"session_id,omitempty"`
Messages []RenderedMessage `json:"messages"`
// SessionID is the optional effective direct or rendered session
// identifier supplied to the model client.
SessionID string `json:"session_id,omitempty"`
// Messages contains rendered messages in definition order.
Messages []RenderedMessage `json:"messages"`
}
// RenderedMessage is a rendered chat message.
// RenderedMessage is a rendered chat message and has a stable JSON
// representation.
type RenderedMessage struct {
Role string `json:"role"`
Content string `json:"content"`
// Role is the definition-supplied chat role.
Role string `json:"role"`
// Content is the rendered message text.
Content string `json:"content"`
// CacheControl is optional provider cache metadata.
CacheControl *CacheControl `json:"cache_control,omitempty"`
}
// CacheControl describes provider cache metadata attached to prompt content.
// CacheControl describes provider cache metadata attached to prompt content
// and has a stable JSON representation.
type CacheControl struct {
// Type identifies the cache behavior.
Type CacheControlType `json:"type"`
TTL string `json:"ttl,omitempty"`
// TTL is an optional provider cache lifetime.
TTL string `json:"ttl,omitempty"`
}
// StructuredOutputSpec describes provider-level structured output.
// StructuredOutputSpec describes provider-level structured output and has a
// stable JSON representation.
type StructuredOutputSpec struct {
Type StructuredOutputType `json:"type"`
// Type identifies the structured-output mechanism.
Type StructuredOutputType `json:"type"`
// JSONSchema contains constraints when Type is StructuredOutputJSONSchema.
JSONSchema *StructuredOutputJSONSpec `json:"json_schema,omitempty"`
}
// StructuredOutputJSONSpec contains JSON Schema output constraints.
// StructuredOutputJSONSpec contains provider-facing JSON Schema output
// constraints and has a stable JSON representation.
type StructuredOutputJSONSpec struct {
Name string `json:"name"`
Strict bool `json:"strict"`
Schema any `json:"schema"`
// Name is the provider-facing schema name.
Name string `json:"name"`
// Strict requests strict provider enforcement of Schema.
Strict bool `json:"strict"`
// Schema is a caller-owned copy of the loaded JSON Schema document.
Schema any `json:"schema"`
}
// LLMClient executes rendered prompts for Engine.Run.
// LLMClient executes rendered prompts for [Engine.Run] and
// [Engine.RunPrepared].
//
// Generate is scheduled according to the resolved backend's capacity policy.
// It may still be called concurrently for different backend pools or unlimited
// backends. Cancellation while waiting for capacity can prevent Generate from
// being called. Once invoked, it must honor context cancellation to make Run
// and RunPrepared responsive to cancellation. The request and all nested maps,
// slices, and pointers are client-owned copies and may be mutated or retained
// without affecting engine state.
//
// Generate receives rendered messages and may receive a direct API key. A
// client must protect those values and any raw output in its logging, storage,
// and retained copies. It is responsible for the cancellation behavior of any
// work it starts and for synchronizing access to retained or shared data.
//
// A returned error makes Run or RunPrepared return ErrLLMGenerate while
// preserving the client error through errors.Is. A nil response with a nil
// error also produces ErrLLMGenerate. Promptkit copies the non-nil response
// before returning from either method.
type LLMClient interface {
Generate(context.Context, GenerateRequest) (*GenerateResponse, error)
}
// GenerateRequest is passed to an injected LLM client.
// GenerateRequest is passed to an injected LLM client and has a stable JSON
// representation. Its String and GoString methods omit rendered content and
// direct credentials.
type GenerateRequest struct {
Prompt RenderedPrompt `json:"prompt"`
Target ExecutionTarget `json:"target"`
TargetPresence ExecutionTargetPresence `json:"target_presence"`
StructuredOutput *StructuredOutputSpec `json:"structured_output,omitempty"`
APIKey string `json:"-"`
// Prompt contains the rendered session ID and messages.
Prompt RenderedPrompt `json:"prompt"`
// Target contains effective model settings without the direct API key.
Target ExecutionTarget `json:"target"`
// TargetPresence distinguishes inherited numeric zeros from explicit
// request overrides.
TargetPresence ExecutionTargetPresence `json:"target_presence"`
// StructuredOutput contains provider response constraints when requested.
StructuredOutput *StructuredOutputSpec `json:"structured_output,omitempty"`
// APIKey is the direct request-scoped credential, if any. It is excluded
// from JSON, String, and GoString output.
APIKey string `json:"-"`
}
// GenerateResponse is returned by an injected LLM client.
// GenerateResponse is returned by an injected LLM client and has a stable JSON
// representation.
type GenerateResponse struct {
Content string `json:"content"`
Usage TokenUsage `json:"usage"`
// Content is the generated output. It must be non-empty when using the
// built-in client; injected clients may return empty content for Promptkit
// validation to classify.
Content string `json:"content"`
// Usage is the client's token accounting.
Usage TokenUsage `json:"usage"`
}
// File returns a file-backed artifact reference.
// File returns a file-backed artifact reference whose URI is path.
//
// The default artifact reader opens path as a caller-selected operating-system
// path without restricting it to an application root or imposing a size limit.
// Applications accepting untrusted paths must validate them before calling
// Promptkit or use [WithArtifactReader] to enforce application policy.
func File(path string) ArtifactRef {
return ArtifactRef{Type: ArtifactRefFile, URI: path}
}
// Inline returns an inline artifact reference.
// Inline returns an inline artifact reference whose Body is body and whose URI
// is empty.
func Inline(body string) ArtifactRef {
return ArtifactRef{Type: ArtifactRefInline, Body: body}
}
// InlineWithURI returns an inline artifact reference with URI metadata.
// InlineWithURI returns an inline artifact reference with body content and uri
// provenance metadata.
func InlineWithURI(uri string, body string) ArtifactRef {
return ArtifactRef{Type: ArtifactRefInline, URI: uri, Body: body}
}