Compare commits
72 Commits
v0.1.0
...
5ccfa4a345
| Author | SHA1 | Date | |
|---|---|---|---|
| 5ccfa4a345 | |||
| 14e03f19d0 | |||
| 7c562a9374 | |||
| ef97d85ac9 | |||
| 805e48c873 | |||
| e9e126dcba | |||
| 32e7a3557c | |||
| 1d1b04e2e0 | |||
| 9748897751 | |||
| 9d020039d5 | |||
| 5247ce0b73 | |||
| c434aa1dae | |||
| ac9b3f3d80 | |||
| 4f12a89a1b | |||
| df31e7f58e | |||
| 0678d242b9 | |||
| 3b4ea21208 | |||
| 1430e85147 | |||
| 34d7a19da5 | |||
| ebf1602635 | |||
| 31f2ce3a09 | |||
| fd06e4ca6b | |||
| e63b8de1e9 | |||
| 9354d2b373 | |||
| 01ca5430bd | |||
| ae2179d103 | |||
| a248433d0f | |||
| bd6cffc9d0 | |||
| e40c4f182b | |||
| 7428e50c2c | |||
| 63c67a4520 | |||
| 25a7052a3d | |||
| fc3255967e | |||
| e920168b30 | |||
| 272b6a4bc1 | |||
| dde48a31fc | |||
| 242eace4a7 | |||
| 0bf5f88136 | |||
| 369ab5392d | |||
| 2ba0146e5d | |||
| 6112c2af0c | |||
| f5e12c00f5 | |||
| 49fe402dd2 | |||
| c301eb8d55 | |||
| c13e9710d9 | |||
| 87b5ec3d75 | |||
| cb4028a637 | |||
| 5a1bff4529 | |||
| 805a7f965d | |||
| 147f5e5ff5 | |||
| e361c97bb5 | |||
| be67707582 | |||
| e61ab700c7 | |||
| d2c4051dd0 | |||
| 861da355d8 | |||
| a752f88166 | |||
| 238fa90bfa | |||
| ffe6d261a9 | |||
| dc39562ff7 | |||
| 0a839aa16d | |||
| f6ee18f6b3 | |||
| eb8ab215e8 | |||
| f89cb94ed2 | |||
| 359b7313f4 | |||
| ae210b3c26 | |||
| 810f80e7c9 | |||
| 8d00354c59 | |||
| d0010689f3 | |||
| b462153483 | |||
| 086cf0fc86 | |||
| c1cecb1ee8 | |||
| bcb327f643 |
19
README.md
19
README.md
@@ -31,4 +31,23 @@ Contributors should start with the [development guide](docs/development.md).
|
||||
The [architecture policy](docs/policy/architecture.md) defines the library
|
||||
boundary and constraints that framework work must preserve.
|
||||
|
||||
## Release Guidance
|
||||
|
||||
Consumers upgrading from `v0.4.0` to `v0.5.0` should read the
|
||||
[v0.5.0 changelog and migration guide](docs/releases/v0.5.0.md).
|
||||
|
||||
Consumers upgrading from `v0.3.0` to `v0.4.0` should read the
|
||||
[v0.4.0 changelog and adoption guide](docs/releases/v0.4.0.md).
|
||||
|
||||
Consumers upgrading from `v0.2.0` to `v0.3.0` should read the
|
||||
[v0.3.0 changelog](docs/releases/v0.3.0.md).
|
||||
|
||||
Consumers moving from `v0.1.0` to `v0.2.0` should read the
|
||||
[v0.2.0 changelog and migration guide](docs/releases/v0.2.0.md).
|
||||
|
||||
## Related Project
|
||||
|
||||
[Scriptorium](https://gitea.maximumdirect.net/eric/scriptorium) is the CLI and
|
||||
HTTP application built on Promptkit.
|
||||
|
||||
Promptkit is licensed under the [GNU General Public License version 3](LICENSE).
|
||||
|
||||
99
backends.go
Normal file
99
backends.go
Normal file
@@ -0,0 +1,99 @@
|
||||
package promptkit
|
||||
|
||||
import (
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/backend"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
)
|
||||
|
||||
// BackendOpenRouter is the reserved ID of Promptkit's built-in OpenRouter
|
||||
// backend.
|
||||
const BackendOpenRouter = backend.OpenRouterID
|
||||
|
||||
// BackendLocal is the case-sensitive conventional ID used by [LocalBackend].
|
||||
// It is not a built-in or reserved backend and must be registered with
|
||||
// [WithBackend].
|
||||
const BackendLocal = "local"
|
||||
|
||||
// Backend configures one engine-scoped OpenAI-compatible backend.
|
||||
//
|
||||
// Backend has no stable JSON representation. Use keyed literals so additions
|
||||
// to this configuration value do not break source compatibility.
|
||||
type Backend struct {
|
||||
// ID is the stable, case-sensitive registry key. NewEngine trims it and
|
||||
// requires a non-blank value. BackendOpenRouter is reserved.
|
||||
ID string
|
||||
// Endpoint is the OpenAI-compatible base endpoint. NewEngine trims it and
|
||||
// requires an absolute HTTP or HTTPS URL with a host and without user
|
||||
// information, a query string, or a fragment. Paths are allowed.
|
||||
Endpoint string
|
||||
// APIKeyEnv optionally names the environment variable containing the API
|
||||
// key. NewEngine trims it and requires the portable form
|
||||
// [A-Za-z_][A-Za-z0-9_]*. Store only the name, never a credential value.
|
||||
APIKeyEnv string
|
||||
// ExtraParams contains backend-wide request defaults. Values must be
|
||||
// JSON-compatible, finite, acyclic, and keyed by non-empty strings. Keys
|
||||
// must not be model, session_id, messages, temperature, max_tokens, top_p,
|
||||
// service_tier, reasoning_effort, or response_format. An empty map supplies
|
||||
// no defaults. NewEngine deeply copies the map.
|
||||
ExtraParams map[string]any
|
||||
// ConcurrencyLimit is the maximum number of simultaneous model-generation
|
||||
// calls allowed for this backend within one Engine. Zero leaves the backend
|
||||
// unlimited. A negative value makes NewEngine fail with ErrInvalidConfig.
|
||||
ConcurrencyLimit int
|
||||
// QueueCapacity controls how many additional Run or RunPrepared calls may
|
||||
// be admitted beyond ConcurrencyLimit. Nil uses 1024 when ConcurrencyLimit
|
||||
// is positive; a pointer uses its exact value, including zero. The pointed-to
|
||||
// value must be non-negative, and QueueCapacity must be nil when
|
||||
// ConcurrencyLimit is zero. Their sum must fit in an int. WithBackend copies
|
||||
// the value and does not retain the pointer.
|
||||
QueueCapacity *int
|
||||
}
|
||||
|
||||
// LocalBackend returns a caller-owned Backend for a conventional local
|
||||
// OpenAI-compatible endpoint. It sets ID to BackendLocal and copies endpoint
|
||||
// and concurrencyLimit into Endpoint and ConcurrencyLimit without
|
||||
// normalization or validation. APIKeyEnv, ExtraParams, and QueueCapacity keep
|
||||
// their zero values.
|
||||
//
|
||||
// LocalBackend does not read environment variables, register the value, or
|
||||
// mutate engine or package state. Supply the returned value through
|
||||
// [WithBackend]; [NewEngine] then applies the ordinary backend validation and
|
||||
// concurrency semantics, including default queue capacity for a positive
|
||||
// limit, unlimited behavior for zero, and ErrInvalidConfig for a negative
|
||||
// limit.
|
||||
func LocalBackend(endpoint string, concurrencyLimit int) Backend {
|
||||
return Backend{
|
||||
ID: BackendLocal,
|
||||
Endpoint: endpoint,
|
||||
ConcurrencyLimit: concurrencyLimit,
|
||||
}
|
||||
}
|
||||
|
||||
// WithBackend adds one Backend registration to the constructed Engine.
|
||||
//
|
||||
// Registrations accumulate in option order. Every normalized ID must be unique
|
||||
// across consumer registrations and built-ins; a duplicate or invalid
|
||||
// definition makes NewEngine fail with ErrInvalidConfig. In particular,
|
||||
// BackendOpenRouter cannot be replaced. The immutable registration is scoped
|
||||
// to the resulting Engine and cannot be enumerated, replaced, removed, or
|
||||
// mutated after construction. WithBackend does not install package-global
|
||||
// state.
|
||||
func WithBackend(backend Backend) Option {
|
||||
queueCapacity := 0
|
||||
queueCapacitySet := backend.QueueCapacity != nil
|
||||
if queueCapacitySet {
|
||||
queueCapacity = *backend.QueueCapacity
|
||||
}
|
||||
return optionFunc(func(options *engineOptions) error {
|
||||
options.backends = append(options.backends, domain.Backend{
|
||||
ID: backend.ID,
|
||||
Endpoint: backend.Endpoint,
|
||||
APIKeyEnv: backend.APIKeyEnv,
|
||||
ExtraParams: backend.ExtraParams,
|
||||
ConcurrencyLimit: backend.ConcurrencyLimit,
|
||||
QueueCapacity: queueCapacity,
|
||||
QueueCapacitySet: queueCapacitySet,
|
||||
})
|
||||
return nil
|
||||
})
|
||||
}
|
||||
409
capacity_contract_test.go
Normal file
409
capacity_contract_test.go
Normal file
@@ -0,0 +1,409 @@
|
||||
package promptkit_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
func TestEngineLimitsInjectedClientConcurrency(t *testing.T) {
|
||||
release := make(chan struct{})
|
||||
client := newCapacityGateClient(release, 8)
|
||||
engine := newBackendCapacityEngine(t, client, 2, capacityInt(4), nil)
|
||||
|
||||
results := make(chan capacityRunResult, 6)
|
||||
for i := 0; i < 6; i++ {
|
||||
go runCapacityRequest(engine, context.Background(), promptkit.RunRequest{
|
||||
PromptID: "prompt",
|
||||
Execution: &promptkit.ExecutionTargetOverride{
|
||||
Endpoint: "http://request.example/v1",
|
||||
},
|
||||
}, results)
|
||||
}
|
||||
|
||||
first := awaitCapacityRequest(t, client.started)
|
||||
second := awaitCapacityRequest(t, client.started)
|
||||
if first.Target.BackendID != "limited" || second.Target.BackendID != "limited" {
|
||||
t.Fatalf("endpoint override changed backend pool: first=%q second=%q",
|
||||
first.Target.BackendID, second.Target.BackendID)
|
||||
}
|
||||
if active, peak, _ := client.snapshot(); active != 2 || peak != 2 {
|
||||
t.Fatalf("client concurrency before release=(active=%d peak=%d), want 2", active, peak)
|
||||
}
|
||||
|
||||
close(release)
|
||||
for i := 0; i < 6; i++ {
|
||||
outcome := awaitCapacityRun(t, results)
|
||||
if outcome.err != nil || outcome.result == nil {
|
||||
t.Fatalf("run outcome=(%+v, %v), want success", outcome.result, outcome.err)
|
||||
}
|
||||
}
|
||||
if _, peak, calls := client.snapshot(); peak > 2 || calls != 6 {
|
||||
t.Fatalf("client observations=(peak=%d calls=%d), want peak <= 2 and 6 calls", peak, calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEngineRejectsRunBeforeCompletionWhenAdmissionIsFull(t *testing.T) {
|
||||
artifactRelease := make(chan struct{})
|
||||
reader := &capacityArtifactReader{
|
||||
entered: make(chan struct{}, 2),
|
||||
release: artifactRelease,
|
||||
}
|
||||
client := newCapacityGateClient(closedCapacityChannel(), 2)
|
||||
engine := newBackendCapacityEngine(t, client, 1, capacityInt(0), reader)
|
||||
firstResult := make(chan capacityRunResult, 1)
|
||||
go runCapacityRequest(engine, context.Background(), capacityInputRequest("http://first.example/v1"), firstResult)
|
||||
|
||||
awaitCapacitySignal(t, reader.entered, "first artifact read")
|
||||
|
||||
canceledContext, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
result, err := engine.Run(canceledContext, capacityInputRequest("http://canceled.example/v1"))
|
||||
if result != nil || !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("canceled capacity admission=(%+v, %v), want context cancellation", result, err)
|
||||
}
|
||||
var canceledCapacityErr *promptkit.CapacityError
|
||||
if errors.Is(err, promptkit.ErrCapacityExceeded) || errors.As(err, &canceledCapacityErr) {
|
||||
t.Fatalf("canceled admission exposed capacity rejection: %v", err)
|
||||
}
|
||||
|
||||
result, err = engine.Run(context.Background(), capacityInputRequest("http://second.example/v1"))
|
||||
if result != nil {
|
||||
t.Fatalf("capacity rejection returned partial result: %+v", result)
|
||||
}
|
||||
if !errors.Is(err, promptkit.ErrCapacityExceeded) {
|
||||
t.Fatalf("capacity rejection=%v, want ErrCapacityExceeded", err)
|
||||
}
|
||||
if errors.Is(err, promptkit.ErrInvalidRequest) || errors.Is(err, promptkit.ErrLLMGenerate) {
|
||||
t.Fatalf("capacity rejection had an unrelated category: %v", err)
|
||||
}
|
||||
var capacityErr *promptkit.CapacityError
|
||||
if !errors.As(err, &capacityErr) || capacityErr == nil {
|
||||
t.Fatalf("capacity rejection=%v, want CapacityError", err)
|
||||
}
|
||||
if capacityErr.BackendID != "limited" {
|
||||
t.Fatalf("capacity backend ID=%q, want limited", capacityErr.BackendID)
|
||||
}
|
||||
capacityErr.BackendID = "changed"
|
||||
|
||||
result, err = engine.Run(context.Background(), capacityInputRequest("http://third.example/v1"))
|
||||
var subsequentCapacityErr *promptkit.CapacityError
|
||||
if result != nil || !errors.As(err, &subsequentCapacityErr) ||
|
||||
subsequentCapacityErr == nil || subsequentCapacityErr.BackendID != "limited" {
|
||||
t.Fatalf("subsequent capacity rejection=(%+v, %v), want independent limited CapacityError", result, err)
|
||||
}
|
||||
if calls := reader.callCount(); calls != 1 {
|
||||
t.Fatalf("artifact calls=%d, want only the admitted run", calls)
|
||||
}
|
||||
if _, _, calls := client.snapshot(); calls != 0 {
|
||||
t.Fatalf("client calls=%d before admitted run was released, want 0", calls)
|
||||
}
|
||||
|
||||
close(artifactRelease)
|
||||
outcome := awaitCapacityRun(t, firstResult)
|
||||
if outcome.err != nil || outcome.result == nil {
|
||||
t.Fatalf("first run outcome=(%+v, %v), want success", outcome.result, outcome.err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBackendCapacityIsIndependentBetweenEngines(t *testing.T) {
|
||||
firstRelease := make(chan struct{})
|
||||
firstClient := newCapacityGateClient(firstRelease, 1)
|
||||
firstEngine := newBackendCapacityEngine(t, firstClient, 1, capacityInt(0), nil)
|
||||
secondClient := newCapacityGateClient(closedCapacityChannel(), 1)
|
||||
secondEngine := newBackendCapacityEngine(t, secondClient, 1, capacityInt(0), nil)
|
||||
|
||||
firstResult := make(chan capacityRunResult, 1)
|
||||
go runCapacityRequest(firstEngine, context.Background(), promptkit.RunRequest{PromptID: "prompt"}, firstResult)
|
||||
awaitCapacityRequest(t, firstClient.started)
|
||||
|
||||
result, err := secondEngine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
|
||||
if err != nil || result == nil {
|
||||
t.Fatalf("second engine run=(%+v, %v), want independent success", result, err)
|
||||
}
|
||||
if _, _, calls := secondClient.snapshot(); calls != 1 {
|
||||
t.Fatalf("second engine client calls=%d, want 1", calls)
|
||||
}
|
||||
|
||||
close(firstRelease)
|
||||
outcome := awaitCapacityRun(t, firstResult)
|
||||
if outcome.err != nil || outcome.result == nil {
|
||||
t.Fatalf("first engine run=(%+v, %v), want success", outcome.result, outcome.err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnlimitedBackendsRetainInjectedClientConcurrency(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
configure func(*testing.T, promptkit.LLMClient) *promptkit.Engine
|
||||
}{
|
||||
{
|
||||
name: "custom backend",
|
||||
configure: func(t *testing.T, client promptkit.LLMClient) *promptkit.Engine {
|
||||
return newBackendCapacityEngine(t, client, 0, nil, nil)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "endpoint-only profile",
|
||||
configure: func(t *testing.T, client promptkit.LLMClient) *promptkit.Engine {
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{},
|
||||
promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."),
|
||||
promptkit.WithProfiles(promptkit.Profile{
|
||||
ID: "profile", Endpoint: "http://endpoint.example/v1", Model: "model",
|
||||
}),
|
||||
promptkit.WithLLMClient(client),
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("construct endpoint-only engine: %v", err)
|
||||
}
|
||||
return engine
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
release := make(chan struct{})
|
||||
client := newCapacityGateClient(release, 2)
|
||||
engine := tc.configure(t, client)
|
||||
results := make(chan capacityRunResult, 2)
|
||||
for i := 0; i < 2; i++ {
|
||||
go runCapacityRequest(
|
||||
engine,
|
||||
context.Background(),
|
||||
promptkit.RunRequest{PromptID: "prompt"},
|
||||
results,
|
||||
)
|
||||
}
|
||||
|
||||
awaitCapacityRequest(t, client.started)
|
||||
awaitCapacityRequest(t, client.started)
|
||||
if active, peak, _ := client.snapshot(); active != 2 || peak != 2 {
|
||||
t.Fatalf("unlimited concurrency=(active=%d peak=%d), want 2", active, peak)
|
||||
}
|
||||
close(release)
|
||||
for i := 0; i < 2; i++ {
|
||||
outcome := awaitCapacityRun(t, results)
|
||||
if outcome.err != nil || outcome.result == nil {
|
||||
t.Fatalf("run outcome=(%+v, %v), want success", outcome.result, outcome.err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCapacityExceededSentinelContract(t *testing.T) {
|
||||
if promptkit.ErrCapacityExceeded == nil {
|
||||
t.Fatal("ErrCapacityExceeded is nil")
|
||||
}
|
||||
var nilCapacityErr *promptkit.CapacityError
|
||||
zeroCapacityErr := &promptkit.CapacityError{}
|
||||
populatedCapacityErr := &promptkit.CapacityError{BackendID: "limited"}
|
||||
for _, capacityErr := range []error{nilCapacityErr, zeroCapacityErr} {
|
||||
if !errors.Is(capacityErr, promptkit.ErrCapacityExceeded) {
|
||||
t.Fatalf("capacity error=%v, want ErrCapacityExceeded", capacityErr)
|
||||
}
|
||||
}
|
||||
var discoveredCapacityErr *promptkit.CapacityError
|
||||
if !errors.As(populatedCapacityErr, &discoveredCapacityErr) || discoveredCapacityErr != populatedCapacityErr {
|
||||
t.Fatalf("populated capacity error is not discoverable: %v", populatedCapacityErr)
|
||||
}
|
||||
|
||||
for _, unrelated := range []error{
|
||||
promptkit.ErrInvalidConfig,
|
||||
promptkit.ErrInvalidRequest,
|
||||
promptkit.ErrLLMGenerate,
|
||||
promptkit.ErrValidation,
|
||||
} {
|
||||
if errors.Is(promptkit.ErrCapacityExceeded, unrelated) ||
|
||||
errors.Is(unrelated, promptkit.ErrCapacityExceeded) ||
|
||||
errors.Is(populatedCapacityErr, unrelated) {
|
||||
t.Fatalf("ErrCapacityExceeded aliases unrelated sentinel %v", unrelated)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type capacityRunResult struct {
|
||||
result *promptkit.RunResult
|
||||
err error
|
||||
}
|
||||
|
||||
func runCapacityRequest(
|
||||
engine *promptkit.Engine,
|
||||
ctx context.Context,
|
||||
request promptkit.RunRequest,
|
||||
results chan<- capacityRunResult,
|
||||
) {
|
||||
result, err := engine.Run(ctx, request)
|
||||
results <- capacityRunResult{result: result, err: err}
|
||||
}
|
||||
|
||||
func newBackendCapacityEngine(
|
||||
t *testing.T,
|
||||
client promptkit.LLMClient,
|
||||
limit int,
|
||||
queueCapacity *int,
|
||||
reader promptkit.ArtifactReader,
|
||||
) *promptkit.Engine {
|
||||
t.Helper()
|
||||
promptFS := contractPromptFS("prompt", "profile", "message")
|
||||
if reader != nil {
|
||||
promptFS = contractInputPromptFS()
|
||||
}
|
||||
options := []promptkit.Option{
|
||||
promptkit.WithPromptFS(promptFS, "."),
|
||||
promptkit.WithBackend(promptkit.Backend{
|
||||
ID: "limited",
|
||||
Endpoint: "http://backend.example/v1",
|
||||
ConcurrencyLimit: limit,
|
||||
QueueCapacity: queueCapacity,
|
||||
}),
|
||||
promptkit.WithProfiles(promptkit.Profile{
|
||||
ID: "profile", BackendID: "limited", Model: "model",
|
||||
}),
|
||||
promptkit.WithLLMClient(client),
|
||||
}
|
||||
if reader != nil {
|
||||
options = append(options, promptkit.WithArtifactReader(reader))
|
||||
}
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{}, options...)
|
||||
if err != nil {
|
||||
t.Fatalf("construct capacity engine: %v", err)
|
||||
}
|
||||
return engine
|
||||
}
|
||||
|
||||
func capacityInputRequest(endpoint string) promptkit.RunRequest {
|
||||
return promptkit.RunRequest{
|
||||
PromptID: "input-prompt",
|
||||
Inputs: map[string]promptkit.ArtifactRef{
|
||||
"input": promptkit.Inline("input"),
|
||||
},
|
||||
Execution: &promptkit.ExecutionTargetOverride{Endpoint: endpoint},
|
||||
}
|
||||
}
|
||||
|
||||
type capacityGateClient struct {
|
||||
mu sync.Mutex
|
||||
active int
|
||||
peak int
|
||||
calls int
|
||||
started chan promptkit.GenerateRequest
|
||||
release <-chan struct{}
|
||||
}
|
||||
|
||||
func newCapacityGateClient(release <-chan struct{}, buffer int) *capacityGateClient {
|
||||
return &capacityGateClient{
|
||||
started: make(chan promptkit.GenerateRequest, buffer),
|
||||
release: release,
|
||||
}
|
||||
}
|
||||
|
||||
func (c *capacityGateClient) Generate(
|
||||
ctx context.Context,
|
||||
request promptkit.GenerateRequest,
|
||||
) (*promptkit.GenerateResponse, error) {
|
||||
c.mu.Lock()
|
||||
c.calls++
|
||||
c.active++
|
||||
if c.active > c.peak {
|
||||
c.peak = c.active
|
||||
}
|
||||
c.mu.Unlock()
|
||||
defer func() {
|
||||
c.mu.Lock()
|
||||
c.active--
|
||||
c.mu.Unlock()
|
||||
}()
|
||||
|
||||
c.started <- request
|
||||
select {
|
||||
case <-c.release:
|
||||
return &promptkit.GenerateResponse{Content: "ok"}, nil
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
func (c *capacityGateClient) snapshot() (active, peak, calls int) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
return c.active, c.peak, c.calls
|
||||
}
|
||||
|
||||
type capacityArtifactReader struct {
|
||||
mu sync.Mutex
|
||||
calls int
|
||||
entered chan struct{}
|
||||
release <-chan struct{}
|
||||
}
|
||||
|
||||
func (r *capacityArtifactReader) Read(
|
||||
ctx context.Context,
|
||||
_ promptkit.ArtifactRef,
|
||||
) (*promptkit.Artifact, error) {
|
||||
r.mu.Lock()
|
||||
r.calls++
|
||||
r.mu.Unlock()
|
||||
r.entered <- struct{}{}
|
||||
select {
|
||||
case <-r.release:
|
||||
return &promptkit.Artifact{Body: []byte("input")}, nil
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
func (r *capacityArtifactReader) callCount() int {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
return r.calls
|
||||
}
|
||||
|
||||
func awaitCapacityRequest(
|
||||
t *testing.T,
|
||||
requests <-chan promptkit.GenerateRequest,
|
||||
) promptkit.GenerateRequest {
|
||||
t.Helper()
|
||||
select {
|
||||
case request := <-requests:
|
||||
return request
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("timed out waiting for client invocation")
|
||||
return promptkit.GenerateRequest{}
|
||||
}
|
||||
}
|
||||
|
||||
func awaitCapacityRun(t *testing.T, results <-chan capacityRunResult) capacityRunResult {
|
||||
t.Helper()
|
||||
select {
|
||||
case result := <-results:
|
||||
return result
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("timed out waiting for Run")
|
||||
return capacityRunResult{}
|
||||
}
|
||||
}
|
||||
|
||||
func awaitCapacitySignal(t *testing.T, signal <-chan struct{}, name string) {
|
||||
t.Helper()
|
||||
select {
|
||||
case <-signal:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatalf("timed out waiting for %s", name)
|
||||
}
|
||||
}
|
||||
|
||||
func capacityInt(value int) *int {
|
||||
return &value
|
||||
}
|
||||
|
||||
func closedCapacityChannel() <-chan struct{} {
|
||||
channel := make(chan struct{})
|
||||
close(channel)
|
||||
return channel
|
||||
}
|
||||
40
capacity_error.go
Normal file
40
capacity_error.go
Normal file
@@ -0,0 +1,40 @@
|
||||
package promptkit
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// CapacityError reports bounded admission rejected for a selected backend.
|
||||
//
|
||||
// Engine-produced values identify only rejection at Promptkit's bounded
|
||||
// [Engine.Run] or [Engine.RunPrepared] admission boundary. BackendID is the
|
||||
// normalized registered backend ID used for routing and capacity; endpoint
|
||||
// overrides do not change it. Every engine-produced value is nonnil and has a
|
||||
// nonblank BackendID. Provider errors, active-generation waiting, and caller
|
||||
// cancellation are not represented by this type.
|
||||
//
|
||||
// Callers own returned values and may mutate BackendID without affecting engine
|
||||
// state or another error. CapacityError and its default Go encoding have no
|
||||
// stable JSON contract. Consumer-constructed values do not establish that an
|
||||
// engine rejected work.
|
||||
type CapacityError struct {
|
||||
// BackendID is the normalized registered backend ID whose admission was
|
||||
// rejected.
|
||||
BackendID string
|
||||
}
|
||||
|
||||
// Error returns diagnostic wording that is not a parsing contract. It is safe
|
||||
// to call on a nil receiver or a value with a blank BackendID.
|
||||
func (e *CapacityError) Error() string {
|
||||
if e == nil || strings.TrimSpace(e.BackendID) == "" {
|
||||
return ErrCapacityExceeded.Error()
|
||||
}
|
||||
return fmt.Sprintf("backend %q admission: %v", e.BackendID, ErrCapacityExceeded)
|
||||
}
|
||||
|
||||
// Unwrap returns ErrCapacityExceeded so errors.Is and errors.As can be used
|
||||
// together. It is safe to call on a nil receiver or a zero value.
|
||||
func (e *CapacityError) Unwrap() error {
|
||||
return ErrCapacityExceeded
|
||||
}
|
||||
53
convert.go
53
convert.go
@@ -4,6 +4,7 @@ import (
|
||||
"reflect"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/jsonvalue"
|
||||
)
|
||||
|
||||
func toDomainRunRequest(req RunRequest) (domain.RunRequest, error) {
|
||||
@@ -15,12 +16,12 @@ func toDomainRunRequest(req RunRequest) (domain.RunRequest, error) {
|
||||
PromptID: req.PromptID,
|
||||
PromptVersion: req.PromptVersion,
|
||||
ProfileID: req.ProfileID,
|
||||
SessionID: req.SessionID,
|
||||
APIKey: req.APIKey,
|
||||
Inputs: toDomainArtifactRefMap(req.Inputs),
|
||||
Vars: copyStringMap(req.Vars),
|
||||
Execution: execution,
|
||||
Validation: toDomainOutputContractPtr(req.Validation),
|
||||
Metadata: copyStringMap(req.Metadata),
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -33,6 +34,7 @@ func fromDomainPreparedRun(prepared *domain.PreparedRun) *PreparedRun {
|
||||
PromptVersion: prepared.PromptVersion,
|
||||
PromptHash: prepared.PromptHash,
|
||||
SelectedProfileID: prepared.SelectedProfileID,
|
||||
SelectedBackendID: prepared.SelectedBackendID,
|
||||
EffectiveModelParams: fromDomainExecutionTarget(prepared.EffectiveModelParams),
|
||||
OutputContract: fromDomainOutputContract(prepared.OutputContract),
|
||||
StructuredOutput: fromDomainStructuredOutputSpec(prepared.StructuredOutput),
|
||||
@@ -58,8 +60,10 @@ func fromDomainRunResult(result *domain.RunResult) *RunResult {
|
||||
PromptID: result.PromptID,
|
||||
PromptVersion: result.PromptVersion,
|
||||
PromptHash: result.PromptHash,
|
||||
SessionID: result.SessionID,
|
||||
RenderedPromptHash: result.RenderedPromptHash,
|
||||
SelectedProfileID: result.SelectedProfileID,
|
||||
SelectedBackendID: result.SelectedBackendID,
|
||||
ModelName: result.ModelName,
|
||||
Endpoint: result.Endpoint,
|
||||
EffectiveModelParams: fromDomainExecutionTarget(result.EffectiveModelParams),
|
||||
@@ -132,7 +136,7 @@ func toDomainExecutionTargetOverride(override *ExecutionTargetOverride) (*domain
|
||||
if override == nil {
|
||||
return nil, nil
|
||||
}
|
||||
extraParams, err := copyPublicJSONMap(override.ExtraParams)
|
||||
extraParams, err := jsonvalue.CopyMap(override.ExtraParams)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -144,7 +148,7 @@ func toDomainExecutionTargetOverride(override *ExecutionTargetOverride) (*domain
|
||||
TopP: copyFloat64Ptr(override.TopP),
|
||||
TimeoutSeconds: copyIntPtr(override.TimeoutSeconds),
|
||||
ServiceTier: override.ServiceTier,
|
||||
ReasoningEffort: override.ReasoningEffort,
|
||||
ReasoningEffort: copyStringPtr(override.ReasoningEffort),
|
||||
APIKeyEnv: override.APIKeyEnv,
|
||||
ExtraParams: extraParams,
|
||||
}, nil
|
||||
@@ -152,6 +156,7 @@ func toDomainExecutionTargetOverride(override *ExecutionTargetOverride) (*domain
|
||||
|
||||
func fromDomainExecutionTarget(target domain.ExecutionTarget) ExecutionTarget {
|
||||
return ExecutionTarget{
|
||||
BackendID: target.BackendID,
|
||||
Endpoint: target.Endpoint,
|
||||
Model: target.Model,
|
||||
Temperature: target.Temperature,
|
||||
@@ -165,6 +170,40 @@ func fromDomainExecutionTarget(target domain.ExecutionTarget) ExecutionTarget {
|
||||
}
|
||||
}
|
||||
|
||||
func fromDomainProfileInspection(inspection *domain.ProfileInspection) *ProfileInspection {
|
||||
if inspection == nil {
|
||||
return nil
|
||||
}
|
||||
return &ProfileInspection{
|
||||
ProfileID: inspection.ProfileID,
|
||||
EffectiveModelParams: fromDomainExecutionTarget(inspection.EffectiveModelParams),
|
||||
APIKeyRequired: inspection.APIKeyRequired,
|
||||
}
|
||||
}
|
||||
|
||||
func fromDomainPromptInspection(inspection *domain.PromptInspection) *PromptInspection {
|
||||
if inspection == nil {
|
||||
return nil
|
||||
}
|
||||
inputs := make([]PromptInputDefinition, len(inspection.Inputs))
|
||||
for i, input := range inspection.Inputs {
|
||||
inputs[i] = PromptInputDefinition{
|
||||
Name: input.Name,
|
||||
Required: input.Required,
|
||||
ContentType: input.ContentType,
|
||||
Description: input.Description,
|
||||
}
|
||||
}
|
||||
return &PromptInspection{
|
||||
PromptID: inspection.PromptID,
|
||||
PromptVersion: inspection.PromptVersion,
|
||||
PromptHash: inspection.PromptHash,
|
||||
DefaultProfileID: inspection.DefaultProfileID,
|
||||
Inputs: inputs,
|
||||
OutputContract: fromDomainOutputContract(inspection.OutputContract),
|
||||
}
|
||||
}
|
||||
|
||||
func fromDomainExecutionTargetPresence(presence domain.ExecutionTargetPresence) ExecutionTargetPresence {
|
||||
return ExecutionTargetPresence{
|
||||
Temperature: presence.Temperature,
|
||||
@@ -397,6 +436,14 @@ func copyFloat64Ptr(src *float64) *float64 {
|
||||
return &v
|
||||
}
|
||||
|
||||
func copyStringPtr(src *string) *string {
|
||||
if src == nil {
|
||||
return nil
|
||||
}
|
||||
v := *src
|
||||
return &v
|
||||
}
|
||||
|
||||
func copyIntPtr(src *int) *int {
|
||||
if src == nil {
|
||||
return nil
|
||||
|
||||
64
doc.go
64
doc.go
@@ -1,8 +1,64 @@
|
||||
// Package promptkit provides an embeddable engine for preparing and executing
|
||||
// prompt-defined LLM workflows.
|
||||
//
|
||||
// Applications construct an Engine with NewEngine, select filesystem or
|
||||
// in-memory definition sources with options, and use Prepare or Run to execute
|
||||
// requests. Concrete repositories, validators, and outbound clients remain
|
||||
// internal implementation details.
|
||||
// Applications construct an [Engine] with [NewEngine], select filesystem or
|
||||
// in-memory sources and optional engine-scoped [Backend] registrations, and
|
||||
// call [Engine.InspectPrompt], [Engine.InspectProfile], [Engine.Prepare],
|
||||
// [Engine.PrepareExecution], [Engine.Run], or [Engine.RunPrepared]. Concrete
|
||||
// registries, repositories, validators, and the built-in OpenAI-compatible
|
||||
// client remain internal implementation details.
|
||||
//
|
||||
// # Concurrency and ownership
|
||||
//
|
||||
// An Engine supports concurrent InspectPrompt, InspectProfile, Prepare,
|
||||
// PrepareExecution, Run, and RunPrepared calls. Engine-local backend policies
|
||||
// bound admitted Run and RunPrepared calls and model generations where
|
||||
// configured, while different backend pools and unlimited backends continue
|
||||
// independently. An injected [LLMClient] or [ArtifactReader] can therefore
|
||||
// still receive concurrent calls and must be safe for that use.
|
||||
//
|
||||
// NewEngine copies in-memory profiles and backend definitions. Prepare,
|
||||
// PrepareExecution, and Run copy request maps, slices, pointer values, and
|
||||
// JSON-compatible extra parameters before using them. InspectPrompt and
|
||||
// InspectProfile return copied inspection values. Returned values and values
|
||||
// passed to extension interfaces are likewise isolated from engine state.
|
||||
// Callers own those copies and may mutate them after the call that supplied or
|
||||
// returned them. Returned structured errors are likewise caller-owned and may
|
||||
// be mutated without affecting engine state or another error.
|
||||
//
|
||||
// # Security and sensitive data
|
||||
//
|
||||
// The default artifact reader treats [File] paths as caller-selected operating
|
||||
// system paths. It does not restrict them to an application root or impose an
|
||||
// inbound request-size policy. Promptkit is not an inbound request or
|
||||
// untrusted-input security boundary. Applications must validate and restrict
|
||||
// untrusted input before constructing a request, or install an [ArtifactReader]
|
||||
// that enforces their filesystem, authorization, and size policies.
|
||||
//
|
||||
// Rendered messages, input and output [Artifact] bodies, [RunResult.RawOutput],
|
||||
// and [ValidationResult.Errors] may contain sensitive data. Credential
|
||||
// exclusion and redaction do not sanitize those values. Applications and
|
||||
// injected collaborators are responsible for access control, retention,
|
||||
// logging, and secret handling appropriate to their data.
|
||||
//
|
||||
// # JSON
|
||||
//
|
||||
// Stable JSON representations are provided for [PreparedRun], [RunResult],
|
||||
// [Artifact], [ExecutionTarget], [OutputContract], [ValidationResult],
|
||||
// [TokenUsage], [RenderedPrompt], [RenderedMessage], [CacheControl],
|
||||
// [StructuredOutputSpec], [StructuredOutputJSONSpec], [GenerateRequest],
|
||||
// [GenerateResponse], [ExecutionTargetPresence], and the string value types
|
||||
// used by those values.
|
||||
//
|
||||
// Construction, inspection, handle, and error values, including [Config],
|
||||
// [Backend], [RunRequest], [ArtifactRef], [ExecutionTargetOverride], [Profile],
|
||||
// [OpenAICompatibleProfileConfig], [ProfileInspection],
|
||||
// [PromptInputDefinition], [PromptInspection], [PreparedExecution], and
|
||||
// [CapacityError], do not have stable JSON representations. Direct API keys
|
||||
// are nevertheless excluded from JSON for every public value.
|
||||
//
|
||||
// JSON timestamps use time.Time's RFC 3339 encoding and are omitted when zero.
|
||||
// PreparedRun and RunResult durations are encoded as integer milliseconds in
|
||||
// duration_ms and omitted when zero. Run IDs and all exposed hashes are opaque:
|
||||
// their spelling, length, character set, and algorithm are not API contracts.
|
||||
package promptkit
|
||||
|
||||
@@ -1,151 +1,431 @@
|
||||
# Package `promptkit`
|
||||
|
||||
Import path:
|
||||
## Purpose
|
||||
|
||||
This guide helps Go consumers assemble Promptkit and choose the main
|
||||
preparation or execution workflow. The declarations and GoDoc in the
|
||||
[root package](../../doc.go) own exact field, option, serialization,
|
||||
concurrency, ownership, failure, and cancellation semantics. The
|
||||
[framework format reference](../formats.md) owns prompt, profile, and schema
|
||||
file contracts.
|
||||
|
||||
Import the package as:
|
||||
|
||||
```go
|
||||
import "gitea.maximumdirect.net/eric/promptkit"
|
||||
```
|
||||
|
||||
Package `promptkit` is the supported Go contract for in-process prompt
|
||||
preparation and execution. The declarations and their GoDoc in the
|
||||
[root package](../../doc.go) own the exact API; this guide explains how the
|
||||
pieces are used together. The [framework format reference](../formats.md) owns
|
||||
prompt, profile, and schema file contracts.
|
||||
The following Go fragments are illustrative and omit surrounding package,
|
||||
import, and error-handling code. Use the maintained examples for complete
|
||||
programs.
|
||||
|
||||
## Engine Construction And Sources
|
||||
## Construct An Engine
|
||||
|
||||
Construct an engine with [`NewEngine`, `Config`, and
|
||||
`Option`](../../engine.go). `PromptDir` is required unless a prompt source
|
||||
option is supplied. `ProfileDir` optionally overlays built-in profiles, and an
|
||||
empty `SchemaDir` uses the current directory. `Timeout` is the transport-wide
|
||||
safety cap for the built-in OpenAI-compatible client. An optional `HTTPClient`
|
||||
is cloned; its positive timeout takes precedence.
|
||||
Create an engine with
|
||||
[`NewEngine`](../../engine.go). A directory-backed setup supplies a prompt
|
||||
directory and may supply profile and schema directories:
|
||||
|
||||
Nil options are ignored. Invalid construction, including a nil injected client
|
||||
or artifact reader, returns an error matching `ErrInvalidConfig`.
|
||||
```go
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{
|
||||
PromptDir: "prompts",
|
||||
ProfileDir: "profiles",
|
||||
SchemaDir: "schemas",
|
||||
})
|
||||
```
|
||||
|
||||
The [source options](../../engine.go) replace their matching directory source:
|
||||
|
||||
- `WithPromptFS` and `WithPromptFile` select prompt definitions;
|
||||
- `WithProfileFS` and `WithProfileFile` overlay built-in profiles;
|
||||
- `WithProfiles` adds in-memory profiles ahead of file and built-in profiles;
|
||||
- `WithSchemaFS` and `WithSchemaFile` select JSON Schema documents;
|
||||
- `WithLLMClient` replaces the built-in model client; and
|
||||
- `WithArtifactReader` replaces the default reader for every input.
|
||||
|
||||
Source selection, path resolution, strict decoding, profile overlays, and
|
||||
file-to-request precedence are defined in the
|
||||
Options support single-file or `fs.FS` sources, in-memory profiles,
|
||||
engine-scoped backends, and injected artifact or model clients. Consult the
|
||||
[constructor and option GoDoc](../../engine.go) for composition, precedence,
|
||||
validation, and default transport behavior. Source discovery, format
|
||||
validation, and profile precedence are defined by the
|
||||
[framework format reference](../formats.md).
|
||||
|
||||
Per-generation timeout values from profiles or requests are independent of
|
||||
the transport cap and caller context. An explicit request value of zero
|
||||
disables only the per-generation deadline. The
|
||||
[outbound integration contract](../integrations/openai-compatible-chat.md#timeout-and-cancellation)
|
||||
defines the complete timeout layering.
|
||||
## Supply Embedded Application Defaults
|
||||
|
||||
## Preparation And Execution
|
||||
Use `WithFallbackProfileFS` when an application packages profile definitions
|
||||
that should apply unless an operator provides an ordinary configured profile
|
||||
with the same ID. For example, an application can embed its defaults while
|
||||
continuing to use `ProfileDir` for operator overrides:
|
||||
|
||||
[`Engine.Prepare` and `Engine.Run`](../../engine.go) accept the public
|
||||
[`RunRequest`](../../types.go). `Prepare` resolves the prompt, profile, input
|
||||
artifacts, validation contract, and rendered messages without calling an LLM.
|
||||
`Run` performs the same preparation, calls the configured client, and validates
|
||||
the generated content. The maintained
|
||||
```go
|
||||
//go:embed profiles/*.yaml
|
||||
var applicationProfiles embed.FS
|
||||
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{
|
||||
PromptDir: "prompts",
|
||||
ProfileDir: operatorProfileDir,
|
||||
},
|
||||
promptkit.WithFallbackProfileFS(applicationProfiles, "profiles"),
|
||||
)
|
||||
```
|
||||
|
||||
Keep application-owned profile IDs and definitions in the embedded source.
|
||||
Use the ordinary configured profile source for operator overrides. Leave
|
||||
`operatorProfileDir` empty when the operator did not configure an override
|
||||
directory; a non-empty path names an authoritative higher-precedence source,
|
||||
so an unavailable or unreadable directory is an error rather than a reason to
|
||||
fall back. The
|
||||
[framework format reference](../formats.md#source-and-profile-precedence)
|
||||
owns the exact profile format and lookup order; the
|
||||
[`WithFallbackProfileFS` GoDoc](../../engine.go) owns its option contract and
|
||||
validation rules.
|
||||
|
||||
## Inspect A Prompt Before Preparation
|
||||
|
||||
Use [`Engine.InspectPrompt`](../../engine.go) to check one configured prompt's
|
||||
declared inputs and output workflow without creating placeholder inputs or
|
||||
resolving a profile:
|
||||
|
||||
```go
|
||||
inspection, err := engine.InspectPrompt(ctx, "meeting.summary", "")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, input := range inspection.Inputs {
|
||||
// Compare the declared input with application configuration.
|
||||
}
|
||||
```
|
||||
|
||||
Use this configuration-time boundary when the application needs only the
|
||||
declared prompt interface. Use `InspectProfile` separately when it must also
|
||||
check a configured profile. Use `Prepare` when it needs inputs, schemas, or
|
||||
rendered messages, and use prepared execution when that work must remain tied
|
||||
to later execution. The method's [GoDoc](../../engine.go) owns exact fields,
|
||||
hash, ownership, and error semantics.
|
||||
|
||||
## Prepare Without Model Execution
|
||||
|
||||
[`Engine.Prepare`](../../engine.go) resolves the selected prompt and profile,
|
||||
loads inputs and any structured-output schema, and renders messages without
|
||||
calling a model client. Choose it when the prepared value is the final
|
||||
inspection or persistence result and no later execution must be tied to that
|
||||
exact snapshot:
|
||||
|
||||
```go
|
||||
prepared, err := engine.Prepare(ctx, promptkit.RunRequest{
|
||||
PromptID: "meeting.summary",
|
||||
Inputs: map[string]promptkit.ArtifactRef{
|
||||
"note": promptkit.Inline("Synthetic meeting notes"),
|
||||
},
|
||||
})
|
||||
```
|
||||
|
||||
The maintained
|
||||
[offline preparation example](../../examples/go-library/prepare/main.go)
|
||||
provides a complete runnable workflow using a prompt file, in-memory profile,
|
||||
and inline input.
|
||||
shows a complete runnable setup with a prompt file, in-memory profile, and
|
||||
inline input. Exact request requirements and prepared-result fields belong to
|
||||
the [`RunRequest` and `PreparedRun` GoDoc](../../types.go).
|
||||
|
||||
[`PreparedRun` and `RunResult`](../../types.go) expose copied public values.
|
||||
Preparation returns effective settings, hashes, rendered messages, selected
|
||||
profile, structured-output information, and timing without resolved secrets or
|
||||
model output. Execution adds the generated artifact and raw output, validation
|
||||
state, model metadata, usage, run ID, and duration.
|
||||
## Prepare Now And Execute The Same Snapshot Later
|
||||
|
||||
A generated-content validation failure returns a result with
|
||||
`Validation.Status == ValidationFailed`. An inability to perform validation
|
||||
returns an error matching `ErrValidation`.
|
||||
Use [`Engine.PrepareExecution`](../../engine.go) when an application must
|
||||
inspect or persist preflight details before deciding whether to start model
|
||||
work, while ensuring that later execution uses those exact rendered messages,
|
||||
inputs, target settings, and validation resources:
|
||||
|
||||
## Requests, Inputs, And Overrides
|
||||
```go
|
||||
preparedExecution, err := engine.PrepareExecution(ctx, promptkit.RunRequest{
|
||||
PromptID: "meeting.summary",
|
||||
Inputs: map[string]promptkit.ArtifactRef{
|
||||
"note": promptkit.Inline("Synthetic meeting notes"),
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer preparedExecution.Discard()
|
||||
|
||||
The [request and value declarations](../../types.go) own the available fields,
|
||||
serialized constants, and result shapes. Use `File`, `Inline`, or
|
||||
`InlineWithURI` to construct artifact references. The
|
||||
[framework format reference](../formats.md) defines declared inputs, template
|
||||
references, output contracts, and the relationship between file values and
|
||||
request overrides.
|
||||
details := preparedExecution.Details()
|
||||
// Inspect or persist an application-selected safe subset of details.
|
||||
|
||||
`ExecutionTargetOverride` uses pointers for numeric settings so an explicit
|
||||
zero remains distinct from no override. `ExtraParams` accepts JSON-compatible
|
||||
strings, booleans, finite numbers, string-keyed objects, arrays or slices, and
|
||||
nil. Unsupported values, non-string map keys, non-finite numbers, and cycles
|
||||
match `ErrInvalidConfig` in profiles or `ErrInvalidRequest` in request
|
||||
overrides.
|
||||
result, err := engine.RunPrepared(ctx, preparedExecution)
|
||||
```
|
||||
|
||||
Returned requests, profiles, prepared values, results, artifacts, maps, and
|
||||
slices are isolated from internal engine state. Consumers and injected
|
||||
extensions should not retain or mutate values owned by another caller.
|
||||
Preparation does not call the model or reserve backend capacity.
|
||||
`RunPrepared` executes from the retained snapshot rather than reloading
|
||||
consumer sources. The handle is opaque in-process state, while `Details`
|
||||
contains rendered content and remains subject to the application's data
|
||||
handling policy. The
|
||||
[`PreparedExecution` and method GoDoc](../../prepared_execution.go) and
|
||||
[engine operation GoDoc](../../engine.go) own exact lifecycle, engine-binding,
|
||||
credential, cancellation, timing, and error semantics.
|
||||
|
||||
## Profiles And Credentials
|
||||
## Execute And Validate
|
||||
|
||||
[`OpenAICompatibleProfile`](../../profiles.go) constructs an ordinary
|
||||
in-memory profile for an OpenAI-compatible chat-completions endpoint.
|
||||
`WithProfiles` rejects duplicate IDs in one call and gives in-memory profiles
|
||||
precedence over explicit file sources and built-ins.
|
||||
[`Engine.Run`](../../engine.go) performs the same preparation, invokes the
|
||||
configured model client, classifies the generated artifact, and validates the
|
||||
content in one call. Choose it when the application does not need a preflight
|
||||
boundary tied to the eventual execution. A completed content check may return
|
||||
`ValidationFailed` in the result; an operational inability to validate returns
|
||||
an error.
|
||||
|
||||
Raw API keys do not belong in profiles. File-backed profiles may name an
|
||||
environment variable, while an in-memory profile can require a request key.
|
||||
A direct `RunRequest.APIKey` is request-scoped and takes precedence over an
|
||||
environment lookup for the built-in client. Profile fields, ranges, built-ins,
|
||||
precedence, and credential rules are owned by the
|
||||
[framework format reference](../formats.md).
|
||||
The maintained
|
||||
[offline execution example](../../examples/go-library/run/main.go) injects a
|
||||
deterministic model client and exercises `Run` without credentials, network
|
||||
access, or paid calls. It is intentionally separate from the preparation
|
||||
example so each workflow and its small prompt fixture can be copied and run on
|
||||
its own.
|
||||
|
||||
API keys are excluded from JSON, prepared values, and results. The public
|
||||
`String` and `GoString` methods report only whether a direct key is present.
|
||||
Avoid reflection-based dumps of request structs, which can bypass that
|
||||
redaction.
|
||||
Use the [`RunResult` and `ValidationResult` GoDoc](../../types.go) for the
|
||||
returned data and the `Engine.Run` GoDoc for failure and cancellation
|
||||
semantics. The
|
||||
[OpenAI-compatible integration contract](../integrations/openai-compatible-chat.md)
|
||||
owns the built-in client's outbound HTTP behavior.
|
||||
|
||||
## Inputs, Profiles, And Overrides
|
||||
|
||||
Use `File`, `Inline`, or `InlineWithURI` to construct request inputs. A request
|
||||
can select a profile explicitly or use the prompt's default profile, and can
|
||||
replace execution settings or the complete output contract.
|
||||
|
||||
The [public value GoDoc](../../types.go) defines nil, empty, zero, replacement,
|
||||
copy, and credential behavior. The
|
||||
[framework format reference](../formats.md) defines how those request values
|
||||
interact with prompt definitions, file-backed and application fallback
|
||||
profiles, built-ins, schemas, and framework defaults.
|
||||
|
||||
For programmatic profiles,
|
||||
[`OpenAICompatibleProfile`](../../profiles.go) converts ordinary
|
||||
OpenAI-compatible settings into a value accepted by `WithProfiles`.
|
||||
|
||||
### Inspect A Profile Before Prompt Work
|
||||
|
||||
Use [`Engine.InspectProfile`](../../engine.go) to validate one configured
|
||||
profile without constructing a synthetic prompt or placeholder inputs. It
|
||||
resolves the profile's effective target but does not prepare or execute a
|
||||
prompt:
|
||||
|
||||
```go
|
||||
inspection, err := engine.InspectProfile(ctx, profileID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
target := inspection.EffectiveModelParams
|
||||
if target.APIKeyEnv != "" {
|
||||
// Apply application policy for the named environment variable.
|
||||
} else if inspection.APIKeyRequired {
|
||||
// Arrange a direct credential before later execution.
|
||||
}
|
||||
```
|
||||
|
||||
Use this configuration-time boundary when only the profile and its target need
|
||||
checking. Use `Prepare` when the application also needs prompt, input, schema,
|
||||
or rendering work; use prepared execution when that work must remain tied to a
|
||||
later execution. Inspection reports credential requirements but leaves the
|
||||
timing of credential enforcement to the application. The method's
|
||||
[GoDoc](../../engine.go) owns its exact result and error contract.
|
||||
|
||||
### Set A Per-Run Session And Reasoning
|
||||
|
||||
Supply a direct session ID when one prompt should be correlated with a
|
||||
consumer-managed conversation or workflow without changing prompt variables:
|
||||
|
||||
```go
|
||||
reasoning := "high"
|
||||
result, err := engine.Run(ctx, promptkit.RunRequest{
|
||||
PromptID: "meeting.summary",
|
||||
SessionID: "conversation-42",
|
||||
Inputs: map[string]promptkit.ArtifactRef{
|
||||
"note": promptkit.Inline("Synthetic meeting notes"),
|
||||
},
|
||||
Execution: &promptkit.ExecutionTargetOverride{
|
||||
ReasoningEffort: &reasoning,
|
||||
},
|
||||
})
|
||||
```
|
||||
|
||||
A nil reasoning pointer inherits the selected profile, a pointer to a
|
||||
nonblank string replaces it, and a pointer to a blank string disables
|
||||
reasoning for that run. Session IDs are correlation metadata, not credentials;
|
||||
use stable, non-secret values that are safe to expose to collaborators and
|
||||
providers. The
|
||||
[`RunRequest` and `ExecutionTargetOverride` GoDoc](../../types.go) owns the
|
||||
exact normalization, precedence, error, copying, and exposure contract.
|
||||
|
||||
### Configure A Local OpenAI-Compatible Endpoint
|
||||
|
||||
Choose the smallest configuration that fits how the endpoint will be reused.
|
||||
|
||||
#### Use An Endpoint-Only Profile
|
||||
|
||||
Put the endpoint directly on an in-memory profile when only that profile needs
|
||||
it and shared backend identity or capacity policy is unnecessary:
|
||||
|
||||
```go
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{
|
||||
PromptDir: "prompts",
|
||||
},
|
||||
promptkit.WithProfiles(promptkit.Profile{
|
||||
ID: "local-summary",
|
||||
Endpoint: "http://localhost:8000/v1",
|
||||
Model: "example-model",
|
||||
}),
|
||||
)
|
||||
```
|
||||
|
||||
Endpoint-only profiles have an empty backend ID and remain unrestricted by
|
||||
backend capacity policy.
|
||||
|
||||
#### Use The Conventional Local Backend
|
||||
|
||||
Use `LocalBackend` when profiles should share the conventional `local`
|
||||
identity, endpoint, and concurrency limit:
|
||||
|
||||
```go
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{
|
||||
PromptDir: "prompts",
|
||||
},
|
||||
promptkit.WithBackend(
|
||||
promptkit.LocalBackend("http://localhost:8000/v1", 2),
|
||||
),
|
||||
promptkit.WithProfiles(promptkit.Profile{
|
||||
ID: "local-summary",
|
||||
BackendID: promptkit.BackendLocal,
|
||||
Model: "example-model",
|
||||
}),
|
||||
)
|
||||
```
|
||||
|
||||
The helper is explicit: it does not pre-register a backend or read environment
|
||||
variables. Supplying a positive limit leaves queue capacity omitted, so normal
|
||||
backend registration selects the existing default waiting capacity of 1024.
|
||||
The returned value still enters the engine through `WithBackend`.
|
||||
|
||||
#### Configure A Complete Backend
|
||||
|
||||
Use a keyed `Backend` value for authentication, extra request parameters, an
|
||||
explicit queue capacity, a custom ID, or multiple local endpoints:
|
||||
|
||||
```go
|
||||
noWaiting := 0
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{
|
||||
PromptDir: "prompts",
|
||||
},
|
||||
promptkit.WithBackend(promptkit.Backend{
|
||||
ID: "local-gpu",
|
||||
Endpoint: "http://gpu-host:8000/v1",
|
||||
APIKeyEnv: "LOCAL_GPU_API_KEY",
|
||||
ExtraParams: map[string]any{"provider_option": "enabled"},
|
||||
ConcurrencyLimit: 2,
|
||||
QueueCapacity: &noWaiting,
|
||||
}),
|
||||
promptkit.WithProfiles(promptkit.Profile{
|
||||
ID: "gpu-summary",
|
||||
BackendID: "local-gpu",
|
||||
Model: "example-model",
|
||||
}),
|
||||
)
|
||||
```
|
||||
|
||||
Use distinct custom IDs when registering multiple local endpoints.
|
||||
Registrations belong to one engine and custom IDs cannot replace built-ins.
|
||||
The [`Backend`, `LocalBackend`, and `WithBackend` GoDoc](../../backends.go)
|
||||
defines exact construction, validation, copying, uniqueness, concurrency, and
|
||||
request-default behavior.
|
||||
|
||||
Both file-backed and in-memory profiles select a registration through
|
||||
`backend` or `Profile.BackendID`. Profile and request endpoint overrides retain
|
||||
that routing and capacity identity. `PreparedRun.SelectedBackendID`,
|
||||
`RunResult.SelectedBackendID`, and the effective `ExecutionTarget.BackendID`
|
||||
expose it to consumers and injected model clients. Endpoint-only profiles
|
||||
remain supported and expose an empty backend ID.
|
||||
|
||||
### Limit Backend Concurrency
|
||||
|
||||
Set `Backend.ConcurrencyLimit` when a backend needs protection from too many
|
||||
simultaneous model calls. Leaving `QueueCapacity` nil, as in the local-backend
|
||||
example above, selects the default waiting capacity of 1024.
|
||||
|
||||
To accept no waiting backlog beyond the active calls, provide an explicit
|
||||
zero:
|
||||
|
||||
```go
|
||||
noWaiting := 0
|
||||
backend := promptkit.Backend{
|
||||
ID: "local-gpu",
|
||||
Endpoint: "http://gpu-host:8000/v1",
|
||||
ConcurrencyLimit: 2,
|
||||
QueueCapacity: &noWaiting,
|
||||
}
|
||||
```
|
||||
|
||||
The pointer distinguishes an explicit zero from omission. Keep using keyed
|
||||
`Backend` literals so additive configuration fields remain source-compatible.
|
||||
Capacity belongs to one engine and the selected backend ID; endpoint-only
|
||||
profiles and custom backends without a configured limit remain unrestricted.
|
||||
Exact validation, defaulting, ownership, and concurrency semantics belong to
|
||||
the [`Backend` GoDoc](../../backends.go).
|
||||
|
||||
## Credentials
|
||||
|
||||
File-backed profiles name an environment variable; in-memory profiles can
|
||||
require a direct request key. Direct keys are request-scoped and are excluded
|
||||
from supported JSON values and the package's `String` and `GoString`
|
||||
summaries. The exact precedence and redaction guarantees belong to
|
||||
[`RunRequest`, `GenerateRequest`, and the profile GoDoc](../../types.go).
|
||||
|
||||
## Protect Files And Generated Data
|
||||
|
||||
The default artifact reader opens a `File` reference as a caller-selected
|
||||
operating-system path. It does not constrain paths to an application root,
|
||||
impose an inbound request-size policy, or establish an untrusted-input security
|
||||
boundary. Applications must validate and restrict untrusted paths and payloads
|
||||
before constructing a request, or inject an artifact reader that enforces
|
||||
their filesystem, authorization, and size policies.
|
||||
|
||||
Rendered messages, input and output artifact bodies, raw model output, and
|
||||
validation diagnostics can contain sensitive data. API-key redaction does not
|
||||
sanitize those values. Treat prepared values, results, collaborator requests,
|
||||
errors, and logs according to the application's data-access, retention, and
|
||||
secret-handling policies.
|
||||
|
||||
## Extension Interfaces
|
||||
|
||||
The [`LLMClient`, `GenerateRequest`, and
|
||||
`GenerateResponse`](../../types.go) boundary lets a consumer replace model
|
||||
generation. Injected clients receive copied rendered messages, effective
|
||||
settings, explicit numeric-setting presence, structured-output constraints,
|
||||
and the request-scoped key. They return generated content and token usage.
|
||||
Inject an [`LLMClient` or `ArtifactReader`](../../types.go) when the built-in
|
||||
behavior does not fit the application. Their GoDoc defines concurrent use,
|
||||
context handling, ownership of copied values, nil responses, and preservation
|
||||
of collaborator errors. Implementations must honor cancellation, safely manage
|
||||
copies they retain, avoid unsafe logging of content or credentials, and enforce
|
||||
the application policy that motivated the injection.
|
||||
|
||||
The [`ArtifactReader`](../../types.go) boundary replaces the default inline and
|
||||
file reader for every input. Readers provide artifact content and metadata; the
|
||||
engine fills an empty artifact name from the input-map key. A reader error
|
||||
matches `ErrArtifactLoad` while preserving the original identity for
|
||||
`errors.Is`. A nil artifact with a nil error is also an artifact-load failure.
|
||||
## Handle Errors
|
||||
|
||||
Extensions should honor context cancellation and avoid logging raw prompts,
|
||||
artifacts, or credentials.
|
||||
Use `errors.Is` with the
|
||||
[public error sentinels and operation GoDoc](../../engine.go). The declarations
|
||||
distinguish invalid construction, invalid requests, absent sources,
|
||||
source-loading failures, collaborator failures, and operational validation
|
||||
failures. Specific request conditions may also match the broader
|
||||
`ErrInvalidRequest`, and injected collaborator identities are preserved where
|
||||
documented. Invalid or duplicate backend registrations match
|
||||
`ErrInvalidConfig`; selecting an unknown backend matches `ErrProfileLoad`.
|
||||
|
||||
## Errors
|
||||
When a limited backend has admitted all active and waiting calls, handle
|
||||
`ErrCapacityExceeded` separately from request errors and provider failures:
|
||||
|
||||
The [public error declarations](../../engine.go) and
|
||||
[mapping](../../errors.go) preserve these sentinel checks through `errors.Is`:
|
||||
```go
|
||||
result, err := engine.Run(ctx, request)
|
||||
if errors.Is(err, promptkit.ErrCapacityExceeded) {
|
||||
var capacityErr *promptkit.CapacityError
|
||||
if errors.As(err, &capacityErr) {
|
||||
// Record capacityErr.BackendID using application-owned diagnostics.
|
||||
}
|
||||
|
||||
- `ErrInvalidConfig`
|
||||
- `ErrInvalidRequest`
|
||||
- `ErrPromptNotFound`
|
||||
- `ErrProfileNotFound`
|
||||
- `ErrProfileRequired`
|
||||
- `ErrPromptLoad`
|
||||
- `ErrProfileLoad`
|
||||
- `ErrAPIKeyEnvMissing`
|
||||
- `ErrArtifactLoad`
|
||||
- `ErrPromptRender`
|
||||
- `ErrLLMGenerate`
|
||||
- `ErrValidation`
|
||||
// Apply application policy: shed work, report overload, or retry later.
|
||||
}
|
||||
```
|
||||
|
||||
`ErrProfileRequired` and `ErrAPIKeyEnvMissing` also match
|
||||
`ErrInvalidRequest`, allowing either broad request handling or a specific
|
||||
condition. Wrapped collaborator errors retain their identity where the public
|
||||
contract promises it.
|
||||
A rejected call returns no partial result and does not invoke the model
|
||||
client. Promptkit does not prescribe retries or map this error to an HTTP
|
||||
status; those choices remain with the consuming application. The
|
||||
[`CapacityError` GoDoc](../../capacity_error.go) owns the exact typed-error
|
||||
contract, while the [`Engine.Run` and error GoDoc](../../engine.go) owns broad
|
||||
error and cancellation identities.
|
||||
|
||||
## Consumer Boundary
|
||||
## Application Boundary
|
||||
|
||||
Promptkit is an importable library. It does not own a command, inbound HTTP
|
||||
API, process configuration, or deployment policy. Scriptorium is one
|
||||
downstream application that maps this root package contract into those
|
||||
application concerns.
|
||||
API, process configuration, or deployment policy. Applications map the root
|
||||
package's results and errors into those concerns, including inbound size and
|
||||
trust policy.
|
||||
|
||||
@@ -44,98 +44,3 @@ Start with:
|
||||
For cross-cutting changes, follow every applicable row. Do not create
|
||||
placeholder documents for packages, APIs, or integrations that do not yet
|
||||
exist.
|
||||
|
||||
## Maintainer-Run Validation
|
||||
|
||||
Promptkit does not currently use hosted CI. Maintainers are responsible for
|
||||
running the documented checks before accepting changes. Run the default Go
|
||||
validation from the Promptkit repository root:
|
||||
|
||||
```sh
|
||||
go test ./...
|
||||
go test -race ./...
|
||||
go vet ./...
|
||||
go build ./...
|
||||
go run ./examples/go-library/prepare
|
||||
```
|
||||
|
||||
Check formatting across every tracked Go file:
|
||||
|
||||
```sh
|
||||
gofmt -l $(git ls-files '*.go')
|
||||
```
|
||||
|
||||
The formatting command must produce no paths. Follow every added or changed
|
||||
Markdown link and confirm its target exists. Finally, check whitespace:
|
||||
|
||||
```sh
|
||||
git diff --check
|
||||
```
|
||||
|
||||
Documentation-only work does not require unrelated new tests, but it still
|
||||
requires link validation and `git diff --check`. Run the Go validation whenever
|
||||
documentation changes commands, examples, generated output, or another
|
||||
behavior checked by the module.
|
||||
|
||||
## Focused Validation
|
||||
|
||||
Use focused checks while iterating, then run the complete validation sequence
|
||||
before accepting the change. The root package supports:
|
||||
|
||||
```sh
|
||||
go test .
|
||||
go vet .
|
||||
go build .
|
||||
```
|
||||
|
||||
Filter tests by name without assuming a fixed internal package layout:
|
||||
|
||||
```sh
|
||||
go test ./... -run 'TestName'
|
||||
```
|
||||
|
||||
Replace `TestName` with a useful regular expression. Target only paths that
|
||||
exist, and consult the internal component overview for their owning
|
||||
documentation. A filtered or package-specific run does not replace the
|
||||
complete repository validation.
|
||||
|
||||
## Coordinated Work With Scriptorium
|
||||
|
||||
Promptkit and Scriptorium must remain independently valid. For temporary local
|
||||
integration, use either a Go workspace outside both repositories or an
|
||||
uncommitted replacement in the consuming module.
|
||||
|
||||
If the repositories are sibling directories, run the workspace commands from
|
||||
their parent directory:
|
||||
|
||||
```sh
|
||||
go work init ./promptkit ./scriptorium
|
||||
go work sync
|
||||
```
|
||||
|
||||
Use the workspace only for coordinated local checks. From the same parent
|
||||
directory, remove it when finished:
|
||||
|
||||
```sh
|
||||
rm -f go.work go.work.sum
|
||||
```
|
||||
|
||||
Alternatively, from the Scriptorium repository root, temporarily point its
|
||||
Promptkit dependency at the sibling checkout:
|
||||
|
||||
```sh
|
||||
go mod edit -replace gitea.maximumdirect.net/eric/promptkit=../promptkit
|
||||
```
|
||||
|
||||
After coordinated checks, remove the replacement and reconcile module
|
||||
metadata:
|
||||
|
||||
```sh
|
||||
go mod edit -dropreplace gitea.maximumdirect.net/eric/promptkit
|
||||
go mod tidy
|
||||
```
|
||||
|
||||
Never commit `go.work`, `go.work.sum`, or a local filesystem `replace`
|
||||
directive. Before committing in either repository, inspect its module files and
|
||||
working tree independently. Published consumer versions must depend on a tagged
|
||||
Promptkit version, not a workspace, local replacement, or unpublished commit.
|
||||
|
||||
@@ -58,6 +58,11 @@ When a request omits a version, the selected prompt ID must identify exactly
|
||||
one definition. When it supplies a version, the ID and version pair must be
|
||||
unique.
|
||||
|
||||
Exact prompt inspection uses this same configured source, strict decoding,
|
||||
referenced content-file resolution, and ID/version selection. It reports the
|
||||
selected definition's declared metadata without changing the prompt format or
|
||||
executing the definition.
|
||||
|
||||
### Inputs
|
||||
|
||||
Each `inputs` item has these fields:
|
||||
@@ -91,7 +96,8 @@ of a named input. Missing variables and input references are errors.
|
||||
|
||||
The optional `session_id` uses the same template data and input helper. Its
|
||||
rendered value is trimmed, omitted when empty, and limited to 256 Unicode code
|
||||
points.
|
||||
points. A nonblank direct request session ID bypasses this template completely;
|
||||
a blank direct value leaves the template behavior unchanged.
|
||||
|
||||
### Cache Control
|
||||
|
||||
@@ -136,7 +142,7 @@ A profile supplies model execution settings:
|
||||
|
||||
```yaml
|
||||
id: local-summary
|
||||
endpoint: http://localhost:8000/v1
|
||||
backend: openrouter
|
||||
model: example-model
|
||||
temperature: 0.2
|
||||
max_tokens: 500
|
||||
@@ -144,7 +150,6 @@ top_p: 0.95
|
||||
timeout_seconds: 90
|
||||
service_tier: flex
|
||||
reasoning_effort: medium
|
||||
api_key_env: EXAMPLE_API_KEY
|
||||
extra_params:
|
||||
provider_option: enabled
|
||||
```
|
||||
@@ -152,7 +157,8 @@ extra_params:
|
||||
| Field | Required | Meaning |
|
||||
| --- | --- | --- |
|
||||
| `id` | yes | Non-empty profile identifier. IDs must be unique within one source. |
|
||||
| `endpoint` | yes | Non-empty OpenAI-compatible base URL, including an API version path when required. |
|
||||
| `backend` | unless `endpoint` is present | Backend registry ID. It is trimmed and registry membership is checked when the profile is prepared or inspected. |
|
||||
| `endpoint` | unless `backend` is present | Non-empty OpenAI-compatible base URL, including an API version path when required. When both connection fields are present, this overrides the backend endpoint without changing backend identity. |
|
||||
| `model` | yes | Non-empty provider model name. |
|
||||
| `temperature` | no | Number from 0 through 2. |
|
||||
| `max_tokens` | no | Integer zero or greater. |
|
||||
@@ -166,6 +172,13 @@ extra_params:
|
||||
Raw `api_key` is prohibited in profile YAML. Store only an environment
|
||||
variable name in `api_key_env`.
|
||||
|
||||
Promptkit does not infer a backend from a model or endpoint. Endpoint-only
|
||||
profiles remain supported and have no effective backend ID.
|
||||
The engine always provides the built-in `openrouter` ID. Consumers can add
|
||||
engine-scoped IDs with
|
||||
[`WithBackend`](../backends.go); exact registration validation belongs to its
|
||||
GoDoc.
|
||||
|
||||
`extra_params` accepts null, booleans, finite numbers, strings, arrays, and
|
||||
objects with string keys. Keys must be non-empty. With the built-in client,
|
||||
they also cannot collide with the standard fields listed in the
|
||||
@@ -175,51 +188,72 @@ they also cannot collide with the standard fields listed in the
|
||||
|
||||
Execution settings resolve in this order:
|
||||
|
||||
1. framework defaults;
|
||||
2. the selected profile; and
|
||||
3. request `ExecutionTargetOverride` values.
|
||||
1. the framework timeout baseline;
|
||||
2. the selected backend, when the profile names one;
|
||||
3. the selected profile; and
|
||||
4. request `ExecutionTargetOverride` values.
|
||||
|
||||
The framework defaults are:
|
||||
The framework baseline is:
|
||||
|
||||
| Setting | Default |
|
||||
| --- | --- |
|
||||
| `temperature` | `0` |
|
||||
| `max_tokens` | `0` |
|
||||
| `top_p` | `1` |
|
||||
| `temperature` | Unspecified and omitted from compatible provider requests unless a profile or runtime override selects it. |
|
||||
| `max_tokens` | Unspecified and omitted from compatible provider requests unless a profile or runtime override selects it. |
|
||||
| `top_p` | Unspecified and omitted from compatible provider requests unless a profile or runtime override selects it. |
|
||||
| `timeout_seconds` | `600` |
|
||||
|
||||
Numeric zero in a file or in-memory profile means that the profile does not
|
||||
replace the framework default. Numeric request overrides use pointers, so an
|
||||
explicit zero is preserved. In particular, an explicit request
|
||||
`timeout_seconds` of zero disables the per-generation deadline while leaving
|
||||
the caller context and transport timeout intact.
|
||||
Numeric zero in a file or in-memory profile does not select a numeric value.
|
||||
For `temperature`, `max_tokens`, and `top_p`, it leaves the provider control
|
||||
unspecified. For `timeout_seconds`, it retains the framework deadline. Numeric
|
||||
request overrides use pointers, so an explicit zero is retained and sent to
|
||||
compatible providers. In particular, an explicit request `timeout_seconds` of
|
||||
zero disables the per-generation deadline while leaving the caller context and
|
||||
transport timeout intact.
|
||||
|
||||
Non-empty request strings replace profile strings. A non-empty request
|
||||
`ExtraParams` map replaces the profile map rather than merging keys.
|
||||
Non-empty profile strings replace backend defaults, and non-empty request
|
||||
strings replace both. Request reasoning is the exception: a nil
|
||||
`ReasoningEffort` pointer inherits the profile, a pointer to a nonblank string
|
||||
trims and replaces it, and a pointer to a blank string clears it. Backend
|
||||
identity is retained when either layer overrides the endpoint, so the override
|
||||
also retains any engine-local capacity policy configured for that backend.
|
||||
Capacity configuration belongs to the Go
|
||||
[`Backend` API](../backends.go), not prompt or profile YAML. A non-empty
|
||||
`extra_params` map at each layer replaces the entire lower-precedence map
|
||||
rather than merging keys.
|
||||
The [outbound integration contract](integrations/openai-compatible-chat.md)
|
||||
defines how the effective settings are serialized.
|
||||
|
||||
### Source And Profile Precedence
|
||||
|
||||
An explicit request profile ID takes precedence over the prompt's
|
||||
`default_profile`. If neither is present, preparation fails.
|
||||
`default_profile`. If neither is present, preparation fails. Exact profile
|
||||
inspection instead takes one explicit profile ID and does not use a prompt
|
||||
default.
|
||||
|
||||
Profile sources resolve matching IDs in this order:
|
||||
|
||||
1. in-memory profiles supplied with `WithProfiles`;
|
||||
2. a profile file, `fs.FS`, or configured profile directory; and
|
||||
3. embedded built-in profiles.
|
||||
2. the ordinary configured source selected by a profile file, `fs.FS`, or
|
||||
configured profile directory;
|
||||
3. application fallback profiles supplied with `WithFallbackProfileFS`; and
|
||||
4. embedded built-in profiles.
|
||||
|
||||
A higher-precedence source falls back only when the profile is absent. An
|
||||
invalid matching profile is an error and does not fall back. In-memory
|
||||
`Profile` values follow the same ranges as YAML profiles. They use
|
||||
`APIKeyRequired` for request-scoped credentials instead of `api_key_env`.
|
||||
A profile source supplies a complete definition; definitions and their fields
|
||||
are not merged across sources. A higher-precedence source falls back only when
|
||||
the requested profile ID is absent. An invalid matching profile is an error and
|
||||
does not fall back. In-memory `Profile` values follow the same ranges as YAML
|
||||
profiles. They use `APIKeyRequired` for request-scoped credentials instead of
|
||||
`api_key_env`. Preparation and exact profile inspection use this same source
|
||||
precedence.
|
||||
|
||||
## Built-In Profile Catalog
|
||||
|
||||
Built-ins use the OpenRouter-compatible endpoint and
|
||||
`OPENROUTER_API_KEY`. A custom or in-memory profile with the same ID takes
|
||||
precedence.
|
||||
Every built-in selects the `openrouter` backend. The engine's built-in backend
|
||||
registry supplies `https://openrouter.ai/api/v1` and the environment-variable
|
||||
name `OPENROUTER_API_KEY`, so individual profiles contain only model and
|
||||
generation settings. Built-in profile files do not repeat those connection
|
||||
values. A configured, application fallback, or in-memory profile with the same
|
||||
profile ID takes precedence.
|
||||
|
||||
| Provider | ID | Model |
|
||||
| --- | --- | --- |
|
||||
@@ -270,6 +304,10 @@ prompt, profile, schema, or example files:
|
||||
- a request can provide a direct `APIKey` or override `APIKeyEnv`; and
|
||||
- a direct request key takes precedence over environment lookup.
|
||||
|
||||
After a direct request key, the credential-source precedence is request
|
||||
`APIKeyEnv`, profile `api_key_env`, then the backend default. An in-memory
|
||||
profile with `APIKeyRequired` clears an inherited backend environment name and
|
||||
requires a direct key unless the request explicitly supplies `APIKeyEnv`.
|
||||
Promptkit validates required credential availability during preparation.
|
||||
Direct keys are excluded from JSON results and redacted by public string
|
||||
formatters. Environment-variable names may appear in prepared metadata, but
|
||||
|
||||
@@ -13,10 +13,15 @@ that produce these outbound settings.
|
||||
## Endpoint And Method
|
||||
|
||||
Generation sends an HTTP `POST` with `Content-Type: application/json`.
|
||||
A non-empty endpoint from the execution target overrides the client's
|
||||
configured base URL. After trailing slashes are removed,
|
||||
`/chat/completions` is appended. Generation fails before sending when neither
|
||||
source supplies an endpoint.
|
||||
Before the client is called, the engine resolves framework, backend, profile,
|
||||
and request values into one execution target. A non-empty endpoint from that
|
||||
target overrides the client's configured base URL. After trailing slashes are
|
||||
removed, `/chat/completions` is appended. Generation fails before sending when
|
||||
neither source supplies an endpoint.
|
||||
|
||||
The target's backend ID is routing metadata for prepared values, results, and
|
||||
injected clients. The built-in client does not derive the URL from that ID and
|
||||
does not serialize it in the provider request.
|
||||
|
||||
## Authentication
|
||||
|
||||
@@ -26,6 +31,12 @@ the client reads that variable and requires a non-empty value. The selected
|
||||
key is sent as `Authorization: Bearer <key>`. No authorization header is sent
|
||||
when neither mechanism is configured.
|
||||
|
||||
The target contains the already resolved environment-variable name: an
|
||||
explicit request override takes precedence over profile metadata, which takes
|
||||
precedence over the backend default. Only the name reaches prepared metadata;
|
||||
the environment value is read just before the provider call and is never added
|
||||
to the JSON body.
|
||||
|
||||
## Request Body
|
||||
|
||||
The request body always contains `model` and `messages`. The execution
|
||||
@@ -36,20 +47,25 @@ Each ordinary message contains its `role` and string `content`. A
|
||||
cache-controlled message instead uses a text content block containing `type`,
|
||||
`text`, and `cache_control`; an empty cache-control TTL is omitted.
|
||||
|
||||
A non-empty session ID is trimmed, checked against the internal domain limit,
|
||||
and sent as top-level `session_id`. It is not sent as a session header.
|
||||
The effective direct or prompt-rendered session ID is trimmed, limited to 256
|
||||
Unicode code points, and sent when nonempty as top-level `session_id`. It is
|
||||
never also sent as a session header.
|
||||
|
||||
The client conditionally includes:
|
||||
|
||||
- `temperature`, `max_tokens`, and `top_p` when non-zero or explicitly
|
||||
present;
|
||||
- non-empty `service_tier` and `reasoning_effort`; and
|
||||
- `temperature`, `max_tokens`, and `top_p` only when selected by a profile or
|
||||
runtime override, including an explicit runtime zero; they are absent when
|
||||
unspecified;
|
||||
- non-empty `service_tier` and effective `reasoning_effort`; an explicitly
|
||||
disabled reasoning setting is empty and therefore omitted; and
|
||||
- `response_format` for JSON Schema structured output, including its name,
|
||||
strict flag, and schema document.
|
||||
|
||||
Extra parameters are merged directly into the top-level body after JSON
|
||||
serialization is verified. Empty keys and collisions with these reserved
|
||||
fields are rejected before any provider call:
|
||||
The engine resolves backend, profile, and request extra-parameter maps by
|
||||
whole-map replacement rather than key merging. The resulting effective map is
|
||||
then merged directly into the top-level body after JSON serialization is
|
||||
verified. Empty keys and collisions with these reserved fields are rejected
|
||||
before any provider call:
|
||||
|
||||
- `model`
|
||||
- `session_id`
|
||||
@@ -61,6 +77,9 @@ fields are rejected before any provider call:
|
||||
- `reasoning_effort`
|
||||
- `response_format`
|
||||
|
||||
`backend_id`, `api_key_env`, and resolved credential values are not provider
|
||||
request fields.
|
||||
|
||||
## Response Handling
|
||||
|
||||
Any 2xx response is decoded as an OpenAI-compatible chat response. The client
|
||||
|
||||
113
docs/internal/capacity.md
Normal file
113
docs/internal/capacity.md
Normal file
@@ -0,0 +1,113 @@
|
||||
# Internal Capacity Management
|
||||
|
||||
## Purpose
|
||||
|
||||
This document describes the implemented engine-local capacity coordination in
|
||||
`internal/capacity`. The [architecture policy](../policy/architecture.md) owns
|
||||
component boundaries, the [backend GoDoc](../../backends.go) owns exact public
|
||||
configuration semantics, and the
|
||||
[internal runner document](runner.md) owns orchestration around admission.
|
||||
|
||||
Capacity scheduling is outside the provider wire contract. It does not add
|
||||
fields to execution targets, generated requests, prompt or profile YAML, or
|
||||
stable JSON values.
|
||||
|
||||
## Construction And Pool Lifecycle
|
||||
|
||||
Each root `NewEngine` call obtains a normalized capacity-policy snapshot from
|
||||
its immutable backend registry and constructs a new `Manager`. The manager
|
||||
creates one pool for each limited backend ID. It has no package-global mutable
|
||||
state, background workers, shutdown protocol, or persistence, so engines with
|
||||
the same registrations still have independent capacity.
|
||||
|
||||
Unlimited registered backends and endpoint-only profiles have no pool. Their
|
||||
admission and generation calls take the unrestricted fast path. An endpoint
|
||||
override does not change the selected backend ID and therefore does not change
|
||||
the pool.
|
||||
|
||||
One pool owns immutable active and total limits plus mutex-protected admission
|
||||
count, active count, and ordered waiter list. Pool state exists only for the
|
||||
lifetime of its engine.
|
||||
|
||||
## Bounded Execution Admission
|
||||
|
||||
For ordinary `Run`, the runner asks the manager to admit after resolving the
|
||||
prompt, profile, selected backend, effective execution target, credentials, and
|
||||
output contract, but before schema loading, artifact loading, or rendering.
|
||||
`PrepareExecution` performs no admission. `RunPrepared` claims its handle,
|
||||
rechecks credential availability, and then asks the manager to admit the
|
||||
frozen backend before generation.
|
||||
|
||||
Admission is immediate: a limited pool either reserves a slot or returns only
|
||||
the internal `ErrCapacityExceeded` identity. The runner attaches the selected
|
||||
backend identity at its use-case boundary, and the root facade translates that
|
||||
typed value without treating it as an invalid request or generation failure.
|
||||
|
||||
The total admitted bound is the active-generation limit plus its configured
|
||||
waiting capacity. The returned release function is idempotent. The runner
|
||||
defers it as soon as admission succeeds. An ordinary run holds the lease across
|
||||
remaining preparation, initial generation, validation, every repair attempt,
|
||||
and all failure or cancellation exits. Prepared execution holds the normal
|
||||
lease across generation, validation, every internal repair attempt, and all
|
||||
execution exits. A repair is part of its original admission and does not
|
||||
reserve another bounded slot.
|
||||
|
||||
## FIFO Generation Permits
|
||||
|
||||
`NewClient` wraps the engine's selected internal model client after public
|
||||
client adaptation or built-in client construction. Initial generation and the
|
||||
default repairer receive the same wrapper.
|
||||
|
||||
For each `Generate` call, the wrapper selects a pool from the request's
|
||||
effective backend ID. An unlimited call passes directly to the next client. A
|
||||
limited call acquires an active permit, invokes the next client, and defers
|
||||
permit release so ordinary returns and panic unwinding both restore capacity.
|
||||
Preparation and validation never hold an active permit.
|
||||
|
||||
When all active permits are occupied, calls join a mutex-protected FIFO waiter
|
||||
list. Releasing a permit transfers it directly to the oldest remaining waiter
|
||||
before making it generally available. Pools do not order work relative to
|
||||
other backend IDs.
|
||||
|
||||
The wrapper passes generation requests, responses, and collaborator errors
|
||||
through unchanged. It owns scheduling only; the concrete model client remains
|
||||
responsible for provider transport behavior.
|
||||
|
||||
## Cancellation And Release
|
||||
|
||||
Admission checks the caller context before reserving a slot. A call canceled
|
||||
while waiting for an active permit removes its waiter under the same pool lock
|
||||
used to grant permits. If cancellation removes the waiter first, the wrapped
|
||||
client is not invoked. If a concurrent grant wins first, the call owns the
|
||||
permit and invokes the client with the original context, allowing the client
|
||||
to observe cancellation normally.
|
||||
|
||||
This grant-or-cancel decision prevents lost and double-released permits.
|
||||
Admission leases and active permits are released after success, collaborator
|
||||
errors, validation failures, cancellation, and panic unwinding. Canceled
|
||||
waiters are unlinked so their contexts and requests are not retained by the
|
||||
pool.
|
||||
|
||||
## Test Ownership
|
||||
|
||||
The [manager tests](../../internal/capacity/manager_test.go) own policy
|
||||
validation, bounded admission, idempotent release, context handling, and
|
||||
unlimited admission. The
|
||||
[client tests](../../internal/capacity/client_test.go) own peak enforcement,
|
||||
FIFO transfer, canceled-waiter removal, grant/cancel races, independent pools,
|
||||
unlimited calls, passthrough behavior, and panic release.
|
||||
|
||||
The [runner tests](../../internal/usecase/runner_test.go) own ordinary early
|
||||
admission, lease lifetime, failure release, and shared initial/repair
|
||||
scheduling. The
|
||||
[prepared-execution use-case tests](../../internal/usecase/prepared_execution_test.go)
|
||||
own deferred admission, credential ordering, and prepared-execution lease
|
||||
release. The
|
||||
[external package capacity tests](../../capacity_contract_test.go) own the
|
||||
assembled public-engine behavior for configured limits, capacity errors,
|
||||
endpoint identity, engine independence, and injected clients. The
|
||||
[prepared-execution contract tests](../../prepared_execution_contract_test.go)
|
||||
own the public prepared-capacity boundary. The
|
||||
[root error-boundary tests](../../errors_internal_test.go) own preservation of
|
||||
the public generation category and context identity when generation is
|
||||
canceled.
|
||||
@@ -20,6 +20,11 @@ orchestration. `OpenAICompatibleClient` is the built-in implementation. It
|
||||
uses internal domain values for rendered prompts, execution targets,
|
||||
structured output, responses, and token usage.
|
||||
|
||||
The runner supplies a fully resolved target after applying backend, profile,
|
||||
and request precedence. The client uses its endpoint, credential metadata,
|
||||
generation fields, and extra parameters. `BackendID` remains routing metadata
|
||||
for the generation boundary and is not mapped into the provider payload.
|
||||
|
||||
Construction validates the configured base URL and clones any supplied
|
||||
`http.Client` so Promptkit can apply its timeout default without mutating the
|
||||
caller's client. Generation then:
|
||||
@@ -31,9 +36,28 @@ caller's client. Generation then:
|
||||
5. performs the outbound request under the applicable deadlines; and
|
||||
6. decodes the first response choice and token usage.
|
||||
|
||||
`internal/llm` owns the set of reserved OpenAI-compatible request fields used
|
||||
when validating extra parameters. Backend registration consumes the same rule
|
||||
without making the model client depend on registry configuration.
|
||||
|
||||
The implementation has no retry loop, tool-call support, provider catalog,
|
||||
inbound HTTP behavior, or durable session store.
|
||||
|
||||
## Prepared Generation
|
||||
|
||||
For [`RunPrepared`](../../engine.go), the runner supplies the model client with
|
||||
the target, rendered messages, and structured-output constraint retained by
|
||||
executable preparation. Execution does not reopen or rerender consumer
|
||||
sources.
|
||||
|
||||
Before backend admission, the runner rechecks that the frozen credential
|
||||
environment-variable name is available. The handle does not retain the
|
||||
environment value; the model client resolves the value visible when generation
|
||||
begins. A direct request key remains in private execution state only until the
|
||||
claimed execution finishes or an unclaimed handle is discarded. Exact public
|
||||
ownership and redaction semantics belong to the
|
||||
[`PreparedExecution` GoDoc](../../prepared_execution.go).
|
||||
|
||||
## Failure Categories
|
||||
|
||||
The package preserves distinct error identities for invalid client
|
||||
@@ -51,5 +75,7 @@ The
|
||||
[OpenAI-compatible client tests](../../internal/llm/openai_compatible_client_test.go)
|
||||
own configuration, client cloning, deterministic deadline precedence,
|
||||
authentication, request and response mapping, malformed data, error identity,
|
||||
cancellation, and response-body suppression. They use local test servers and
|
||||
test transports; the default suite makes no live or paid provider requests.
|
||||
cancellation, and response-body suppression. The root transport contract test
|
||||
also verifies that resolved backend settings reach this client without
|
||||
serializing backend identity. All use local test servers or test transports;
|
||||
the default suite makes no live or paid provider requests.
|
||||
|
||||
@@ -11,19 +11,23 @@ contributor workflow and validation.
|
||||
|
||||
| Component | Implemented responsibility | References |
|
||||
| --- | --- | --- |
|
||||
| Root `promptkit` package | Provides the supported engine facade, source and injection options, public request and result values, built-in profile construction, extension interfaces, value conversion, redacted formatting, and public error mapping. | [Package GoDoc](../../doc.go), [engine assembly](../../engine.go) |
|
||||
| Root `promptkit` package | Provides the supported engine facade, source, backend-registration, and injection options, public request, result, prompt-inspection, and profile-inspection values, opaque prepared-execution handles, profile construction, extension interfaces, value conversion, redacted formatting, typed capacity errors, public error mapping, and engine-local profile-source assembly including application fallbacks. | [Package GoDoc](../../doc.go), [prepared execution](../../prepared_execution.go), [backend API](../../backends.go), [engine assembly](../../engine.go) |
|
||||
| `examples/go-library/prepare` | Demonstrates an offline downstream consumer using a prompt file, in-memory profile, inline input, and `Prepare`. It is not a public library package. | [Example program](../../examples/go-library/prepare/main.go) |
|
||||
| `examples/go-library/run` | Demonstrates an offline downstream consumer using a prompt file, in-memory profile, inline input, an injected deterministic model client, and `Run`. It is not a public library package. | [Example program](../../examples/go-library/run/main.go) |
|
||||
| `internal/backend` | Constructs each engine's immutable registry from the built-in OpenRouter definition and consumer additions, validates and defensively copies definitions through the shared JSON-value package, and consumes the LLM-owned OpenAI-compatible reserved request-field rule. | [Backend registry](../../internal/backend/registry.go) |
|
||||
| `internal/capacity` | Owns engine-local bounded execution admission and FIFO model-generation permits for limited backend IDs, including cancellation-safe waiter removal and client wrapping. | [Internal capacity management](capacity.md) |
|
||||
| `internal/domain` | Defines internal framework values for requests, artifacts, prompt definitions, profiles, execution targets, rendering, generation, and validation. | [Domain declarations](../../internal/domain/domain.go) |
|
||||
| `internal/defaults` | Defines application-neutral framework constants and constructs the default execution target. It contains no CLI, server, or inbound HTTP limits. | [Framework defaults](../../internal/defaults/defaults.go) |
|
||||
| `internal/filecatalog` | Provides deterministic YAML discovery and path helpers for operating-system filesystems and `fs.FS` sources. | [File catalog](../../internal/filecatalog/catalog.go) |
|
||||
| `internal/jsonvalue` | Validates and deeply copies JSON-compatible extra-parameter and prepared-schema trees while preserving supported concrete value types. | [JSON values](../../internal/jsonvalue/jsonvalue.go) |
|
||||
| `internal/promptdef` | Loads strictly decoded, validated prompt definitions from filesystem and `fs.FS` sources, including version selection and contained file-backed message content. | [Framework formats](../formats.md), [prompt-definition repository](../../internal/promptdef/filesystem_repository.go) |
|
||||
| `internal/profile` | Loads strictly decoded, validated execution profiles from filesystem and `fs.FS` sources and composes repositories with error-preserving fallback. | [Framework formats](../formats.md), [profile repositories](../../internal/profile/filesystem_repository.go) |
|
||||
| `internal/profile/builtin` | Embeds the built-in execution profile catalog and combines it with an optional primary repository. | [Built-in catalog](../formats.md#built-in-profile-catalog), [repository](../../internal/profile/builtin/repository.go) |
|
||||
| `internal/profile` | Loads strictly decoded, validated execution profiles, including backend selection, from filesystem and `fs.FS` sources and composes repositories with error-preserving fallback. | [Framework formats](../formats.md), [profile repositories](../../internal/profile/filesystem_repository.go) |
|
||||
| `internal/profile/builtin` | Embeds the built-in profile catalog, whose entries select OpenRouter. | [Built-in catalog](../formats.md#built-in-profile-catalog), [repository](../../internal/profile/builtin/repository.go) |
|
||||
| `internal/prompt` | Renders prompt messages from Go templates with artifact, variable, session, and cache-control data. | [Go-template renderer](../../internal/prompt/go_renderer.go) |
|
||||
| `internal/artifact` | Resolves ordinary inline and unrestricted caller-selected file references into copied artifacts with metadata and hashes. | [Internal sources and validation](sources.md) |
|
||||
| `internal/validate` | Validates basic, JSON, and JSON Schema output using operating-system filesystem or `fs.FS` schema sources. | [Framework formats](../formats.md#schemas), [internal sources and validation](sources.md) |
|
||||
| `internal/llm` | Defines the internal generation boundary and implements outbound OpenAI-compatible chat requests, response decoding, authentication, and deadline handling. | [Internal model client](llm.md) |
|
||||
| `internal/usecase` | Coordinates preparation and execution across internal sources, rendering, artifact loading, generation, validation, and optional repair. | [Internal runner](runner.md) |
|
||||
| `internal/validate` | Validates basic, JSON, and JSON Schema output using operating-system filesystem or `fs.FS` schema sources and creates frozen validation plans for prepared execution. | [Framework formats](../formats.md#schemas), [internal sources and validation](sources.md) |
|
||||
| `internal/llm` | Defines the internal generation boundary and implements outbound OpenAI-compatible chat requests from resolved execution targets, including response decoding, authentication, deadline handling, and ownership of the OpenAI-compatible reserved request-field policy. | [Internal model client](llm.md) |
|
||||
| `internal/usecase` | Resolves prompt definitions and hashes, profiles, backends, and targets for exact inspection and request settings for preparation, and coordinates ordinary execution and one-attempt prepared execution across internal sources, rendering, artifact loading, generation, validation, capacity, and optional repair. | [Internal runner](runner.md), [prepared-execution implementation](../../internal/usecase/prepared_execution.go) |
|
||||
|
||||
The root package assembles these internal components without exposing their
|
||||
representations. Consumers depend only on the root facade.
|
||||
|
||||
@@ -17,7 +17,10 @@ and override semantics consumed by the runner.
|
||||
## Collaborators
|
||||
|
||||
`Runner` coordinates narrow internal interfaces for prompt definitions,
|
||||
profiles, artifacts, rendering, model generation, and validation. Schema
|
||||
profiles, backend resolution, artifacts, rendering, model generation, and
|
||||
validation. The root engine supplies one immutable registry containing the
|
||||
built-in backend and validated consumer additions, one engine-local run
|
||||
admitter, and a model client wrapped by the same capacity manager. Schema
|
||||
documents are loaded through the validator's optional schema-loader interface.
|
||||
An output repairer can be injected internally, but the ordinary runner
|
||||
constructor does not enable one.
|
||||
@@ -25,43 +28,121 @@ constructor does not enable one.
|
||||
Each invocation carries its state in request, prepared-run, and result values.
|
||||
The runner has no durable run or session store.
|
||||
|
||||
## Preparation Flow
|
||||
## Shared Prompt Selection
|
||||
|
||||
`Prepare` performs the reusable pre-generation workflow:
|
||||
The runner uses one prompt-selection and hashing boundary for ordinary
|
||||
preparation and exact prompt inspection. Preparation retains its early
|
||||
request-ID check before direct-session normalization; both operations then use
|
||||
the configured prompt repository to select one definition, load referenced
|
||||
message content, and calculate the same prompt hash.
|
||||
|
||||
1. validate the prompt selection and load the prompt definition;
|
||||
2. hash the loaded definition;
|
||||
Inspection stops after that structural lookup. It does not parse templates or
|
||||
touch profile, artifact, schema, renderer, validator, admission, or model
|
||||
collaborators. The root [`Engine.InspectPrompt`](../../engine.go) GoDoc owns
|
||||
the public operation's exact contract.
|
||||
|
||||
## Shared Profile Selection
|
||||
|
||||
The runner uses one profile-selection and target-resolution boundary for
|
||||
ordinary preparation and exact profile inspection. Preparation first selects a
|
||||
request profile or a prompt default; inspection begins with its required
|
||||
explicit profile ID. Both then apply the ordinary source precedence, resolve a
|
||||
named backend, and construct the effective target from framework, backend, and
|
||||
profile values.
|
||||
|
||||
Inspection stops after the resulting endpoint and model are structurally
|
||||
validated. It does not check credential availability or perform prompt,
|
||||
artifact, schema, rendering, admission, or model-client work. The root
|
||||
[`Engine.InspectProfile`](../../engine.go) GoDoc owns the public operation's
|
||||
exact contract.
|
||||
|
||||
## Shared Preparation Pipeline
|
||||
|
||||
`Prepare` and `Run` share one private preparation pipeline split at the point
|
||||
where a run can be assigned to its selected backend pool. The resolution phase
|
||||
performs only the work needed to validate routing and admission:
|
||||
|
||||
1. validate the required prompt selection and normalize any direct session ID;
|
||||
2. load the prompt definition and hash the original definition;
|
||||
3. select the request profile or the prompt's default profile;
|
||||
4. resolve application-neutral defaults, profile values, and explicit request
|
||||
overrides in that order;
|
||||
5. validate endpoint, model, numeric overrides, and credential requirements;
|
||||
6. resolve the output contract and load a structured-output schema when
|
||||
required;
|
||||
7. load and hash input artifacts;
|
||||
8. render and hash the prompt; and
|
||||
9. return the effective settings, source identities, messages, hashes, and
|
||||
preparation timing.
|
||||
4. resolve the profile's backend ID, when present;
|
||||
5. resolve application-neutral defaults, backend defaults, profile values,
|
||||
and explicit request overrides in that order;
|
||||
6. validate endpoint, model, numeric overrides, and credential requirements;
|
||||
7. resolve the effective output contract without loading its schema; and
|
||||
8. retain the definition, source identities, effective settings, output
|
||||
contract, and preparation start time in invocation-local state.
|
||||
|
||||
The completion phase consumes that state without reloading the prompt,
|
||||
profile, or backend:
|
||||
|
||||
1. load structured-output schema metadata when required;
|
||||
2. load and hash input artifacts;
|
||||
3. render messages and the prompt-defined session;
|
||||
4. apply any direct session ID;
|
||||
5. hash the effective rendered prompt; and
|
||||
6. construct the prepared value and preparation timing.
|
||||
|
||||
`Prepare` runs both phases consecutively and never performs capacity admission.
|
||||
`Run` performs backend admission between the phases. This structure preserves
|
||||
one execution-precedence and error-ordering implementation while allowing a
|
||||
full backend pool to reject work before expensive schema, artifact, and
|
||||
rendering operations.
|
||||
|
||||
Pointer-based numeric overrides preserve an explicit zero. Invalid negative or
|
||||
out-of-range values fail as invalid requests. A direct API key takes
|
||||
precedence over environment lookup for execution; secret values remain
|
||||
excluded from serialized metadata.
|
||||
out-of-range values fail as invalid requests. Endpoint overrides do not change
|
||||
the selected backend identity. Non-empty extra-parameter maps replace whole
|
||||
lower-precedence maps. A direct API key takes precedence over environment
|
||||
lookup; otherwise request, profile, and backend environment-variable names
|
||||
apply in that order. A profile requiring a direct key clears an inherited
|
||||
backend environment name unless the request supplies its own name. Secret
|
||||
values remain excluded from serialized metadata.
|
||||
|
||||
Reasoning overrides are tri-state: nil inherits the profile, a pointer to a
|
||||
nonblank string trims and replaces it, and a pointer to a blank string clears
|
||||
it. A nonblank direct session is normalized before source loading, bypasses
|
||||
the prompt session template, and is applied after ordinary message rendering.
|
||||
A blank direct value retains prompt-template behavior. The runner clears the
|
||||
template only on a value copy of the definition, so the definition hash always
|
||||
describes the original source while the rendered-prompt hash includes the
|
||||
effective direct or rendered session.
|
||||
|
||||
The registry is read-only after engine construction. Concurrent `Prepare` and
|
||||
`Run` calls resolve independent defensive backend values and keep all
|
||||
invocation state local.
|
||||
|
||||
## Run Flow
|
||||
|
||||
`Run` calls `Prepare` rather than maintaining a second preparation path. It
|
||||
performs one initial generation call, builds the named output artifact, and
|
||||
validates that artifact. Invalid generated content remains a validation result;
|
||||
an inability to perform validation is an operational error.
|
||||
`Run` records its start time, performs the shared resolution phase, and asks
|
||||
its `RunAdmitter` to reserve capacity for the effective backend ID. A nil
|
||||
admitter is an internal unlimited fallback. After successful admission, `Run`
|
||||
immediately defers the returned release function, performs the completion
|
||||
phase, makes one initial generation call, builds the named output artifact,
|
||||
and validates that artifact. Invalid generated content remains a validation
|
||||
result; an inability to perform validation is an operational error.
|
||||
|
||||
The admission lease covers completion-phase preparation, initial generation,
|
||||
validation, every repair, and every exit. It bounds accepted work without
|
||||
serializing preparation or validation behind the active-generation limit.
|
||||
The wrapped model client separately acquires a FIFO active permit only around
|
||||
each actual generation call.
|
||||
|
||||
When an internal repairer is present, a JSON or JSON Schema content failure can
|
||||
trigger bounded repair attempts. Repair receives the effective execution
|
||||
target, validation errors, prior output, and structured-output specification.
|
||||
This capability remains internal and is not a public option.
|
||||
target and session ID, validation errors, prior output, and structured-output
|
||||
specification. The default repairer uses the same wrapped client as initial
|
||||
generation, so each repair reacquires the selected backend's active permit
|
||||
while remaining inside its original admission lease. Repair never performs a
|
||||
second bounded admission. This capability remains internal and is not a public
|
||||
option.
|
||||
|
||||
A successful result includes the output artifact and raw output, validation
|
||||
state, prompt and rendered-prompt hashes, selected profile, effective settings,
|
||||
input hashes, token usage, a generated run identifier, and UTC timing.
|
||||
state, effective session ID, prompt and rendered-prompt hashes, selected
|
||||
profile and backend, effective settings, input hashes, token usage, a generated
|
||||
run identifier, and UTC timing. The same effective session reaches initial
|
||||
generation and any repair attempt through the rendered prompt. The same
|
||||
effective target, including backend identity, reaches generation and any
|
||||
repair attempt.
|
||||
|
||||
## Failure Categories
|
||||
|
||||
@@ -69,17 +150,40 @@ Package errors distinguish invalid requests, required profile selection,
|
||||
credential failures, and prompt, profile, artifact, rendering, generation, and
|
||||
validation failures. Wrapping preserves the package identities mapped by the
|
||||
public facade and retains collaborator identities where they are part of the
|
||||
internal contract. Context cancellation propagates through the invoked
|
||||
collaborator and is classified by the owning operation.
|
||||
internal contract.
|
||||
|
||||
Admission capacity exhaustion retains the internal capacity identity. At the
|
||||
use-case boundary, the runner attaches the selected backend ID in an internal
|
||||
typed error, and the root facade copies that value into the public
|
||||
[`CapacityError`](../../capacity_error.go) without parsing diagnostic text. It
|
||||
is not recategorized as an invalid request or generation failure, and no
|
||||
partial result is returned. A context already done at admission retains its
|
||||
context identity directly. Cancellation while waiting for an active generation
|
||||
permit prevents client invocation when it wins the grant race; the model-client
|
||||
boundary then preserves the context error through the generation-failure
|
||||
category. Deferred release restores the admission lease on preparation,
|
||||
generation, validation, repair, and cancellation failures.
|
||||
|
||||
Other context cancellation propagates through the invoked collaborator and is
|
||||
classified by the owning operation.
|
||||
An overlong direct session is an invalid request before source loading, while
|
||||
an invalid or overlong prompt session template remains a prompt-render failure.
|
||||
An unknown selected backend, or a selected backend with no configured resolver,
|
||||
is classified as a profile-load failure.
|
||||
|
||||
## Test Ownership And Changes
|
||||
|
||||
The [runner tests](../../internal/usecase/runner_test.go) own preparation order,
|
||||
selection and override precedence, schema-before-generation behavior, hashing,
|
||||
generation and validation outcomes, bounded repair, credentials and redaction,
|
||||
error categories, artifact metadata, usage, and timing.
|
||||
selection and override precedence, the two-phase boundary, early admission,
|
||||
lease lifetime and release, direct-session resolution, schema-before-generation
|
||||
behavior, hashing, generation and validation outcomes, backend propagation,
|
||||
bounded repair, shared initial/repair capacity, credentials and redaction,
|
||||
error categories, artifact metadata, usage, and timing. The
|
||||
[capacity subsystem document](capacity.md) identifies the focused pool,
|
||||
waiter, and wrapped-client tests.
|
||||
|
||||
Changes to orchestration should continue to use the existing package
|
||||
interfaces, keep request state local to an invocation, and preserve `Run`'s use
|
||||
of `Prepare`. Source, renderer, validator, or model-client contract changes
|
||||
belong first in their owning package and document.
|
||||
interfaces, keep request state local to an invocation, and preserve the shared
|
||||
resolution and completion pipeline. Source, renderer, validator, or
|
||||
model-client contract changes belong first in their owning package and
|
||||
document.
|
||||
|
||||
@@ -16,6 +16,11 @@ validation modes, built-in catalog, and source precedence.
|
||||
definitions, selects an ID and optional version, and resolves file-backed
|
||||
message content within the selected operating-system or `fs.FS` source.
|
||||
|
||||
Exact prompt inspection performs one point-in-time lookup through that same
|
||||
repository and validates referenced message content before returning declared
|
||||
metadata. It does not parse templates or read profile, input, or schema
|
||||
sources, and it does not retain the definition for a later execution.
|
||||
|
||||
Its package tests own prompt selection, strict decoding, definition validation,
|
||||
duplicate detection, and source containment:
|
||||
[prompt-definition repository tests](../../internal/promptdef/repository_test.go).
|
||||
@@ -23,14 +28,30 @@ duplicate detection, and source containment:
|
||||
## Profiles And Built-Ins
|
||||
|
||||
`internal/profile` loads and validates execution profiles from an
|
||||
operating-system filesystem or an `fs.FS`. It supports a primary repository
|
||||
with fallback only when the primary reports that a profile is absent.
|
||||
operating-system filesystem or an `fs.FS`. Its overlay repository consults the
|
||||
next repository only when the higher-precedence repository reports that a
|
||||
profile is absent. Strict YAML decoding recognizes the optional `backend`
|
||||
field, trims its value, and requires a model plus at least one non-blank
|
||||
backend or endpoint. Loading does not check registry membership because the
|
||||
available registry belongs to the assembled engine; the runner checks
|
||||
membership during preparation and exact profile inspection.
|
||||
|
||||
`internal/profile/builtin` embeds the maintained built-in profile catalog and
|
||||
can place a caller-selected repository ahead of that catalog. Profile behavior
|
||||
is owned by the
|
||||
[profile repository tests](../../internal/profile/repository_test.go), while
|
||||
catalog completeness, duplicate IDs, and overlay behavior are owned by the
|
||||
The root engine assembles profile repositories in precedence order: in-memory
|
||||
profiles, one ordinary configured source, an application fallback source, then
|
||||
the embedded built-in catalog. An explicit file or `fs.FS` profile source
|
||||
replaces `Config.ProfileDir` within the ordinary configured-source category.
|
||||
|
||||
Exact profile inspection performs one point-in-time lookup through those
|
||||
profile sources and checks the resolved target without reading prompt, input,
|
||||
or schema sources. It does not retain that lookup for a later execution.
|
||||
|
||||
`internal/profile/builtin` embeds the maintained built-in profile catalog.
|
||||
Every embedded profile selects `openrouter` and inherits its endpoint and
|
||||
credential environment-variable name from the built-in backend registry rather
|
||||
than repeating those values. Profile loading and overlay behavior are owned by
|
||||
the [profile repository tests](../../internal/profile/repository_test.go),
|
||||
while catalog completeness, the backend-selection invariant, and duplicate IDs
|
||||
are owned by the
|
||||
[built-in repository tests](../../internal/profile/builtin/repository_test.go).
|
||||
|
||||
## Ordinary Artifacts
|
||||
@@ -61,6 +82,22 @@ filesystem or an `fs.FS`. Invalid generated content is returned as a validation
|
||||
result; inability to load, register, or compile a schema is an operational
|
||||
error.
|
||||
|
||||
For executable preparation, the built-in validators create a frozen validation
|
||||
plan. None, basic, and JSON modes retain the effective output contract without
|
||||
source access. JSON Schema mode loads the root document, resolves and compiles
|
||||
every transitive reference during preparation, and retains the compiled
|
||||
validator. The provider-facing structured-output metadata uses that same
|
||||
captured root document.
|
||||
|
||||
`PrepareExecution` also completes prompt and profile selection, artifact
|
||||
loading and hashing, session and message rendering, and target resolution.
|
||||
`RunPrepared` uses the retained source-derived state and validation plan; it
|
||||
does not reopen prompt, profile, input, or schema sources and does not rerender
|
||||
the request. By contrast, ordinary `Prepare` produces a preparation value only:
|
||||
a later `Run` performs its own source resolution and preparation.
|
||||
|
||||
The [validator tests](../../internal/validate/standard_validator_test.go) own
|
||||
basic, JSON, JSON Schema, source resolution, schema loading, compilation, and
|
||||
content-failure behavior.
|
||||
basic, JSON, JSON Schema, source resolution, schema loading, compilation,
|
||||
frozen-reference behavior, and content-failure behavior. Prepared execution
|
||||
orchestration is owned by the
|
||||
[use-case tests](../../internal/usecase/prepared_execution_test.go).
|
||||
|
||||
@@ -21,10 +21,16 @@ The implemented internal components consist of:
|
||||
|
||||
- `internal/domain`, which owns framework data values shared by later internal
|
||||
components;
|
||||
- `internal/backend`, which owns validated immutable OpenAI-compatible backend
|
||||
definitions and the built-in OpenRouter definition;
|
||||
- `internal/capacity`, which owns engine-local bounded run admission and
|
||||
model-generation scheduling for limited backends;
|
||||
- `internal/defaults`, which owns application-neutral framework defaults and
|
||||
constructs the default execution target;
|
||||
- `internal/filecatalog`, which discovers YAML files and provides source-path
|
||||
helpers for filesystem and `fs.FS` consumers;
|
||||
- `internal/jsonvalue`, which validates and defensively copies JSON-compatible
|
||||
extra-parameter trees;
|
||||
- `internal/promptdef`, which loads and validates prompt definitions from
|
||||
filesystem and `fs.FS` sources;
|
||||
- `internal/profile`, which loads, validates, and overlays execution profiles
|
||||
@@ -41,21 +47,25 @@ The implemented internal components consist of:
|
||||
- `internal/usecase`, which coordinates preparation and execution across the
|
||||
internal framework components.
|
||||
|
||||
The `examples/go-library/prepare` package is a maintained downstream consumer
|
||||
of the root facade. It does not expose a library package or participate in
|
||||
internal assembly.
|
||||
The `examples/go-library/prepare` and `examples/go-library/run` packages are
|
||||
maintained downstream consumers of the root facade. They do not expose library
|
||||
packages or participate in internal assembly.
|
||||
|
||||
The root facade assembles the internal repositories, renderer, validator,
|
||||
outbound client, and use-case runner while translating public values and
|
||||
errors at the library boundary. The defaults and renderer depend on the domain
|
||||
model. Prompt-definition and profile repositories use the domain model, file
|
||||
catalog, and YAML decoder. The built-in profile repository supplies an
|
||||
embedded `fs.FS` to the profile package. Artifact reading uses the domain model
|
||||
and application-neutral defaults. Validation uses the domain model, file
|
||||
The root facade assembles one immutable backend registry, one capacity manager,
|
||||
the internal repositories, renderer, validator, outbound client, and use-case
|
||||
runner while translating public values and errors at the library boundary. The
|
||||
registry contains built-ins plus validated engine-scoped consumer additions.
|
||||
The facade constructs the capacity manager from the registry's immutable
|
||||
policy snapshot, wraps the selected built-in or injected model client, and
|
||||
supplies bounded admission to the runner. The defaults and renderer depend on
|
||||
the domain model. Prompt-definition and profile repositories use the domain
|
||||
model, file catalog, and YAML decoder. The built-in profile repository supplies
|
||||
an embedded `fs.FS` to the profile package. Artifact reading uses the domain
|
||||
model and application-neutral defaults. Validation uses the domain model, file
|
||||
catalog, and JSON Schema implementation. The model client uses the domain
|
||||
model, application-neutral defaults, and an injected or standard-library HTTP
|
||||
client. The use-case runner depends on the narrow interfaces owned by each
|
||||
internal component.
|
||||
internal component, including backend lookup and run admission.
|
||||
|
||||
The current implementation follows this dependency direction:
|
||||
|
||||
@@ -72,9 +82,15 @@ downstream consumers, including Scriptorium
|
||||
narrow injected abstractions
|
||||
```
|
||||
|
||||
The facade coordinates internal components and adapts the supported public
|
||||
extension interfaces to narrow internal abstractions. Internal components must
|
||||
not depend on consumers or on Scriptorium.
|
||||
The backend registry depends on the domain model and shared JSON-value
|
||||
validation, has no mutation API after construction, and consumes the
|
||||
OpenAI-compatible reserved request-field rule owned by the model client. The
|
||||
capacity component depends on the domain model and the narrow internal
|
||||
model-client boundary, not on provider transport implementation. The model
|
||||
client does not depend on registry or capacity configuration. The facade
|
||||
coordinates internal components and adapts the supported public extension
|
||||
interfaces to narrow internal abstractions. Internal components must not depend
|
||||
on consumers or on Scriptorium.
|
||||
|
||||
## Repository And Consumer Boundary
|
||||
|
||||
|
||||
@@ -78,6 +78,7 @@ mechanisms, not secret values.
|
||||
| Framework file formats | `docs/formats.md` | Prompt-definition and profile YAML fields, schema references, defaults, validation modes, built-in profiles, credentials, and file-to-request precedence. | Exported Go declarations, outbound wire behavior, internal parsing mechanics, and application configuration. |
|
||||
| Consumer guidance | `docs/consumers/`, when consumer workflows require dedicated guidance | Task-oriented use of implemented public APIs, minimal examples, and consumer responsibilities. | Exact exported declarations and internal mechanics. |
|
||||
| Durable integration contracts | `docs/integrations/`, when integrations exist | External formats and protocols, compatibility behavior, and upstream or downstream responsibilities. | Internal transformations and public Go declarations. |
|
||||
| Supplemental release guidance | None. `docs/releases/` may be used when a release benefits from a changelog or migration guide. | No canonical content. These files may briefly summarize release-specific changes, compatibility, and consumer migration paths, and may be corrected, consolidated, archived, or removed when no longer useful. | Public API and behavior contracts, formats, integrations, architecture, release procedure, and the authoritative annotated-tag release record. |
|
||||
| Implemented component inventory | `docs/internal/overview.md` | Current packages and components, their implemented responsibilities, and links to focused internal documents. | Normative architecture, contributor workflow, external contracts, and proposed components. |
|
||||
| Internal subsystem behavior | Other files under `docs/internal/`, when a subsystem needs durable detail | Implementation flow, internal collaborators and state transitions, package-local guarantees and failures, and relevant tests. | Global architecture invariants, public API definitions, and future package plans. |
|
||||
| Architectural decision history | `docs/adr/`, when repository-local decisions require records | Significant decisions, context, alternatives, rationale, consequences, and supersession history. | Current behavior reference, implementation status, and task sequencing. |
|
||||
@@ -85,9 +86,9 @@ mechanisms, not secret values.
|
||||
| Complete copyable artifacts | `examples/` | Valid inputs, Go programs, and other files intended to be copied or run. | Field-by-field reference, exact API declarations, and prose explanation. |
|
||||
|
||||
Conditional owners do not require placeholder files or directories. Create a
|
||||
consumer, integration, subsystem, ADR, roadmap, or example document only when
|
||||
the corresponding implemented interface, decision, planned effort, or
|
||||
maintained artifact exists.
|
||||
consumer, integration, release, subsystem, ADR, roadmap, or example document
|
||||
only when the corresponding implemented interface, release, decision, planned
|
||||
effort, or maintained artifact exists.
|
||||
|
||||
## Boundary Rules
|
||||
|
||||
@@ -109,6 +110,23 @@ but must link to its canonical definition rather than restate it.
|
||||
The [framework format reference](../formats.md) owns exact prompt, profile, and
|
||||
schema-file contracts. Integration documents own external wire formats.
|
||||
|
||||
### Supplemental Release Guidance
|
||||
|
||||
Files under `docs/releases/` may provide changelog-style summaries and
|
||||
migration guidance for a particular release. They are navigation and
|
||||
orientation aids, not canonical owners of public APIs, behavior, formats,
|
||||
integrations, architecture, release procedure, or other durable facts. When a
|
||||
reader needs detail beyond a short release-specific note, the release document
|
||||
must link to the applicable canonical documentation rather than reproduce its
|
||||
contract.
|
||||
|
||||
The annotated tag message required by the
|
||||
[release procedure](../release.md#write-the-release-note) remains the
|
||||
authoritative release record. Supplemental release documents may be corrected,
|
||||
consolidated, archived, or removed at any time when they are no longer useful,
|
||||
provided maintained documentation does not depend on them and the annotated
|
||||
tag record remains intact.
|
||||
|
||||
### Security Topics
|
||||
|
||||
This policy owns what documentation and examples may contain. Architecture owns
|
||||
@@ -160,6 +178,10 @@ durable owners, update incoming links, and archive or remove the roadmap
|
||||
according to repository practice. Do not preserve completed roadmaps as a
|
||||
second current-state reference.
|
||||
|
||||
Supplemental release documents may likewise be removed without preserving a
|
||||
replacement. Before removal, update maintained incoming links so current
|
||||
documentation does not depend on an optional historical guide.
|
||||
|
||||
Before completing documentation work:
|
||||
|
||||
- verify affected behavior and examples;
|
||||
|
||||
254
docs/release.md
254
docs/release.md
@@ -7,27 +7,91 @@ tags. It does not publish runnable binaries or binary packages and does not
|
||||
currently use hosted CI. The release maintainer performs and records the
|
||||
required validation.
|
||||
|
||||
The first planned release is `v0.1.0`. Do not create that tag until the
|
||||
framework has been extracted and the resulting public library has passed this
|
||||
procedure. Later tags use the `vMAJOR.MINOR.PATCH` form. While Promptkit remains
|
||||
pre-`v1`, release notes must identify intentional public API changes and any
|
||||
consumer migration required by them.
|
||||
`v0.1.0` is the initial published release. Later releases use semantic
|
||||
`vMAJOR.MINOR.PATCH` tags. Before `v1`, minor releases may change the public
|
||||
API and patch releases preserve compatibility within their minor line. Every
|
||||
pre-`v1` release note must summarize compatibility, identify public API
|
||||
changes, and state any action required of consumers.
|
||||
|
||||
## Prepare The Release
|
||||
Promptkit releases are source-only. The annotated tag message is the release
|
||||
note; there is no separate hosted release or binary packaging step.
|
||||
|
||||
Work from a clean checkout of the intended release commit, outside any Go
|
||||
workspace and without a local module replacement. Confirm the source commit is
|
||||
already published through the normal branch workflow.
|
||||
## Establish The Candidate
|
||||
|
||||
From the Promptkit repository root, verify the checkout:
|
||||
Choose a version that has not been published and export it as
|
||||
`RELEASE_VERSION`. Run every command in this procedure from the Promptkit
|
||||
repository root in the same POSIX shell. Do not reuse `v0.1.0` or another
|
||||
existing version.
|
||||
|
||||
The following guard derives the release commit from `HEAD` and stops on a
|
||||
missing or malformed version, a checkout other than synchronized `main`,
|
||||
uncommitted changes, an active Go workspace, a module replacement, a vendor
|
||||
tree, or an existing local or remote tag:
|
||||
|
||||
```sh
|
||||
gowork=$(go env GOWORK)
|
||||
test -z "$gowork" || test "$gowork" = off
|
||||
test -z "$(git status --short)"
|
||||
git fetch --tags origin
|
||||
set -eu
|
||||
|
||||
: "${RELEASE_VERSION:?export an unpublished vMAJOR.MINOR.PATCH version}"
|
||||
if ! printf '%s\n' "$RELEASE_VERSION" |
|
||||
grep -Eq '^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$'
|
||||
then
|
||||
printf '%s\n' "invalid release version: $RELEASE_VERSION" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
RELEASE_COMMIT=$(git rev-parse --verify 'HEAD^{commit}')
|
||||
export RELEASE_COMMIT
|
||||
|
||||
check_release_candidate() {
|
||||
test "$(git branch --show-current)" = main
|
||||
test -z "$(git status --porcelain)"
|
||||
|
||||
gowork_value=$(go env GOWORK)
|
||||
case "$gowork_value" in
|
||||
''|off) ;;
|
||||
*)
|
||||
printf '%s\n' "active Go workspace: $gowork_value" >&2
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
|
||||
test -z "$(git ls-files go.work go.work.sum)"
|
||||
test ! -e vendor
|
||||
if grep -Eq '^[[:space:]]*replace([[:space:]]|\()' go.mod
|
||||
then
|
||||
printf '%s\n' 'go.mod contains a replacement' >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
git fetch origin main --tags
|
||||
test "$RELEASE_COMMIT" = \
|
||||
"$(git rev-parse --verify 'refs/remotes/origin/main^{commit}')"
|
||||
|
||||
if git show-ref --verify --quiet "refs/tags/$RELEASE_VERSION"
|
||||
then
|
||||
printf '%s\n' "local tag already exists: $RELEASE_VERSION" >&2
|
||||
return 1
|
||||
fi
|
||||
if test -n "$(
|
||||
git ls-remote --tags origin \
|
||||
"refs/tags/$RELEASE_VERSION" \
|
||||
"refs/tags/$RELEASE_VERSION^{}"
|
||||
)"
|
||||
then
|
||||
printf '%s\n' "remote tag already exists: $RELEASE_VERSION" >&2
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
check_release_candidate
|
||||
```
|
||||
|
||||
Do not continue unless the guard completes successfully. In particular, push
|
||||
the intended commit through the normal `main` branch workflow before release;
|
||||
the tag procedure is not a substitute for publishing the source commit.
|
||||
|
||||
## Validate The Candidate
|
||||
|
||||
Confirm the module and root package metadata:
|
||||
|
||||
```sh
|
||||
@@ -42,7 +106,7 @@ gitea.maximumdirect.net/eric/promptkit 1.25.5
|
||||
promptkit gitea.maximumdirect.net/eric/promptkit
|
||||
```
|
||||
|
||||
Run the same default Go validation required by the
|
||||
Run the complete maintainer validation required by the
|
||||
[development guide](development.md):
|
||||
|
||||
```sh
|
||||
@@ -53,84 +117,158 @@ go build ./...
|
||||
go run ./examples/go-library/prepare
|
||||
```
|
||||
|
||||
Check every tracked Go file and repository whitespace:
|
||||
Check every tracked Go file. This command must produce no output:
|
||||
|
||||
```sh
|
||||
gofmt -l $(git ls-files '*.go')
|
||||
unformatted=$(
|
||||
git ls-files '*.go' |
|
||||
while IFS= read -r go_file
|
||||
do
|
||||
gofmt -l "$go_file"
|
||||
done
|
||||
)
|
||||
test -z "$unformatted"
|
||||
```
|
||||
|
||||
Follow every maintained Markdown link and confirm that its local or published
|
||||
target exists. Review the repository for generated binaries, test or coverage
|
||||
output, credentials, template residue, downloaded assets, and other files that
|
||||
do not belong in source control.
|
||||
|
||||
Recheck module and repository hygiene, whitespace, and the clean checkout:
|
||||
|
||||
```sh
|
||||
test -z "$(git ls-files go.work go.work.sum)"
|
||||
test ! -e vendor
|
||||
if grep -Eq '^[[:space:]]*replace([[:space:]]|\()' go.mod
|
||||
then
|
||||
printf '%s\n' 'go.mod contains a replacement' >&2
|
||||
exit 1
|
||||
fi
|
||||
git diff --check
|
||||
test -z "$(git status --porcelain)"
|
||||
```
|
||||
|
||||
The formatting command must produce no paths. Follow every maintained Markdown
|
||||
link and confirm its target exists. Review the repository for generated
|
||||
binaries, test or coverage output, credentials, template residue, and other
|
||||
files that do not belong in source control.
|
||||
## Write The Release Note
|
||||
|
||||
Confirm that no workspace override is tracked and that `go.mod` contains no
|
||||
`replace` directive:
|
||||
Prepare a plain-text annotated-tag message outside the repository and export
|
||||
its path as `RELEASE_NOTES_FILE`. Use this form, replacing each summary with
|
||||
release-specific text; write `None.` when there are no public API changes or
|
||||
consumer actions:
|
||||
|
||||
```sh
|
||||
git ls-files go.work go.work.sum
|
||||
rg -n '^replace\b' go.mod
|
||||
```text
|
||||
Promptkit vMAJOR.MINOR.PATCH
|
||||
|
||||
Validated commit: full commit ID
|
||||
Compatibility: compatibility summary
|
||||
Public API changes: changes or None.
|
||||
Consumer action: required action or None.
|
||||
```
|
||||
|
||||
Both commands must produce no output. Re-run `git status --short` and require a
|
||||
clean result after every validation and review check.
|
||||
|
||||
## Create And Publish The Tag
|
||||
|
||||
Choose the semantic version from the intended compatibility change. Record the
|
||||
release commit before tagging:
|
||||
After writing it, require all release-note fields, the selected version, and
|
||||
the validated commit to be present:
|
||||
|
||||
```sh
|
||||
release_version=v0.1.0
|
||||
release_commit=$(git rev-parse HEAD)
|
||||
: "${RELEASE_NOTES_FILE:?export the path to the release-note file}"
|
||||
test -f "$RELEASE_NOTES_FILE"
|
||||
test -s "$RELEASE_NOTES_FILE"
|
||||
grep -F "Promptkit $RELEASE_VERSION" "$RELEASE_NOTES_FILE"
|
||||
grep -F "Validated commit: $RELEASE_COMMIT" "$RELEASE_NOTES_FILE"
|
||||
grep -F 'Compatibility:' "$RELEASE_NOTES_FILE"
|
||||
grep -F 'Public API changes:' "$RELEASE_NOTES_FILE"
|
||||
grep -F 'Consumer action:' "$RELEASE_NOTES_FILE"
|
||||
```
|
||||
|
||||
Replace the example version for later releases and keep both values in the same
|
||||
shell for the remaining commands. Confirm the tag does not already exist
|
||||
locally or remotely:
|
||||
Inspect the complete message and confirm that it accurately records the
|
||||
compatibility impact, public API changes, and required consumer action.
|
||||
|
||||
## Create And Inspect The Tag
|
||||
|
||||
Run the candidate guard again immediately before tag creation. This ensures
|
||||
that validation or release-note preparation did not change the checkout and
|
||||
that the commit is still published and untagged:
|
||||
|
||||
```sh
|
||||
test -z "$(git tag --list "$release_version")"
|
||||
test -z "$(git ls-remote --tags origin "refs/tags/$release_version")"
|
||||
check_release_candidate
|
||||
```
|
||||
|
||||
Create an annotated tag whose message identifies the release and records that
|
||||
the documented validation passed for the tagged commit:
|
||||
Create the annotated tag from the prepared release note and bind it explicitly
|
||||
to the validated commit:
|
||||
|
||||
```sh
|
||||
git tag --annotate "$release_version" \
|
||||
--message "Promptkit $release_version; documented validation passed for $release_commit"
|
||||
git tag --annotate "$RELEASE_VERSION" \
|
||||
--file "$RELEASE_NOTES_FILE" \
|
||||
"$RELEASE_COMMIT"
|
||||
```
|
||||
|
||||
Inspect the tag before publication:
|
||||
Inspect both the tag message and its source commit before publication:
|
||||
|
||||
```sh
|
||||
git show --no-patch --decorate "$release_version"
|
||||
test "$(git rev-list -n 1 "$release_version")" = "$release_commit"
|
||||
test "$(git cat-file -t "refs/tags/$RELEASE_VERSION")" = tag
|
||||
git show --no-patch --decorate "refs/tags/$RELEASE_VERSION"
|
||||
test "$(
|
||||
git rev-parse --verify "refs/tags/$RELEASE_VERSION^{commit}"
|
||||
)" = "$RELEASE_COMMIT"
|
||||
```
|
||||
|
||||
Publish the tag without relying on a hosting-provider-specific release
|
||||
interface:
|
||||
If inspection finds an error, delete the unpublished local tag, correct the
|
||||
release note or candidate, and repeat the guards. Never move or recreate a tag
|
||||
that has been published.
|
||||
|
||||
## Publish The Selected Tag
|
||||
|
||||
Push only the selected tag ref. Do not use `git push --tags`:
|
||||
|
||||
```sh
|
||||
git push origin "refs/tags/$release_version"
|
||||
git push origin \
|
||||
"refs/tags/$RELEASE_VERSION:refs/tags/$RELEASE_VERSION"
|
||||
```
|
||||
|
||||
## Verify Publication
|
||||
|
||||
Confirm that the remote tag object matches the local annotated tag and still
|
||||
resolves to the intended source commit:
|
||||
Compare the remote annotated-tag object with the local object, then compare the
|
||||
remote peeled source commit with the validated commit:
|
||||
|
||||
```sh
|
||||
remote_tag=$(git ls-remote --tags origin "refs/tags/$release_version" | awk '{print $1}')
|
||||
test "$remote_tag" = "$(git rev-parse "refs/tags/$release_version")"
|
||||
test "$(git rev-list -n 1 "refs/tags/$release_version")" = "$release_commit"
|
||||
remote_tag=$(
|
||||
git ls-remote --tags origin "refs/tags/$RELEASE_VERSION" |
|
||||
awk 'NR == 1 { print $1 }'
|
||||
)
|
||||
remote_commit=$(
|
||||
git ls-remote --tags origin "refs/tags/$RELEASE_VERSION^{}" |
|
||||
awk 'NR == 1 { print $1 }'
|
||||
)
|
||||
test -n "$remote_tag"
|
||||
test "$remote_tag" = \
|
||||
"$(git rev-parse --verify "refs/tags/$RELEASE_VERSION")"
|
||||
test "$remote_commit" = "$RELEASE_COMMIT"
|
||||
```
|
||||
|
||||
Promptkit must publish the required tag before Scriptorium or another consumer
|
||||
publishes a release that depends on that version. Released consumer modules
|
||||
must not use a local replacement or unpublished Promptkit revision.
|
||||
Finally, resolve the version as an ordinary Go module in a temporary module
|
||||
outside this repository and without a workspace or replacement:
|
||||
|
||||
```sh
|
||||
resolution_dir=$(mktemp -d)
|
||||
(
|
||||
trap 'rm -rf "$resolution_dir"' 0 1 2 15
|
||||
cd "$resolution_dir"
|
||||
GOWORK=off go mod init example.com/promptkit-release-check
|
||||
GOWORK=off go mod download \
|
||||
"gitea.maximumdirect.net/eric/promptkit@$RELEASE_VERSION"
|
||||
resolved_version=$(
|
||||
GOWORK=off go list -m -f '{{.Version}}' \
|
||||
"gitea.maximumdirect.net/eric/promptkit@$RELEASE_VERSION"
|
||||
)
|
||||
test "$resolved_version" = "$RELEASE_VERSION"
|
||||
)
|
||||
```
|
||||
|
||||
Promptkit must publish and verify the required version before Scriptorium or
|
||||
another consumer publishes a release that depends on it. This ordering does
|
||||
not replace the consumer project's own release procedure. Released consumers
|
||||
must select the published Promptkit tag through ordinary module resolution,
|
||||
without a workspace, replacement, vendored Promptkit source, or unpublished
|
||||
revision.
|
||||
|
||||
## Policy Changes
|
||||
|
||||
|
||||
243
docs/releases/v0.2.0.md
Normal file
243
docs/releases/v0.2.0.md
Normal file
@@ -0,0 +1,243 @@
|
||||
# Promptkit v0.2.0
|
||||
|
||||
This supplemental changelog and migration guide summarizes the consumer-facing
|
||||
changes from `v0.1.0` to `v0.2.0`. The annotated `v0.2.0` tag is the
|
||||
authoritative release record. Exact current contracts belong to the linked
|
||||
GoDoc and durable documentation.
|
||||
|
||||
## Summary
|
||||
|
||||
`v0.2.0` adds three major capabilities:
|
||||
|
||||
- an engine-scoped registry for reusable OpenAI-compatible backend
|
||||
definitions;
|
||||
- bounded, backend-specific run admission and model-generation concurrency;
|
||||
and
|
||||
- direct per-run session IDs and tri-state reasoning-effort overrides.
|
||||
|
||||
Existing endpoint-only profiles remain supported. Consumers can adopt backend
|
||||
registration and runtime overrides incrementally rather than rewriting all
|
||||
profiles during the upgrade.
|
||||
|
||||
## Compatibility At A Glance
|
||||
|
||||
Promptkit remains pre-`v1`, and this minor release includes source-level and
|
||||
behavioral changes that deserve review.
|
||||
|
||||
| Area | `v0.1.0` consumer impact |
|
||||
| --- | --- |
|
||||
| Endpoint-only profiles | Continue to work without migration. |
|
||||
| Built-in profiles | Continue to use OpenRouter and `OPENROUTER_API_KEY`; they now select the built-in `openrouter` backend. |
|
||||
| Custom backends | Registration is optional. Existing profiles may keep their endpoint and credential configuration. |
|
||||
| Reasoning overrides | String assignments must migrate to the new pointer field. |
|
||||
| `RunRequest.Metadata` | Removed; delete assignments to this field. |
|
||||
| OpenRouter concurrency | Now limited to 16 active generations with waiting capacity of 1024 per engine. |
|
||||
| Public JSON | `v0.2.0` formalizes supported JSON representations; consumers relying on `v0.1.0` encodings should review the notes below. |
|
||||
| Unkeyed public struct literals | May require updates because fields were added. Keyed literals are recommended. |
|
||||
|
||||
## Upgrade
|
||||
|
||||
After the `v0.2.0` tag is published, update the module dependency with:
|
||||
|
||||
```sh
|
||||
go get gitea.maximumdirect.net/eric/promptkit@v0.2.0
|
||||
go mod tidy
|
||||
```
|
||||
|
||||
Run the consuming project's ordinary tests and race-enabled tests after the
|
||||
upgrade, especially if it calls one engine concurrently or persists Promptkit
|
||||
JSON values.
|
||||
|
||||
## Backend Registry
|
||||
|
||||
Consumers may now register reusable OpenAI-compatible backend definitions with
|
||||
`WithBackend`, then select them by ID from file-backed or in-memory profiles.
|
||||
A backend can supply its endpoint, API-key environment-variable name,
|
||||
request-wide extra parameters, and optional capacity policy.
|
||||
|
||||
Registrations are immutable and belong to one engine. Consumer registrations
|
||||
can add new IDs but cannot replace Promptkit's reserved `openrouter` backend.
|
||||
Profiles that select a backend may still override its endpoint without losing
|
||||
the backend's routing or capacity identity.
|
||||
|
||||
An existing endpoint-only in-memory profile remains valid:
|
||||
|
||||
```go
|
||||
promptkit.Profile{
|
||||
ID: "local",
|
||||
Endpoint: "http://localhost:8000/v1",
|
||||
Model: "example-model",
|
||||
}
|
||||
```
|
||||
|
||||
Adopting the registry is optional and can be done when several profiles should
|
||||
share connection or capacity settings:
|
||||
|
||||
```go
|
||||
engine, err := promptkit.NewEngine(
|
||||
promptkit.Config{PromptDir: "prompts"},
|
||||
promptkit.WithBackend(promptkit.Backend{
|
||||
ID: "local",
|
||||
Endpoint: "http://localhost:8000/v1",
|
||||
APIKeyEnv: "LOCAL_LLM_API_KEY",
|
||||
}),
|
||||
promptkit.WithProfiles(promptkit.Profile{
|
||||
ID: "local-summary",
|
||||
BackendID: "local",
|
||||
Model: "example-model",
|
||||
}),
|
||||
)
|
||||
```
|
||||
|
||||
See the
|
||||
[local-endpoint consumer guide](../consumers/pkg-promptkit.md#configure-a-local-openai-compatible-endpoint)
|
||||
for task-oriented usage. The
|
||||
[`Backend` and `WithBackend` GoDoc](../../backends.go) owns exact registration,
|
||||
validation, copying, defaulting, and uniqueness semantics. The
|
||||
[framework format reference](../formats.md) owns the profile `backend` field
|
||||
and execution precedence.
|
||||
|
||||
## Backend-Specific Concurrency
|
||||
|
||||
Each registered backend may now define:
|
||||
|
||||
- an active model-generation limit; and
|
||||
- a bounded number of additional admitted `Run` calls.
|
||||
|
||||
Promptkit owns scheduling for both its built-in model client and an injected
|
||||
`LLMClient`. `Run` remains synchronous: an admitted caller waits for its
|
||||
ordinary result, while a call beyond the bounded admission capacity returns
|
||||
`ErrCapacityExceeded`. Capacity is engine-local and keyed by backend ID.
|
||||
Endpoint-only profiles and custom backends without a configured limit remain
|
||||
unlimited.
|
||||
|
||||
The built-in OpenRouter backend now permits 16 active generations and 1024
|
||||
additional admitted calls per engine. Applications that can exceed this bound
|
||||
should handle capacity exhaustion separately from provider and request
|
||||
failures:
|
||||
|
||||
```go
|
||||
result, err := engine.Run(ctx, request)
|
||||
if errors.Is(err, promptkit.ErrCapacityExceeded) {
|
||||
// Apply application-specific overload or retry policy.
|
||||
}
|
||||
```
|
||||
|
||||
Promptkit does not prescribe retries or map this error to an HTTP status. See
|
||||
the
|
||||
[concurrency consumer guidance](../consumers/pkg-promptkit.md#limit-backend-concurrency)
|
||||
and the [`Backend` GoDoc](../../backends.go) for the canonical configuration
|
||||
contract. Runtime behavior and public error identities belong to the
|
||||
[`Engine.Run` GoDoc](../../engine.go).
|
||||
|
||||
## Per-Run Session IDs
|
||||
|
||||
`RunRequest.SessionID` can now supply a consumer-managed correlation ID for one
|
||||
`Prepare` or `Run` invocation. A nonblank direct value overrides the prompt's
|
||||
session template and is exposed in prepared values, results, injected-client
|
||||
requests, and provider observability. Session IDs should therefore be stable,
|
||||
non-secret values.
|
||||
|
||||
```go
|
||||
result, err := engine.Run(ctx, promptkit.RunRequest{
|
||||
PromptID: "meeting.summary",
|
||||
SessionID: "conversation-42",
|
||||
})
|
||||
```
|
||||
|
||||
The built-in OpenAI-compatible client sends a nonempty effective session as the
|
||||
top-level `session_id` request-body field, not as an `x-session-id` header. See
|
||||
the
|
||||
[session and reasoning consumer guide](../consumers/pkg-promptkit.md#set-a-per-run-session-and-reasoning),
|
||||
the [`RunRequest` GoDoc](../../types.go), and the
|
||||
[OpenAI-compatible request contract](../integrations/openai-compatible-chat.md#request-body)
|
||||
for exact normalization, length, exposure, and wire behavior.
|
||||
|
||||
## Per-Run Reasoning Effort
|
||||
|
||||
`ExecutionTargetOverride.ReasoningEffort` changed from `string` to `*string` so
|
||||
one request can distinguish inheritance, replacement, and explicit clearing.
|
||||
|
||||
Update a `v0.1.0` override like this:
|
||||
|
||||
```go
|
||||
// v0.1.0
|
||||
Execution: &promptkit.ExecutionTargetOverride{
|
||||
ReasoningEffort: "high",
|
||||
}
|
||||
```
|
||||
|
||||
to:
|
||||
|
||||
```go
|
||||
// v0.2.0
|
||||
reasoning := "high"
|
||||
Execution: &promptkit.ExecutionTargetOverride{
|
||||
ReasoningEffort: &reasoning,
|
||||
}
|
||||
```
|
||||
|
||||
The three states are:
|
||||
|
||||
- `nil` inherits the selected profile's value;
|
||||
- a pointer to a nonblank string replaces it for that invocation; and
|
||||
- a pointer to an empty or whitespace-only string clears it for that
|
||||
invocation.
|
||||
|
||||
This allows consumers to consolidate profiles that differed only by reasoning
|
||||
effort. The [`ExecutionTargetOverride` GoDoc](../../types.go) owns the exact
|
||||
override contract.
|
||||
|
||||
## Other Migration Notes
|
||||
|
||||
### Remove `RunRequest.Metadata`
|
||||
|
||||
`RunRequest.Metadata` is no longer part of the public request. Remove any
|
||||
assignment to that field. Use application-owned state keyed by `RunResult.RunID`
|
||||
or a direct `SessionID` when correlation is needed; these identifiers have
|
||||
different purposes, so choose according to the application's lifecycle.
|
||||
|
||||
### Review Persisted JSON
|
||||
|
||||
`v0.2.0` defines stable JSON representations for the public result, artifact,
|
||||
execution, validation, and model-client values listed in the
|
||||
[package documentation](../../doc.go). Consumers that treated `v0.1.0`
|
||||
reflection-derived encodings as stable should update fixtures and stored-data
|
||||
adapters.
|
||||
|
||||
In particular:
|
||||
|
||||
- `RunResult` encodes elapsed time as integer milliseconds in `duration_ms`
|
||||
instead of encoding `time.Duration` under `duration`;
|
||||
- result JSON can include the new `session_id` and `selected_backend_id`
|
||||
fields;
|
||||
- execution-target JSON can include `backend_id`; and
|
||||
- artifact and target-presence fields now use their documented lower-case
|
||||
names.
|
||||
|
||||
The `v0.2.0` `RunResult` decoder reads `duration_ms`; it does not translate a
|
||||
persisted `v0.1.0` `duration` field. Transform old payloads before decoding
|
||||
when preserving their elapsed duration matters.
|
||||
|
||||
### Prefer Keyed Struct Literals
|
||||
|
||||
New fields were added to several public structs. Replace positional composite
|
||||
literals with keyed literals so future additive fields do not cause another
|
||||
source migration.
|
||||
|
||||
## Migration Checklist
|
||||
|
||||
- Update the module dependency and run the consumer's tests.
|
||||
- Change reasoning overrides from strings to pointers.
|
||||
- Remove uses of `RunRequest.Metadata`.
|
||||
- Review unkeyed Promptkit struct literals.
|
||||
- Decide whether shared endpoints should move into registered backends.
|
||||
- If using built-in OpenRouter profiles at high concurrency, handle
|
||||
`ErrCapacityExceeded` and review the new engine-local bound.
|
||||
- Review stored JSON, fixtures, and downstream decoders.
|
||||
- Optionally replace profile-specific session or reasoning variants with
|
||||
per-run overrides.
|
||||
|
||||
For complete consumer workflows, use the
|
||||
[package consumer guide](../consumers/pkg-promptkit.md) and maintained
|
||||
[offline execution example](../../examples/go-library/run/main.go).
|
||||
74
docs/releases/v0.3.0.md
Normal file
74
docs/releases/v0.3.0.md
Normal file
@@ -0,0 +1,74 @@
|
||||
# Promptkit v0.3.0
|
||||
|
||||
This supplemental changelog summarizes the consumer-facing changes from
|
||||
`v0.2.0` to `v0.3.0`. The annotated `v0.3.0` tag is the authoritative release
|
||||
record. Exact current contracts belong to the linked GoDoc and durable
|
||||
documentation.
|
||||
|
||||
## Summary
|
||||
|
||||
`v0.3.0` adds a concise way to register the common local OpenAI-compatible
|
||||
backend configuration:
|
||||
|
||||
- `BackendLocal` provides the conventional, non-reserved backend ID `"local"`;
|
||||
and
|
||||
- `LocalBackend` constructs an ordinary `Backend` from an endpoint and
|
||||
concurrency limit.
|
||||
|
||||
The helper is explicit and additive. It does not pre-register a backend, read
|
||||
environment variables, select a model, or replace the complete `Backend`
|
||||
configuration interface.
|
||||
|
||||
## Compatibility
|
||||
|
||||
Existing `v0.2.0` consumers require no migration. Endpoint-only profiles,
|
||||
complete custom `Backend` values, the built-in OpenRouter backend, and existing
|
||||
registrations using the literal ID `"local"` continue to work unchanged.
|
||||
|
||||
## Upgrade
|
||||
|
||||
Update the module dependency with:
|
||||
|
||||
```sh
|
||||
go get gitea.maximumdirect.net/eric/promptkit@v0.3.0
|
||||
go mod tidy
|
||||
```
|
||||
|
||||
Run the consuming project's ordinary tests and race-enabled tests after the
|
||||
upgrade.
|
||||
|
||||
## Configure A Local Backend
|
||||
|
||||
Register the convenience value through the existing `WithBackend` option and
|
||||
select it from one or more profiles:
|
||||
|
||||
```go
|
||||
engine, err := promptkit.NewEngine(
|
||||
promptkit.Config{PromptDir: "prompts"},
|
||||
promptkit.WithBackend(
|
||||
promptkit.LocalBackend("http://localhost:8000/v1", 2),
|
||||
),
|
||||
promptkit.WithProfiles(promptkit.Profile{
|
||||
ID: "local-summary",
|
||||
BackendID: promptkit.BackendLocal,
|
||||
Model: "example-model",
|
||||
}),
|
||||
)
|
||||
```
|
||||
|
||||
Use an endpoint-only profile when shared backend identity and capacity policy
|
||||
are unnecessary. Continue to use a complete keyed `Backend` value for custom
|
||||
IDs, authentication, extra request parameters, explicit queue capacity, or
|
||||
multiple local endpoints.
|
||||
|
||||
See the
|
||||
[local-endpoint consumer guide](../consumers/pkg-promptkit.md#configure-a-local-openai-compatible-endpoint)
|
||||
for task-oriented configuration choices. The
|
||||
[`BackendLocal`, `LocalBackend`, and `WithBackend` GoDoc](../../backends.go)
|
||||
owns their exact construction, registration, validation, and concurrency
|
||||
semantics.
|
||||
|
||||
## Consumer Action
|
||||
|
||||
None. Adopt the convenience constructor when it simplifies local endpoint
|
||||
configuration.
|
||||
189
docs/releases/v0.4.0.md
Normal file
189
docs/releases/v0.4.0.md
Normal file
@@ -0,0 +1,189 @@
|
||||
# Promptkit v0.4.0
|
||||
|
||||
This supplemental changelog and adoption guide summarizes the consumer-facing
|
||||
changes from `v0.3.0` to `v0.4.0`. The annotated `v0.4.0` tag is the
|
||||
authoritative release record. Exact current contracts belong to the linked
|
||||
GoDoc and durable documentation.
|
||||
|
||||
## Summary
|
||||
|
||||
`v0.4.0` adds four complementary capabilities:
|
||||
|
||||
- opaque prepared-execution handles for preparing once, inspecting safe
|
||||
details, and executing the same frozen snapshot;
|
||||
- exact prompt-definition inspection without profile resolution or execution;
|
||||
- exact profile inspection without selecting a prompt or checking credential
|
||||
availability; and
|
||||
- structured backend identity on engine admission-capacity rejection.
|
||||
|
||||
These APIs let consumers perform more precise preflight work and retain useful
|
||||
operational context without reproducing Promptkit's internal resolution logic.
|
||||
|
||||
## Compatibility
|
||||
|
||||
The release is additive for `v0.3.0` consumers. Existing uses of `Prepare`,
|
||||
`Run`, backend registration, endpoint-only profiles, local-backend helpers,
|
||||
runtime overrides, public JSON values, and error sentinels continue to work
|
||||
without migration.
|
||||
|
||||
Capacity rejection now returns a structured error while continuing to match
|
||||
`ErrCapacityExceeded` through `errors.Is`. Error-string wording and direct
|
||||
sentinel equality were not public contracts.
|
||||
|
||||
The new inspection values, capacity error, and prepared-execution handle do not
|
||||
have stable JSON representations. `PreparedExecution.Details` returns the
|
||||
existing stable `PreparedRun` value.
|
||||
|
||||
## Upgrade
|
||||
|
||||
Update the module dependency with:
|
||||
|
||||
```sh
|
||||
go get gitea.maximumdirect.net/eric/promptkit@v0.4.0
|
||||
go mod tidy
|
||||
```
|
||||
|
||||
Run the consuming project's ordinary and race-enabled tests after upgrading.
|
||||
No source migration is required.
|
||||
|
||||
## Prepare Once And Execute The Same Snapshot
|
||||
|
||||
Consumers that need to persist preparation details before generation can now
|
||||
prepare an opaque, engine-bound execution:
|
||||
|
||||
```go
|
||||
prepared, err := engine.PrepareExecution(ctx, request)
|
||||
if err != nil {
|
||||
// Handle preparation failure.
|
||||
}
|
||||
defer prepared.Discard()
|
||||
|
||||
details := prepared.Details()
|
||||
// Persist a consumer-selected, appropriately protected preparation record.
|
||||
|
||||
result, err := engine.RunPrepared(ctx, prepared)
|
||||
```
|
||||
|
||||
Preparation freezes the selected sources, rendered messages, effective
|
||||
settings, input content, structured-output metadata, and validation resources
|
||||
needed by execution. `Details` returns a fresh, caller-owned,
|
||||
credential-redacted `PreparedRun`.
|
||||
|
||||
A handle belongs to its creating engine and permits one execution attempt.
|
||||
`RunPrepared` consumes that attempt on success and on operational failure.
|
||||
`Discard` is idempotent and releases an unclaimed handle's execution-only
|
||||
state. Consumers should discard handles they will not execute, particularly
|
||||
when a direct request API key may be retained privately until claim or
|
||||
discard.
|
||||
|
||||
Prepared execution does not reserve backend admission during preparation.
|
||||
Credential availability and backend admission are checked when execution
|
||||
begins. The execution context is independent of the preparation context.
|
||||
|
||||
See the
|
||||
[prepared-execution consumer guide](../consumers/pkg-promptkit.md#prepare-now-and-execute-the-same-snapshot-later),
|
||||
the [`PreparedExecution` GoDoc](../../prepared_execution.go), and the
|
||||
[`Engine.PrepareExecution` and `Engine.RunPrepared` GoDoc](../../engine.go)
|
||||
for the exact lifecycle, ownership, cancellation, capacity, timing, and
|
||||
failure contracts.
|
||||
|
||||
## Inspect A Prompt
|
||||
|
||||
`Engine.InspectPrompt` resolves one prompt ID and optional version through the
|
||||
engine's configured prompt source:
|
||||
|
||||
```go
|
||||
inspection, err := engine.InspectPrompt(ctx, "report.summary", "")
|
||||
```
|
||||
|
||||
The result includes prompt identity, the opaque prompt hash, declared default
|
||||
profile ID, declared input metadata, and normalized output contract. It
|
||||
structurally loads the selected definition and referenced message content but
|
||||
does not resolve a profile, load schemas or artifacts, render templates,
|
||||
reserve capacity, or contact a model.
|
||||
|
||||
Use inspection for exact configuration checks and metadata discovery. Use
|
||||
`PrepareExecution` rather than relying on a prior inspection when later
|
||||
execution must freeze one exact source state, because filesystem-backed
|
||||
inspection is only a point-in-time lookup.
|
||||
|
||||
See the
|
||||
[prompt-inspection consumer guide](../consumers/pkg-promptkit.md#inspect-a-prompt-before-preparation)
|
||||
and [`Engine.InspectPrompt` GoDoc](../../engine.go) for exact selection,
|
||||
ownership, and error behavior.
|
||||
|
||||
## Inspect A Profile
|
||||
|
||||
`Engine.InspectProfile` resolves one explicit profile independently of a
|
||||
prompt:
|
||||
|
||||
```go
|
||||
inspection, err := engine.InspectProfile(ctx, "report-production")
|
||||
```
|
||||
|
||||
The result includes the resolved effective execution target and whether a
|
||||
later request must provide a direct credential. Environment-variable names may
|
||||
be reported, but inspection does not read credential values or require the
|
||||
named variable to be populated.
|
||||
|
||||
Inspection applies the engine's profile source precedence and resolves any
|
||||
selected backend. It does not load a prompt, render content, reserve capacity,
|
||||
or contact a model.
|
||||
|
||||
See the
|
||||
[profile-inspection consumer guide](../consumers/pkg-promptkit.md#inspect-a-profile-before-prompt-work)
|
||||
and [`Engine.InspectProfile` GoDoc](../../engine.go) for the exact resolution,
|
||||
credential, ownership, and error contracts.
|
||||
|
||||
## Identify Capacity-Rejected Backends
|
||||
|
||||
Calls rejected at Promptkit's bounded engine admission boundary continue to
|
||||
match `ErrCapacityExceeded`. Consumers can additionally obtain the selected
|
||||
registered backend ID without parsing diagnostic text:
|
||||
|
||||
```go
|
||||
result, err := engine.Run(ctx, request)
|
||||
if errors.Is(err, promptkit.ErrCapacityExceeded) {
|
||||
var capacityErr *promptkit.CapacityError
|
||||
if errors.As(err, &capacityErr) {
|
||||
// Record capacityErr.BackendID using application-owned diagnostics.
|
||||
}
|
||||
|
||||
// Apply application-owned overload or retry policy.
|
||||
}
|
||||
```
|
||||
|
||||
The structured error applies to `Run` and `RunPrepared` admission rejection.
|
||||
It does not represent provider throttling, quota exhaustion, cancellation
|
||||
while waiting for generation capacity, or another model-client failure.
|
||||
Promptkit does not prescribe retry timing or transport status mapping.
|
||||
|
||||
See the
|
||||
[error-handling consumer guide](../consumers/pkg-promptkit.md#handle-errors),
|
||||
the [`CapacityError` GoDoc](../../capacity_error.go), and the
|
||||
[`ErrCapacityExceeded` GoDoc](../../engine.go) for the canonical contracts.
|
||||
|
||||
## Public API Additions
|
||||
|
||||
The release adds:
|
||||
|
||||
- `Engine.PrepareExecution`;
|
||||
- `Engine.RunPrepared`;
|
||||
- `PreparedExecution`, including `Details`, `Discard`, `String`, and
|
||||
`GoString`;
|
||||
- `Engine.InspectPrompt`;
|
||||
- `PromptInspection`;
|
||||
- `PromptInputDefinition`;
|
||||
- `Engine.InspectProfile`;
|
||||
- `ProfileInspection`; and
|
||||
- `CapacityError`.
|
||||
|
||||
No public API was removed.
|
||||
|
||||
## Consumer Action
|
||||
|
||||
None. Existing `v0.3.0` workflows may upgrade without adopting the new APIs.
|
||||
|
||||
Consumers that adopt prepared execution should discard unused handles.
|
||||
Consumers that need backend-specific capacity diagnostics may add an
|
||||
`errors.As` check while retaining their existing `errors.Is` classification.
|
||||
125
docs/releases/v0.5.0.md
Normal file
125
docs/releases/v0.5.0.md
Normal file
@@ -0,0 +1,125 @@
|
||||
# Promptkit v0.5.0
|
||||
|
||||
This supplemental changelog and migration guide summarizes the consumer-facing
|
||||
changes from `v0.4.0` to `v0.5.0`. The annotated `v0.5.0` tag is the
|
||||
authoritative release record. Exact current contracts belong to the linked
|
||||
GoDoc and durable documentation.
|
||||
|
||||
## Summary
|
||||
|
||||
`v0.5.0` makes provider requests less prescriptive and adds an application
|
||||
fallback layer for profile definitions:
|
||||
|
||||
- unset optional provider controls are omitted from OpenAI-compatible request
|
||||
bodies instead of being populated with framework values; and
|
||||
- `WithFallbackProfileFS` lets an application package profile defaults that
|
||||
operators can override through the existing ordinary profile sources.
|
||||
|
||||
These changes let compatible providers apply their own model defaults while
|
||||
giving applications stable embedded profile IDs without weakening operator
|
||||
configuration precedence.
|
||||
|
||||
## Compatibility
|
||||
|
||||
The release adds one public function and removes no public declaration.
|
||||
Existing source code should continue to compile.
|
||||
|
||||
There is one intentional behavior change: when no profile or runtime override
|
||||
selects `top_p`, Promptkit no longer sends the former framework value of `1`.
|
||||
It omits `top_p` and lets the provider choose its behavior. Unset
|
||||
`temperature` and `max_tokens` are likewise omitted. Explicit nonzero profile
|
||||
values and runtime values—including explicit runtime zero values—retain their
|
||||
precedence and wire effect.
|
||||
|
||||
Consumers that relied on Promptkit always sending `top_p: 1` should add that
|
||||
value to the relevant profile or runtime override before upgrading. Consumers
|
||||
that did not rely on the implicit sampling value require no migration.
|
||||
|
||||
Application fallback profiles are opt-in. Engines that do not call
|
||||
`WithFallbackProfileFS` retain the previous profile-source behavior.
|
||||
|
||||
## Upgrade
|
||||
|
||||
Update the module dependency with:
|
||||
|
||||
```sh
|
||||
go get gitea.maximumdirect.net/eric/promptkit@v0.5.0
|
||||
go mod tidy
|
||||
```
|
||||
|
||||
Run the consuming project's ordinary and race-enabled tests after upgrading.
|
||||
If request payloads or model behavior are asserted in fixtures, review them for
|
||||
the optional-parameter omission described below.
|
||||
|
||||
## Omitted Optional Provider Controls
|
||||
|
||||
The built-in OpenAI-compatible client now includes `temperature`,
|
||||
`max_tokens`, and `top_p` only when a profile or runtime override selects the
|
||||
value. An explicit runtime zero remains present because runtime override
|
||||
pointers distinguish zero from an unspecified value.
|
||||
|
||||
Promptkit's positive generation deadline remains a framework concern and is
|
||||
not a provider request-body default. Required request fields, session IDs,
|
||||
structured output, reasoning selection, credentials, and explicit extra
|
||||
parameters retain their existing behavior.
|
||||
|
||||
See the [framework default and precedence reference](../formats.md#defaults-and-overrides),
|
||||
the [`ExecutionTargetOverride` GoDoc](../../types.go), and the
|
||||
[OpenAI-compatible request-body contract](../integrations/openai-compatible-chat.md#request-body)
|
||||
for current details.
|
||||
|
||||
## Embedded Application Fallback Profiles
|
||||
|
||||
Applications can package ordinary profile YAML in an `fs.FS` and register it
|
||||
as a fallback source:
|
||||
|
||||
```go
|
||||
//go:embed profiles/*.yaml
|
||||
var applicationProfiles embed.FS
|
||||
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{
|
||||
PromptDir: "prompts",
|
||||
ProfileDir: operatorProfileDir,
|
||||
},
|
||||
promptkit.WithFallbackProfileFS(applicationProfiles, "profiles"),
|
||||
)
|
||||
```
|
||||
|
||||
Leave `operatorProfileDir` empty when no operator source is configured. A
|
||||
configured ordinary source is authoritative: a matching definition overrides
|
||||
the application fallback, while a read or validation failure remains an error
|
||||
instead of silently reaching a lower layer.
|
||||
|
||||
Profile definitions resolve in this order:
|
||||
|
||||
1. in-memory profiles supplied with `WithProfiles`;
|
||||
2. the ordinary configured source selected by `WithProfileFile`,
|
||||
`WithProfileFS`, or `Config.ProfileDir`;
|
||||
3. the application source supplied with `WithFallbackProfileFS`; and
|
||||
4. Promptkit's embedded built-in profiles.
|
||||
|
||||
Only an absent profile ID falls through. Sources provide complete profiles and
|
||||
do not merge fields. Loading remains lazy, and the new source uses the existing
|
||||
strict profile YAML and credential rules.
|
||||
|
||||
See the
|
||||
[embedded-default consumer guidance](../consumers/pkg-promptkit.md#supply-embedded-application-defaults),
|
||||
the [`WithFallbackProfileFS` GoDoc](../../engine.go), and the
|
||||
[profile source reference](../formats.md#source-and-profile-precedence) for
|
||||
current details.
|
||||
|
||||
## Public API Changes
|
||||
|
||||
The release adds:
|
||||
|
||||
- `WithFallbackProfileFS`.
|
||||
|
||||
No public declaration was removed or changed.
|
||||
|
||||
## Consumer Action
|
||||
|
||||
- Review any workflow that depended on Promptkit's implicit `top_p: 1` and
|
||||
configure the value explicitly when required.
|
||||
- Optionally adopt `WithFallbackProfileFS` when an application should package
|
||||
overridable profile defaults.
|
||||
- Run consumer tests after updating the module dependency.
|
||||
635
docs/roadmap/audit-sequence.md
Normal file
635
docs/roadmap/audit-sequence.md
Normal file
@@ -0,0 +1,635 @@
|
||||
# Codebase Audit Sequence
|
||||
|
||||
## Purpose
|
||||
|
||||
This document defines the staged sequence for auditing Promptkit before further
|
||||
feature development. The audit is intended to identify high-confidence
|
||||
opportunities to improve correctness, efficiency, duplication, implementation
|
||||
clarity, and test-suite quality without changing production behavior during the
|
||||
review itself.
|
||||
|
||||
The audit findings belong in `audit.md`. A later, separate planning pass will
|
||||
translate accepted findings into a staged remediation plan in
|
||||
`implementation.md`. Neither this sequence nor the findings log owns current
|
||||
behavior; the canonical sources identified by the
|
||||
[documentation policy](../policy/documentation.md) remain authoritative.
|
||||
|
||||
Each stage below is deliberately scoped for one LLM coding-agent prompt. Run
|
||||
the stages in order and do not combine them. A stage may discover a concern
|
||||
outside its scope, but it should record that concern for the owning later stage
|
||||
rather than expanding its own review.
|
||||
|
||||
## Governing Policies And Boundaries
|
||||
|
||||
Every stage must follow:
|
||||
|
||||
- the [development guide](../development.md), including its task-specific
|
||||
reading guide;
|
||||
- the [architecture policy](../policy/architecture.md), especially the public
|
||||
facade, internal-package, dependency-direction, and consumer boundaries;
|
||||
- the [testing policy](../policy/testing.md), including its risk-based,
|
||||
behavior-oriented standard; and
|
||||
- the [documentation policy](../policy/documentation.md), including canonical
|
||||
ownership and the temporary nature of roadmap documents.
|
||||
|
||||
This is an audit, not an implementation pass:
|
||||
|
||||
- Do not change production code, tests, examples, fixtures, public contracts,
|
||||
or current-state documentation.
|
||||
- Limit repository edits to the audit artifacts explicitly authorized for the
|
||||
stage.
|
||||
- Do not silently repair an issue while investigating it.
|
||||
- Do not treat coverage, complexity, similarity, lint, or graph output as a
|
||||
finding without confirming the underlying behavior in source and tests.
|
||||
- Do not recommend centralization merely because code looks similar. The code
|
||||
must implement the same semantic rule, and consolidation must improve
|
||||
ownership or reduce a credible drift risk.
|
||||
- Do not recommend performance work without identifying a relevant execution
|
||||
path and establishing a defensible cost model, measurement, or complexity
|
||||
problem.
|
||||
- Preserve unrelated working-tree changes. Record the audit baseline rather
|
||||
than requiring an otherwise unrelated dirty tree to be cleaned.
|
||||
|
||||
## Finding Standard
|
||||
|
||||
Record each actionable finding in `audit.md` with:
|
||||
|
||||
- a stable ID in the form `SNN-FNN`, where the first number is the stage;
|
||||
- category: correctness, efficiency, duplication, clarity, testing, or
|
||||
contract-documentation consistency;
|
||||
- severity: critical, high, medium, or low;
|
||||
- confidence: confirmed, high, medium, or low;
|
||||
- affected packages, files, symbols, and tests;
|
||||
- the contract, invariant, policy, or maintenance concern at issue;
|
||||
- concrete evidence and a concise explanation of the failure mode or cost;
|
||||
- the recommended direction, without implementation-level sequencing;
|
||||
- the verification or regression protection that remediation would require;
|
||||
and
|
||||
- status: accepted, deferred, rejected, superseded, or resolved.
|
||||
|
||||
Use **confirmed** confidence when the problem is reproduced or follows
|
||||
unavoidably from a complete trace. Use **high** confidence when direct source
|
||||
and test evidence establishes the problem but a safe reproduction is not
|
||||
practical. Medium- and low-confidence concerns belong in a separate
|
||||
observations section until a later stage confirms or rejects them; they must
|
||||
not enter the remediation plan as if they were findings.
|
||||
|
||||
Severity describes impact, not implementation effort:
|
||||
|
||||
- **Critical:** credible data disclosure, data corruption, deadlock, unbounded
|
||||
resource consumption, or a broadly unusable public contract.
|
||||
- **High:** violation of an important public contract or invariant, a likely
|
||||
concurrency or resource-lifecycle defect, or a failure with substantial
|
||||
downstream impact.
|
||||
- **Medium:** a real but narrower behavioral defect, meaningful avoidable cost,
|
||||
duplicated policy with credible drift risk, or a material testing gap.
|
||||
- **Low:** a bounded clarity, maintainability, or testing-friction problem with
|
||||
a concrete improvement and little behavioral risk.
|
||||
|
||||
When a reviewed area yields no finding, record the important behavior or risk
|
||||
that was inspected and found adequately implemented or tested. This coverage
|
||||
ledger prevents later reviewers from mistaking silence for omission.
|
||||
|
||||
## Per-Stage Procedure
|
||||
|
||||
Unless a stage says otherwise, its single agent prompt should:
|
||||
|
||||
1. Read the required policies, focused internal documentation, production
|
||||
files, and tests for that stage.
|
||||
2. Use the code knowledge graph for symbol discovery, callers, callees, and
|
||||
cross-package traces; confirm important conclusions against source.
|
||||
3. Trace normal, boundary, and failure paths through the narrowest relevant
|
||||
public or package contract.
|
||||
4. Review correctness, meaningful runtime cost, semantic duplication,
|
||||
responsibility clarity, and the value and ownership of tests in scope.
|
||||
5. Run the narrowest existing tests needed to validate conclusions. Use
|
||||
race-enabled or repeated focused tests when concurrency or nondeterminism is
|
||||
in scope. Do not add permanent tests during the audit.
|
||||
6. Add the stage result to `audit.md`: accepted findings, unresolved
|
||||
observations, areas verified, commands run, and any handoff to a later
|
||||
stage.
|
||||
7. Recheck the working tree and confirm that only the authorized audit artifact
|
||||
changed.
|
||||
|
||||
## Stage 0: Initialize The Audit And Establish The Baseline
|
||||
|
||||
Create `audit.md` and establish a reproducible starting point before reviewing
|
||||
individual components.
|
||||
|
||||
Record:
|
||||
|
||||
- the audited commit, branch, Go version, module identity, and working-tree
|
||||
state;
|
||||
- unrelated pre-existing changes that all later stages must preserve;
|
||||
- the implemented package and public-facade inventory;
|
||||
- the baseline validation results; and
|
||||
- the finding template, status vocabulary, and coverage ledger used by later
|
||||
stages.
|
||||
|
||||
Refresh the code knowledge graph for the recorded commit. Run the repository's
|
||||
ordinary tests, race tests, vet, build, maintained offline preparation example,
|
||||
Go formatting check, Markdown link check, and repository-hygiene checks. Run
|
||||
package coverage once as a diagnostic and record the result without defining a
|
||||
coverage target or committing generated output. Measure coarse package test
|
||||
duration only if it can be done without adding tooling or changing tests.
|
||||
|
||||
Compare the validation requirements stated by the testing policy, development
|
||||
guide, and release procedure. Record a finding if their ownership or command
|
||||
sets are materially inconsistent; do not edit those documents in this stage.
|
||||
|
||||
**Exit condition:** `audit.md` contains the baseline, ledger structure, and
|
||||
validation result, and no component-level audit has begun.
|
||||
|
||||
## Stage 1: Public Values, Conversion, Errors, And Formatting
|
||||
|
||||
Review the root facade's public request, result, inspection, prepared-run, and
|
||||
error values together with public-to-internal and internal-to-public
|
||||
conversion. Scope the review to `doc.go`, `types.go`, `convert.go`, `errors.go`,
|
||||
`capacity_error.go`, `formatting.go`, and `prepared_execution.go`, plus the
|
||||
directly relevant portions of root tests.
|
||||
|
||||
Focus on:
|
||||
|
||||
- zero-value and nil behavior;
|
||||
- defensive copying, aliasing, and immutable snapshots;
|
||||
- lossless conversion and field precedence;
|
||||
- error identity through `errors.Is` and `errors.As`;
|
||||
- containment of internal representations;
|
||||
- safe `String`, `GoString`, and diagnostic formatting;
|
||||
- accidental disclosure of credentials, prompt content, generated content, or
|
||||
other private state; and
|
||||
- conversion or copying logic that represents the same rule in multiple
|
||||
places.
|
||||
|
||||
Review only tests that own these value and boundary contracts. Defer engine
|
||||
assembly, execution coordination, and transport behavior to their later
|
||||
stages.
|
||||
|
||||
**Exit condition:** all root value-conversion and error-formatting paths have a
|
||||
recorded audit result without evaluating engine orchestration.
|
||||
|
||||
## Stage 2: Public Configuration And Extension Adapters
|
||||
|
||||
Review the smaller public construction and extension surfaces in
|
||||
`backends.go`, `profiles.go`, `artifact_reader.go`, `json.go`, and
|
||||
`llm_adapter.go`, together with their directly relevant root and internal
|
||||
adapter tests.
|
||||
|
||||
Focus on:
|
||||
|
||||
- validation performed at the public boundary;
|
||||
- ownership and copying of caller-supplied maps, slices, filesystems, readers,
|
||||
and clients;
|
||||
- adapter error propagation and cancellation;
|
||||
- consistency between convenience constructors and general configuration;
|
||||
- whether extension interfaces are as narrow as their consumers require;
|
||||
- whether public helpers duplicate internal policy or merely translate it;
|
||||
and
|
||||
- whether tests protect consumer-visible behavior rather than private adapter
|
||||
choreography.
|
||||
|
||||
Do not review how `NewEngine` combines these values; that belongs to Stage 3.
|
||||
|
||||
**Exit condition:** every non-engine public configuration helper and adapter
|
||||
has a recorded result and any assembly questions are handed to Stage 3.
|
||||
|
||||
## Stage 3: Engine Construction, Options, And Source Assembly
|
||||
|
||||
Review the construction and configuration portions of `engine.go` and the
|
||||
corresponding tests in `engine_test.go`. Limit the scope to `NewEngine`, option
|
||||
application, dependency defaults, backend registration, profile and prompt
|
||||
source composition, fallback-profile placement, validator and client
|
||||
selection, capacity-manager construction, and construction-time validation.
|
||||
|
||||
Focus on:
|
||||
|
||||
- deterministic option precedence;
|
||||
- required versus optional dependencies;
|
||||
- isolation between engine instances;
|
||||
- freezing or copying consumer configuration at the correct boundary;
|
||||
- correct dependency direction and absence of process-global mutable state;
|
||||
- failure atomicity and useful public errors;
|
||||
- consistency between configured backends and capacity policies; and
|
||||
- assembly logic that is repeated or split across unclear owners.
|
||||
|
||||
Do not audit the runtime behavior of `Run`, `Prepare`, or inspection methods;
|
||||
that belongs to Stage 4 and the internal use-case stages.
|
||||
|
||||
**Exit condition:** engine construction and source assembly are fully accounted
|
||||
for, including tests, without expanding into runtime orchestration.
|
||||
|
||||
## Stage 4: Engine Operations And Root Contract Coverage
|
||||
|
||||
Review the remaining public methods in `engine.go` and their directly relevant
|
||||
root tests, including the external-package contracts in
|
||||
`public_contract_test.go` and `prepared_execution_contract_test.go` only where
|
||||
they exercise the engine boundary under review.
|
||||
|
||||
Focus on:
|
||||
|
||||
- request translation and context propagation;
|
||||
- ordinary run, preparation, inspection, and prepared-execution entry points;
|
||||
- public error mapping and preservation of injected dependency errors;
|
||||
- result and prepared-state ownership;
|
||||
- consistency between method and package-level convenience functions;
|
||||
- public behavior that is asserted redundantly in root internal tests and
|
||||
external-package contract tests; and
|
||||
- important public behavior that is tested only through internal packages.
|
||||
|
||||
Treat internal runner, transport, validation, and capacity mechanics as black
|
||||
boxes in this stage. Hand questions about their implementation to their owning
|
||||
later stages.
|
||||
|
||||
**Exit condition:** the public execution boundary and its contract-test
|
||||
ownership are recorded without duplicating internal component audits.
|
||||
|
||||
## Stage 5: Internal Domain And JSON-Compatible Values
|
||||
|
||||
Review `internal/domain` and `internal/jsonvalue`, including all of their tests.
|
||||
|
||||
Focus on:
|
||||
|
||||
- domain invariants and invalid states;
|
||||
- session normalization;
|
||||
- prepared-run and schema immutability;
|
||||
- deep-copy correctness for every supported JSON-compatible shape;
|
||||
- numeric-type preservation and rejection policy;
|
||||
- cycles, excessive nesting, unsupported values, and nil distinctions;
|
||||
- avoidable repeated copying on execution paths; and
|
||||
- whether generic value machinery has a single clear owner.
|
||||
|
||||
Trace important callers to confirm that these packages enforce the invariants
|
||||
their consumers assume, but do not audit the callers' broader behavior.
|
||||
|
||||
**Exit condition:** shared value semantics and their test ownership are fully
|
||||
recorded.
|
||||
|
||||
## Stage 6: Backend Registry, Defaults, And Built-In Profiles
|
||||
|
||||
Review `internal/backend`, `internal/defaults`, and
|
||||
`internal/profile/builtin`, including their focused tests and the relevant
|
||||
backend-policy traces into engine assembly and the LLM reserved-field rule.
|
||||
|
||||
Focus on:
|
||||
|
||||
- immutable registry construction and lookup;
|
||||
- built-in versus consumer ID collision rules;
|
||||
- endpoint, credential-environment, header, parameter, and concurrency
|
||||
validation;
|
||||
- defensive copies at registry boundaries;
|
||||
- application-neutral default ownership;
|
||||
- built-in profile/backend consistency;
|
||||
- reserved request-field ownership without dependency inversion; and
|
||||
- duplicated validation or default policy across public and internal layers.
|
||||
|
||||
Defer scheduling mechanics to Stage 15 and actual HTTP request construction to
|
||||
Stage 14.
|
||||
|
||||
**Exit condition:** registry and default-policy correctness are recorded, with
|
||||
transport and scheduling questions handed to their owning stages.
|
||||
|
||||
## Stage 7: File Discovery And Prompt Definitions
|
||||
|
||||
Review `internal/filecatalog` and `internal/promptdef`, including their tests
|
||||
and fixtures. Read the framework format reference and internal source document
|
||||
before evaluating behavior.
|
||||
|
||||
Focus on:
|
||||
|
||||
- deterministic discovery and duplicate handling;
|
||||
- filesystem and `fs.FS` parity;
|
||||
- root and relative-path normalization;
|
||||
- strict YAML decoding and version selection;
|
||||
- prompt ID, message, input, cache-control, and validation declarations;
|
||||
- inline versus file-backed content rules;
|
||||
- containment of referenced files where promised;
|
||||
- malformed input and contextual error behavior;
|
||||
- unnecessary repeated directory scans or file reads; and
|
||||
- fixture and case duplication that does not protect distinct parser risks.
|
||||
|
||||
Do not audit rendering, artifact loading, profile loading, or schema validation
|
||||
in this stage.
|
||||
|
||||
**Exit condition:** discovery and prompt-definition parsing have complete
|
||||
findings and coverage-ledger entries.
|
||||
|
||||
## Stage 8: Profile Sources And Repository Composition
|
||||
|
||||
Review `internal/profile` excluding its built-in subpackage, including all
|
||||
repository tests and profile fixtures. Read the profile format contract first.
|
||||
|
||||
Focus on:
|
||||
|
||||
- strict decoding and profile validation;
|
||||
- filesystem and `fs.FS` parity;
|
||||
- repository overlay and fallback precedence;
|
||||
- distinction between absence and a malformed authoritative source;
|
||||
- preservation of useful error identity and context;
|
||||
- conversion to immutable execution profiles;
|
||||
- duplicate IDs and deterministic selection;
|
||||
- repeated parsing, validation, or copying; and
|
||||
- whether tests at repository, engine, and public-contract layers have clear,
|
||||
nonduplicative ownership.
|
||||
|
||||
Defer resolution of a profile with runtime overrides and backend definitions to
|
||||
Stage 11.
|
||||
|
||||
**Exit condition:** profile-source and repository-composition behavior are
|
||||
fully recorded.
|
||||
|
||||
## Stage 9: Artifact Loading And Prompt Rendering
|
||||
|
||||
Review `internal/artifact` and `internal/prompt`, including all focused tests.
|
||||
Read the internal source document and format reference first.
|
||||
|
||||
Focus on:
|
||||
|
||||
- inline and file artifact ownership, metadata, hashing, and error behavior;
|
||||
- copied versus shared byte storage;
|
||||
- caller-selected path semantics and architecture-policy boundaries;
|
||||
- template parsing and execution;
|
||||
- artifact, variable, session, and cache-control rendering;
|
||||
- missing, extra, nil, and malformed input behavior;
|
||||
- deterministic output and safe diagnostics;
|
||||
- unnecessary repeated reads, hashes, parses, or allocations on common paths;
|
||||
and
|
||||
- tests coupled to incidental template or struct implementation.
|
||||
|
||||
Do not audit the runner's decision about when rendering occurs.
|
||||
|
||||
**Exit condition:** input materialization and rendering are accounted for
|
||||
through their package boundaries.
|
||||
|
||||
## Stage 10: Output Validation And Frozen Validation Plans
|
||||
|
||||
Review `internal/validate`, including all tests, schema fixtures used by the
|
||||
root contract suite, and traces from preparation into frozen validation plans.
|
||||
Read the format and internal source documents first.
|
||||
|
||||
Focus on:
|
||||
|
||||
- basic, JSON, and JSON Schema mode semantics;
|
||||
- schema-path resolution and filesystem/`fs.FS` parity;
|
||||
- schema compilation, transitive references, and source-lifetime independence;
|
||||
- output normalization and preservation;
|
||||
- malformed schema and malformed model-output errors;
|
||||
- thread safety of reusable validators and prepared plans;
|
||||
- expensive recompilation or copying on repeated execution; and
|
||||
- whether parser, validator, runner, and public tests each own distinct risks.
|
||||
|
||||
Do not audit repair decisions or provider request construction.
|
||||
|
||||
**Exit condition:** validation behavior, plan lifetime, and focused test value
|
||||
are fully recorded.
|
||||
|
||||
## Stage 11: Inspection And Execution-Target Resolution
|
||||
|
||||
Review `internal/usecase/profile_inspection.go`,
|
||||
`internal/usecase/prompt_inspection.go`, and the preparation and target-
|
||||
resolution portions of `internal/usecase/runner.go`, together with their
|
||||
focused tests. Use graph traces to define the exact helper and call-path scope
|
||||
before reviewing.
|
||||
|
||||
Focus on:
|
||||
|
||||
- prompt and profile selection;
|
||||
- backend lookup and endpoint overrides;
|
||||
- reasoning, session, and other runtime precedence;
|
||||
- merge semantics for default, profile, backend, and per-run values;
|
||||
- inspection fidelity versus actual execution;
|
||||
- credential-name versus credential-value handling;
|
||||
- prompt-definition and schema freezing during preparation;
|
||||
- stable error identity and context; and
|
||||
- duplicated resolution rules across inspection, preparation, and execution.
|
||||
|
||||
Do not review model invocation, repair execution, or prepared-handle lifecycle;
|
||||
those belong to Stages 12 and 13.
|
||||
|
||||
**Exit condition:** all selection, merge, inspection, and preparation rules are
|
||||
traced and recorded once.
|
||||
|
||||
## Stage 12: Ordinary Execution, Validation, And Repair Coordination
|
||||
|
||||
Review `internal/usecase/runner.go`, `internal/usecase/repairer.go`, and
|
||||
`internal/usecase/capacity_error.go` only for the ordinary execution path after
|
||||
preparation, together with the corresponding sections of `runner_test.go`.
|
||||
Use the Stage 11 resolution result as an established input rather than
|
||||
reauditing it.
|
||||
|
||||
Focus on:
|
||||
|
||||
- rendering, generation, validation, and optional repair transitions;
|
||||
- context cancellation and dependency-error propagation;
|
||||
- partial result and usage accounting;
|
||||
- exact attempt count and repair eligibility;
|
||||
- avoidance of unintended retries;
|
||||
- capacity-error translation;
|
||||
- cleanup and failure behavior on every exit path;
|
||||
- repeated orchestration or request construction; and
|
||||
- oversized tests, helpers, or case matrices that obscure distinct behavior.
|
||||
|
||||
Treat LLM transport and capacity scheduling as injected package contracts;
|
||||
their mechanics belong to Stages 14 and 15.
|
||||
|
||||
**Exit condition:** the ordinary execution state machine and its test ownership
|
||||
are fully recorded.
|
||||
|
||||
## Stage 13: Prepared Execution Lifecycle
|
||||
|
||||
Review `internal/usecase/prepared_execution.go`, its focused tests, and the
|
||||
prepared-execution portions of the root facade and external contract tests.
|
||||
Do not repeat the public value review from Stages 1 and 4 or the resolution
|
||||
review from Stage 11.
|
||||
|
||||
Focus on:
|
||||
|
||||
- single-attempt or other lifecycle guarantees;
|
||||
- concurrent use and synchronization;
|
||||
- discard behavior and resource release;
|
||||
- frozen source, target, credential, capacity, timing, and schema semantics;
|
||||
- independence of returned details and results;
|
||||
- context and error behavior;
|
||||
- consistency between ordinary and prepared execution where promised;
|
||||
- private-state containment in formatting; and
|
||||
- redundant assertions across internal, root, and external-package tests.
|
||||
|
||||
Run focused race tests and repeated tests for lifecycle behavior where useful.
|
||||
|
||||
**Exit condition:** prepared execution has one complete lifecycle analysis and
|
||||
a clear map of which test layer owns each guarantee.
|
||||
|
||||
## Stage 14: OpenAI-Compatible Transport
|
||||
|
||||
Review `internal/llm`, including all transport tests. Read the
|
||||
OpenAI-compatible integration contract and internal LLM document first.
|
||||
|
||||
Focus on:
|
||||
|
||||
- request endpoint, headers, authentication, and JSON body construction;
|
||||
- omission versus explicit zero-value behavior;
|
||||
- reserved-field enforcement and extra-parameter collision handling;
|
||||
- session ID and reasoning encoding;
|
||||
- structured-output and cache-control translation;
|
||||
- client and per-generation deadlines;
|
||||
- cancellation, body closure, bounded response reads, and decode failures;
|
||||
- non-success HTTP response behavior;
|
||||
- response choices, usage, and malformed-success handling;
|
||||
- wire-visible compatibility and safe error disclosure;
|
||||
- unnecessary marshaling, copying, or buffering; and
|
||||
- whether the large transport test file can be simplified without losing
|
||||
protocol-risk coverage.
|
||||
|
||||
Use `httptest`-based existing tests; do not contact a live provider.
|
||||
|
||||
**Exit condition:** every outbound and inbound wire path has a recorded result,
|
||||
including focused test ownership.
|
||||
|
||||
## Stage 15: Capacity, Admission, And Concurrency
|
||||
|
||||
Review `internal/capacity`, its tests, `capacity_contract_test.go`, and the
|
||||
integration points already identified in engine and use-case stages. Read the
|
||||
internal capacity document first.
|
||||
|
||||
Focus on:
|
||||
|
||||
- bounded run admission and queue-capacity enforcement;
|
||||
- per-backend limited and unlimited scheduling;
|
||||
- FIFO behavior and cancellation-safe waiter removal;
|
||||
- permit release on success, error, panic-relevant boundaries, and
|
||||
cancellation;
|
||||
- goroutine, timer, and waiter lifecycle;
|
||||
- starvation, deadlock, race, and engine-isolation risks;
|
||||
- lock scope and meaningful contention or allocation costs;
|
||||
- preservation of injected-client concurrency where promised;
|
||||
- relational testing of configured limits rather than duplicated defaults;
|
||||
and
|
||||
- duplication between internal concurrency tests and public contract tests.
|
||||
|
||||
Run focused ordinary, race-enabled, and repeated tests. Repetition must remain
|
||||
bounded and diagnostic; a test that passes many times is not proof of
|
||||
correctness without a source-level synchronization analysis.
|
||||
|
||||
**Exit condition:** concurrency invariants have both a source trace and a
|
||||
test-ownership assessment.
|
||||
|
||||
## Stage 16: Repository-Wide Test Strategy And Maintained Examples
|
||||
|
||||
Perform a suite-level review after every component has been audited. Review
|
||||
the testing policy, test inventory, fixtures, external-package root tests,
|
||||
`architecture_test.go`, and both maintained examples. Use the component-stage
|
||||
coverage ledger instead of repeating every individual test assertion.
|
||||
|
||||
Construct a risk-to-owner matrix for:
|
||||
|
||||
- public compatibility and error identity;
|
||||
- parsing, validation, and serialization;
|
||||
- immutability and data integrity;
|
||||
- external wire behavior;
|
||||
- cancellation, failure propagation, and recovery;
|
||||
- concurrency and resource lifecycle; and
|
||||
- representative assembled consumer workflows.
|
||||
|
||||
Identify only evidence-backed cases of:
|
||||
|
||||
- consequential behavior with no credible test owner;
|
||||
- the same semantic rule asserted redundantly at several layers;
|
||||
- tests coupled to private helpers, internal constants, exact noncontractual
|
||||
wording, or collaborator choreography;
|
||||
- low-value or obsolete cases whose lifetime cost exceeds their protection;
|
||||
- missing failure, cancellation, race, or boundary coverage;
|
||||
- nondeterminism, shared state, environment dependence, fixed ports, or test
|
||||
ordering assumptions;
|
||||
- helpers and fixtures whose complexity is not justified; and
|
||||
- maintained examples that duplicate one another without protecting distinct
|
||||
workflows.
|
||||
|
||||
Use coverage and timing only to direct attention. Do not propose tests solely
|
||||
to raise percentages or remove tests solely to shorten the suite.
|
||||
|
||||
**Exit condition:** every important risk has a named test owner or an accepted
|
||||
finding, and every proposed test deletion or consolidation states what
|
||||
protection remains.
|
||||
|
||||
## Stage 17: Cross-Cutting Duplication, Efficiency, And Architecture Review
|
||||
|
||||
Review the codebase as a whole using the completed component findings, graph
|
||||
traces, complexity signals, similarity signals, and package dependency map.
|
||||
Do not reopen settled package behavior without new cross-cutting evidence.
|
||||
|
||||
Focus on:
|
||||
|
||||
- one semantic policy implemented by multiple packages;
|
||||
- repeated public/internal transformations with credible drift risk;
|
||||
- interfaces broader than their actual consumers;
|
||||
- responsibilities split across packages or concentrated in the facade
|
||||
contrary to the architecture policy;
|
||||
- repeated parsing, copying, schema compilation, request construction, or
|
||||
source traversal on important paths;
|
||||
- avoidable lock contention or serial work supported by the concurrency audit;
|
||||
- abstractions that add indirection without enforcing a boundary; and
|
||||
- discrepancies between implemented package responsibilities and their
|
||||
canonical architecture or internal documentation.
|
||||
|
||||
For each possible consolidation, state why the code represents one rule, which
|
||||
package should own it, and why the resulting dependency direction remains
|
||||
valid. For each efficiency finding, state the path frequency, input scale,
|
||||
complexity or measurement evidence, and the benchmark or invariant needed to
|
||||
verify a remediation.
|
||||
|
||||
**Exit condition:** all cross-cutting opportunities are either accepted with
|
||||
high confidence, retained as explicitly lower-confidence observations, or
|
||||
rejected with a short rationale.
|
||||
|
||||
## Stage 18: Consolidate And Close The Audit
|
||||
|
||||
Perform a findings-only synthesis. Do not change code and do not write the
|
||||
remediation plan yet.
|
||||
|
||||
- Recheck every accepted finding against the final audited tree.
|
||||
- Merge duplicates and mark superseded IDs without erasing their history.
|
||||
- Separate shared root causes from downstream symptoms.
|
||||
- Confirm that every accepted item is confirmed or high confidence.
|
||||
- Confirm that severity describes impact rather than effort.
|
||||
- Reject speculative cleanup, coverage-driven test work, and centralization
|
||||
without a clear owner or drift risk.
|
||||
- Record dependencies and a recommended remediation order.
|
||||
- Distinguish behavioral fixes, safe refactors, performance work, test gaps,
|
||||
test consolidation, and documentation synchronization.
|
||||
- Add an audit summary stating what was reviewed, what validation ran, the
|
||||
accepted finding counts by category and severity, and any residual
|
||||
uncertainty.
|
||||
- Re-run baseline validation if audit-only investigation could have affected
|
||||
repository state, and confirm that only authorized roadmap files differ from
|
||||
the recorded baseline.
|
||||
|
||||
The recommended ordering should place correctness, data-integrity,
|
||||
resource-lifecycle, and concurrency defects first; policy duplication and
|
||||
missing protection for consequential behavior next; then clarity, test
|
||||
consolidation, and demonstrated efficiency improvements. Actual implementation
|
||||
stages must be decided in the later `implementation.md` planning pass, where
|
||||
files, dependencies, acceptance criteria, and validation can be made
|
||||
decision-complete.
|
||||
|
||||
**Exit condition:** `audit.md` is a complete, internally consistent input to a
|
||||
separate remediation-planning prompt, with no code or test changes mixed into
|
||||
the audit.
|
||||
|
||||
## Completion Criteria
|
||||
|
||||
The audit is complete only when:
|
||||
|
||||
- every production component and public boundary appears in the coverage
|
||||
ledger;
|
||||
- every test file and maintained example has been reviewed at its owning stage
|
||||
or in the suite-wide stage;
|
||||
- important cross-package paths have been traced end to end;
|
||||
- concurrency-sensitive behavior has received source and race-test review;
|
||||
- every accepted finding meets the evidence and confidence standard;
|
||||
- lower-confidence observations are visibly separated from remediation
|
||||
candidates;
|
||||
- proposed test additions, deletions, and consolidations are justified against
|
||||
the testing policy;
|
||||
- proposed simplifications identify a durable responsibility owner;
|
||||
- proposed efficiency work has a relevant cost model or measurement plan; and
|
||||
- the repository remains unchanged except for the authorized audit roadmap
|
||||
artifacts.
|
||||
4606
docs/roadmap/audit.md
Normal file
4606
docs/roadmap/audit.md
Normal file
File diff suppressed because it is too large
Load Diff
61
docs/roadmap/deferred.md
Normal file
61
docs/roadmap/deferred.md
Normal file
@@ -0,0 +1,61 @@
|
||||
# Deferred Feature Ideas
|
||||
|
||||
## Purpose
|
||||
|
||||
This document catalogs feature ideas that remain potentially useful but have
|
||||
been deliberately postponed. These ideas are not awaiting ordinary selection
|
||||
from the [future feature catalog](future.md); each has a stated reason to wait
|
||||
and should be reconsidered only when its trigger becomes relevant.
|
||||
|
||||
Deferred entries are not commitments, schedules, active implementation plans,
|
||||
or descriptions of current behavior. When an entry is reactivated, move it to
|
||||
`future.md` for evaluation or directly into a focused roadmap after its open
|
||||
design dependencies have been resolved.
|
||||
|
||||
## Deferred Ideas
|
||||
|
||||
### Semantic Execution-Target Fingerprints
|
||||
|
||||
**Reason for deferral:** A stable digest requires a deliberate semantic-
|
||||
equality and versioning design. Notarius can safely use conservative source
|
||||
hashes and a Promptkit release marker today, while Weatherreporter does not
|
||||
currently reuse LLM-dependent checkpoints.
|
||||
|
||||
Promptkit could expose an opaque equality value for a resolved profile and its
|
||||
effective generation target. This would let checkpointing consumers detect
|
||||
generation-affecting configuration changes without hashing YAML presentation
|
||||
or depending on Promptkit's built-in catalog layout.
|
||||
|
||||
The digest should change with semantically relevant state such as the resolved
|
||||
model, endpoint, backend routing identity, request defaults, extra parameters,
|
||||
profile generation settings, and selected built-in profile semantics. It
|
||||
should exclude credential values, concurrency and queue policy, source paths,
|
||||
comments, formatting, and other representation-only changes. Whether a
|
||||
credential environment-variable name affects equality must be decided
|
||||
explicitly. The encoding should remain opaque and internally versioned so
|
||||
Promptkit can deliberately invalidate earlier digests when its resolution
|
||||
semantics change.
|
||||
|
||||
Reconsider this idea when a downstream consumer needs Promptkit-owned
|
||||
checkpoint equality or when a broader semantic identity design is selected.
|
||||
|
||||
### Eager Source Validation
|
||||
|
||||
**Reason for deferral:** Exact prompt and profile inspection may already
|
||||
provide a sufficiently small validation surface. Experience from downstream
|
||||
adoption should establish whether an engine-wide operation would add enough
|
||||
value to justify its broader contract.
|
||||
|
||||
Promptkit could provide an explicit offline operation that discovers and
|
||||
structurally validates configured prompt, profile, and schema sources without
|
||||
model generation. The normal `NewEngine` path would remain lazy.
|
||||
|
||||
An eager operation would need coherent handling for duplicate prompt IDs and
|
||||
versions, strict YAML decoding, referenced content files, profile/backend
|
||||
membership, schema syntax and transitive references, context cancellation,
|
||||
and source-specific public errors. Credential declarations must remain
|
||||
separate from credential values; checking current environment availability,
|
||||
if supported at all, should be an explicit option and must not expose secrets.
|
||||
|
||||
Reconsider this idea after downstream use of `InspectPrompt`,
|
||||
`InspectProfile`, and fixture-based preparation demonstrates a concrete gap.
|
||||
82
docs/roadmap/future.md
Normal file
82
docs/roadmap/future.md
Normal file
@@ -0,0 +1,82 @@
|
||||
# Future Feature Ideas
|
||||
|
||||
## Purpose
|
||||
|
||||
This document catalogs reasonably specific ideas that may be useful in future
|
||||
Promptkit development. It is an idea pool, not a commitment, schedule, or
|
||||
description of current behavior.
|
||||
|
||||
Ideas belong here while they are worth retaining but have not been selected
|
||||
for active development. Keep each entry at the level of intended capability,
|
||||
consumer value, and important scope boundaries. Defer API design,
|
||||
implementation details, sequencing, and acceptance criteria until an idea is
|
||||
selected.
|
||||
|
||||
Ideas that have been deliberately postponed rather than left available for
|
||||
ordinary selection belong in the [deferred catalog](deferred.md).
|
||||
|
||||
## Using This Catalog
|
||||
|
||||
- Add an idea when its purpose and likely value can be stated clearly.
|
||||
- Keep entries independent enough that maintainers can evaluate and select
|
||||
them individually.
|
||||
- Note significant dependencies or boundary concerns, but do not turn entries
|
||||
into implementation plans.
|
||||
- Treat inclusion as an invitation to evaluate, not as approval or priority.
|
||||
- When an idea is selected, move its active planning to a focused roadmap or,
|
||||
when it requires a durable architectural decision, an ADR. Update
|
||||
current-state documentation only when implementation lands.
|
||||
- Move an idea to `deferred.md` when maintainers decide to retain it but wait
|
||||
for a stated design dependency, demand signal, or reconsideration trigger.
|
||||
- Remove ideas that are no longer relevant. Retain a rejected idea only when
|
||||
its rationale is likely to prevent repeated reconsideration.
|
||||
|
||||
Future capabilities must continue to respect the
|
||||
[architecture policy](../policy/architecture.md), particularly Promptkit's
|
||||
role as an application-neutral library and its boundary with downstream
|
||||
consumers.
|
||||
|
||||
## Ideas
|
||||
|
||||
### Public bounded output repair
|
||||
|
||||
After the codebase-audit remediations are complete, Promptkit should make its
|
||||
bounded output-repair capability available through the public engine. A
|
||||
consumer should be able to request a limited number of corrective generation
|
||||
attempts when JSON or JSON Schema output fails content validation, without
|
||||
having to reproduce Promptkit's generation, validation, capacity, and result-
|
||||
accounting orchestration.
|
||||
|
||||
- Repair is validation recovery, not a general provider retry, failover, or
|
||||
backoff policy. Transport failures, cancellation, and operational schema or
|
||||
validation errors must retain their ordinary error behavior.
|
||||
- Repair must stop after the first valid result or the configured attempt
|
||||
bound. Exhausting the bound should preserve the final invalid result and its
|
||||
validation diagnostics rather than inventing success.
|
||||
- Initial generation and every repair attempt must use the same resolved
|
||||
backend, effective execution settings and presence semantics, session,
|
||||
credential boundary, structured-output contract, and backend-capacity
|
||||
policy.
|
||||
- Results should report the number of repair attempts and cumulative usage for
|
||||
every model call made by the run.
|
||||
- Ordinary and prepared execution should expose coherent behavior, including
|
||||
cancellation, frozen prepared state, error identity, and capacity lifetime.
|
||||
|
||||
Select this work only after the accepted audit findings affecting shared
|
||||
execution invariants, validation, orchestration, transport, and repair
|
||||
internals have been remediated.
|
||||
|
||||
## Entry Format
|
||||
|
||||
Use a short heading followed by a concise summary. Add focused bullets when
|
||||
they help preserve important scope boundaries without becoming an
|
||||
implementation plan:
|
||||
|
||||
```markdown
|
||||
### Idea name
|
||||
|
||||
Describe the intended capability, who benefits, and the most important scope
|
||||
boundary or dependency.
|
||||
|
||||
- Optionally record an important behavior or boundary.
|
||||
```
|
||||
840
docs/roadmap/implementation.md
Normal file
840
docs/roadmap/implementation.md
Normal file
@@ -0,0 +1,840 @@
|
||||
# Audit Remediation Implementation Plan
|
||||
|
||||
## Purpose
|
||||
|
||||
This document is the decision-complete implementation plan for the accepted
|
||||
findings in the [codebase audit](audit.md). It is written for a
|
||||
`gpt-5.6-terra` coding agent that will implement one numbered stage per prompt,
|
||||
in order.
|
||||
|
||||
The audit remains the evidence and rationale for each finding. This plan owns
|
||||
implementation order, selected policy decisions, required code and test work,
|
||||
and stage gates. It does not activate the future public output-repair feature
|
||||
described in the [future feature catalog](future.md); it only corrects and
|
||||
protects the retained internal repair machinery on which that later feature
|
||||
may build.
|
||||
|
||||
## Implementation Policies
|
||||
|
||||
Every stage must follow the [development guide](../development.md),
|
||||
[architecture policy](../policy/architecture.md),
|
||||
[testing policy](../policy/testing.md), and
|
||||
[documentation policy](../policy/documentation.md). Before changing a
|
||||
subsystem, read the focused current-state documents identified by the
|
||||
development guide and inspect the exact implementation and tests named by the
|
||||
stage.
|
||||
|
||||
Apply these rules throughout:
|
||||
|
||||
- Implement exactly one stage per agent prompt. Do not combine stages or begin
|
||||
a later stage early.
|
||||
- Inspect the working tree before editing and preserve unrelated changes.
|
||||
- Use the code knowledge graph to locate symbols, callers, and dependency
|
||||
paths; confirm important conclusions against source.
|
||||
- Keep the root package as the public facade and implementation under
|
||||
`internal/`. Do not expose internal representations or add a public package.
|
||||
- Put source-neutral invariants in their assigned internal owner while
|
||||
preserving source-specific normalization, error classification, and public
|
||||
translation at existing boundaries.
|
||||
- Add regression protection at the narrowest stable owner in the same stage
|
||||
as a behavioral fix. Retain only representative integration coverage at
|
||||
higher layers.
|
||||
- Do not add tests to raise coverage percentages. Do not preserve tests that
|
||||
assert an incidental algorithm, private constant, dormant serialization
|
||||
shape, or duplicated lower-layer truth table.
|
||||
- Keep all tests deterministic, offline, race-safe, credential-free, and free
|
||||
of fixed-port or mutable-service assumptions.
|
||||
- Do not add engine-wide caches, generic facade abstractions, scheduler
|
||||
changes, provider retry policy, or new consumer configuration unless a stage
|
||||
explicitly requires it.
|
||||
- Update canonical GoDoc and current-state documents in the same stage as the
|
||||
behavior they describe. Do not describe a later stage as already
|
||||
implemented.
|
||||
- Format changed Go files. Run the stage's focused commands, then at least
|
||||
`go test ./...` and `go vet ./...`. Run focused race tests wherever the
|
||||
stage changes ownership, cancellation, shared state, or lifecycle behavior.
|
||||
- Do not commit, push, tag, or publish unless separately instructed.
|
||||
|
||||
## Decisions Fixed By This Plan
|
||||
|
||||
The implementing agent must not reopen these choices:
|
||||
|
||||
1. **JSON-compatible numbers:** accept every value Go can faithfully encode as
|
||||
a JSON number: every signed and unsigned integer width, finite `float32` and
|
||||
`float64` values, and a `json.Number` whose text is valid JSON-number syntax.
|
||||
Do not impose the current IEEE-754 safe-integer restriction. Reject NaN,
|
||||
infinities, and malformed `json.Number` text. Preserve supported concrete
|
||||
numeric types when copying.
|
||||
2. **JSON-shaped traversal bounds:** allow at most 100 JSON container levels
|
||||
and 100,000 produced JSON value nodes per `Copy` or `CopyMap` operation.
|
||||
Count the root, each map/slice/array container, and every produced child
|
||||
value; map keys are not separate nodes. Pointer and interface indirection
|
||||
do not add JSON depth or an extra node. Repeated appearances of an acyclic
|
||||
shared value count each produced occurrence. Continue rejecting active-path
|
||||
cycles and return deterministic, path-aware validation errors on either
|
||||
bound.
|
||||
3. **Execution timeout bound:** a positive `TimeoutSeconds` must fit in
|
||||
`time.Duration` after multiplication by `time.Second`. Derive the maximum
|
||||
from `math.MaxInt64` and `time.Second`; do not duplicate its numeric literal
|
||||
in tests or documentation.
|
||||
4. **Output contracts:** the only valid formats are `text`, `markdown`, and
|
||||
`json`; the only valid validation modes are `none`, `basic`, `json`, and
|
||||
`json_schema`; repair attempts are non-negative; and `json_schema` requires
|
||||
a nonblank schema path. A non-nil request replacement defaults an empty
|
||||
format to `text` before shared validation. It does not default an empty
|
||||
validation mode.
|
||||
5. **Prompt content paths:** every `content_file` is an exact, relative path
|
||||
resolved from its prompt file and contained by the configured prompt source
|
||||
root. Directory, `fs.FS`, and single-file sources all reject absolute and
|
||||
escaping paths. A single-file source's root is the containing directory of
|
||||
that selected prompt file. Trimming determines only whether a value is
|
||||
blank; it must not change the path opened. OS containment must account for
|
||||
symlinks; containment inside an injected `fs.FS` remains expressed in that
|
||||
filesystem's namespace.
|
||||
6. **Profile IDs:** normalize file-backed IDs with `strings.TrimSpace` once,
|
||||
just as in-memory IDs are normalized. Use the normalized value for
|
||||
selection, duplicate detection, results, and diagnostics. A whitespace-only
|
||||
ID is invalid, and IDs that become equal after normalization are
|
||||
duplicates.
|
||||
7. **Ordinary artifact files:** the built-in `File` reader supports regular
|
||||
files, including symlinks whose targets are regular files. It rejects
|
||||
directories, FIFOs, devices, sockets, and other non-regular targets before
|
||||
consuming them. It remains unrestricted by an application root and does
|
||||
not introduce an application-specific byte limit.
|
||||
8. **Validation cancellation:** do not return early by abandoning goroutines
|
||||
around `fs.FS` or the JSON Schema dependency. Promptkit must check
|
||||
cancellation before, between, and after work it controls; read opened files
|
||||
in context-checked chunks; and let a canceled context win before publishing
|
||||
a result after synchronous decode, compile, or validation calls. Go's
|
||||
`fs.FS` and the current JSON Schema library expose no general mechanism to
|
||||
preempt a blocked `Open`, `Read`, compile, or validation method, so canonical
|
||||
documentation must describe this synchronous limitation rather than claim
|
||||
impossible asynchronous interruption.
|
||||
9. **Successful provider-response limit:** the built-in OpenAI-compatible
|
||||
client accepts at most 16 MiB (`16 << 20` bytes) for the complete successful
|
||||
HTTP response body, including surrounding whitespace. The limit is fixed,
|
||||
internal, and application-neutral. Exactly the limit is allowed; the first
|
||||
byte beyond it fails as `internal/llm.ErrMalformedResponse`. Do not add a
|
||||
public setting. Non-success response parsing remains outside this audit
|
||||
remediation and belongs to the separate structured-generation-error
|
||||
roadmap.
|
||||
10. **Repair machinery:** retain and fix the internal repairer, cumulative
|
||||
usage, and bounded repair state machine. The public engine must continue to
|
||||
install no repairer and remain single-pass. Do not activate public repair
|
||||
in this plan.
|
||||
|
||||
## Stage 1: Centralize Execution-Setting And Session Invariants
|
||||
|
||||
**Findings:** S05-F01, S17-F01, S14-F02. This stage also resolves the
|
||||
source-specific evidence in S02-F02, S08-F01, and S11-F01.
|
||||
|
||||
Add a source-neutral execution-setting validator to `internal/domain`. It must
|
||||
validate temperature, maximum tokens, top-p, and timeout on a domain execution
|
||||
target: temperature and top-p must be finite and within their closed ranges,
|
||||
maximum tokens must be non-negative, and timeout must be non-negative and no
|
||||
greater than the derived duration-safe maximum. Keep optional-pointer presence,
|
||||
profile required fields, normalization, and error wrapping outside this
|
||||
validator.
|
||||
|
||||
Use that owner from:
|
||||
|
||||
- in-memory profile validation in the root package;
|
||||
- OS and `fs.FS` profile validation;
|
||||
- resolved request/target validation in `internal/usecase`; and
|
||||
- the built-in model client as a defensive final boundary.
|
||||
|
||||
Remove the duplicated scalar comparisons from those callers. Preserve
|
||||
`ErrInvalidConfig` for in-memory construction, profile-load identities for file
|
||||
profiles, `ErrInvalidRequest` for runtime overrides, and the LLM package's
|
||||
defensive invalid-request identity. Explicit numeric zero must retain its
|
||||
presence semantics.
|
||||
|
||||
Update `internal/domain.NormalizeSessionID` to reject invalid UTF-8 before
|
||||
trimming or rune counting. Preserve the existing blank and 256-code-point
|
||||
rules. Direct requests must still map failures to `ErrInvalidRequest`, while
|
||||
session-template failures remain renderer failures.
|
||||
|
||||
Add one domain-owned table for every exact setting boundary, finite neighbors,
|
||||
NaN, both infinities, negative values, and the timeout representability edge.
|
||||
Retain small boundary-integration cases for in-memory profiles, both file
|
||||
source forms, request overrides through `Prepare` and `PrepareExecution`, and
|
||||
the model-client defense. Add malformed UTF-8 session cases before, within,
|
||||
and after otherwise valid content.
|
||||
|
||||
Update the architecture policy and internal component overview so
|
||||
`internal/domain` explicitly owns source-neutral invariants for its shared
|
||||
execution values, without claiming ownership of source-specific policy.
|
||||
|
||||
Run focused domain, profile, use-case, root, and LLM tests, including the
|
||||
affected race-enabled request and profile cases, followed by the repository
|
||||
test and vet gates.
|
||||
|
||||
## Stage 2: Centralize Output-Contract Legality
|
||||
|
||||
**Finding:** S17-F02, including the request-boundary symptom S11-F02.
|
||||
|
||||
Add one pure `internal/domain` validator for `OutputContract`. It must enforce
|
||||
the format, validation-mode, non-negative repair-attempt, and JSON-Schema path
|
||||
rules fixed above. It must not load schemas or apply source/request defaults.
|
||||
|
||||
Make prompt-definition normalization call the shared validator after its file-
|
||||
specific normalization. Keep prompt-required fields and contextual
|
||||
`ErrInvalidPromptDefinition` ownership in `internal/promptdef`. Make request
|
||||
resolution default an empty replacement format to `text`, then call the same
|
||||
validator and translate failure to `ErrInvalidRequest` before artifact,
|
||||
rendering, validation, admission, or generation work. Keep schema loading and
|
||||
compilation in `internal/validate`.
|
||||
|
||||
Add a domain table covering every supported and unsupported enum, empty values,
|
||||
negative and non-negative repair counts, and schema-path relationships. Retain
|
||||
small prompt-source and use-case integration tables that prove correct error
|
||||
categories and parity between `Prepare` and `PrepareExecution`; do not repeat
|
||||
the entire domain table at those layers.
|
||||
|
||||
Update the architecture and internal overview language added in Stage 1 to
|
||||
include source-neutral output-contract invariants. Run focused domain,
|
||||
prompt-definition, use-case, and root tests, then repository test and vet
|
||||
gates.
|
||||
|
||||
## Stage 3: Make JSON-Compatible Value Handling Coherent And Bounded
|
||||
|
||||
**Findings:** S05-F02, S05-F03, S05-F04.
|
||||
|
||||
Refactor `internal/jsonvalue` around the numeric and traversal decisions fixed
|
||||
by this plan. Remove the safe-integer restriction and apply one numeric rule to
|
||||
all supported representations. Preserve concrete named and unnamed scalar,
|
||||
map, slice, and array types where the existing contract promises preservation;
|
||||
keep nil versus empty container distinctions and `Copy` versus `CopyMap` empty-
|
||||
key behavior.
|
||||
|
||||
Extend the traversal state to track JSON container depth and produced-node
|
||||
work. Enforce the 100-level and 100,000-node limits before allocation or
|
||||
descent would cross them. Continue using active-path identity for cycle
|
||||
detection; do not use alias memoization that would make distinct JSON paths
|
||||
share mutable output. Errors must identify the structural path and whether the
|
||||
depth or work budget was exceeded.
|
||||
|
||||
Expand the focused package tables by behavior branch: signed and unsigned
|
||||
integer widths, ordinary and named finite floats, `json.Number`, pointers and
|
||||
interfaces, named maps/slices/arrays, nil and empty values, mixed nested trees,
|
||||
arrays, mutation isolation, active cycles, alternating just-below/at/over
|
||||
depth, and shared acyclic subgraphs just below and over the work budget. Tests
|
||||
must derive their edges from package constants or relationships instead of
|
||||
copying unexplained literals.
|
||||
|
||||
Retain only representative public/backend/profile/prepared integration cases
|
||||
that prove error translation and ownership. Update public GoDoc only if it
|
||||
currently states the narrower safe-integer behavior; otherwise the existing
|
||||
finite JSON-compatible-number contract remains canonical. Update the relevant
|
||||
public value GoDoc and format/internal documentation to state that excessively
|
||||
deep or large JSON-shaped values are rejected for safety; keep the exact
|
||||
numeric limits owned by the internal constants rather than duplicating them
|
||||
throughout consumer documentation. Run focused package and caller tests,
|
||||
focused race tests, repository tests, and vet.
|
||||
|
||||
## Stage 4: Consolidate Stable Public JSON And Remove Dormant Internal JSON
|
||||
|
||||
**Findings:** S02-F01, S02-F05, S05-F05.
|
||||
|
||||
Refactor `json.go` so each public value has one ordinary field mapping. Use
|
||||
private aliases or embedded wire representations for ordinary fields and keep
|
||||
only timestamp, millisecond-duration, and intentional omission exceptions
|
||||
explicit. Preserve every existing JSON name and omission rule.
|
||||
|
||||
Before converting `duration_ms`, reject values outside the millisecond range
|
||||
that can be multiplied by `time.Millisecond` without overflow. Derive both
|
||||
edges from `time.Duration` bounds. Return a contextual decode error and do not
|
||||
partially update the receiver on failure.
|
||||
|
||||
Add fully populated `PreparedRun` and `RunResult` contract cases. Verify all
|
||||
ordinary fields, intentional omissions, zero and nonzero timing, complete
|
||||
round trips, the largest safe positive and negative millisecond values, and
|
||||
their first unsafe neighbors.
|
||||
|
||||
Remove unused JSON tags and serialization tests from
|
||||
`internal/domain.PreparedRun` after confirming production never marshals that
|
||||
type. Keep credential absence protected at preparation/clone producers and
|
||||
move any useful cache-control JSON assertion to the public `PreparedRun`
|
||||
contract. Do not retain a parallel internal wire format.
|
||||
|
||||
Run focused domain and root JSON tests, repository tests, and vet.
|
||||
|
||||
## Stage 5: Harden Public Ownership And Diagnostic Contracts
|
||||
|
||||
**Findings:** S01-F01, S02-F03, S02-F04, S13-F01.
|
||||
|
||||
Extend the existing run-request formatting test with distinct input URI,
|
||||
input-body, variable, and API-key sentinels. Require their absence from
|
||||
`String`, `GoString`, `%v`, `%+v`, and `%#v` while retaining positive structural
|
||||
summary assertions.
|
||||
|
||||
Add one focused public-LLM-adapter ownership test. Have the injected client
|
||||
mutate and retain prompt messages, cache-control pointers, nested target extra
|
||||
parameters, and structured-output schema values; prove the domain/prepared
|
||||
source remains unchanged and later details or execution cannot race with those
|
||||
mutations.
|
||||
|
||||
Add one direct all-field mapping test for `OpenAICompatibleProfile`. Populate
|
||||
every field distinctly and compare the complete returned `Profile`. Keep only
|
||||
the existing higher-level cases that prove normal validation and nested-value
|
||||
ownership.
|
||||
|
||||
Make copied `PreparedExecution` values format opaquely by using value-receiver
|
||||
formatting behavior shared by non-nil pointers and values. A nil pointer may
|
||||
use Go's normal `<nil>` formatting, but formatting must never panic or expose
|
||||
internal types, field names, addresses, credentials, or content. Cover original
|
||||
pointers, copied values, zero values, and nil pointers under string, Go-string,
|
||||
and ordinary fmt verbs, and prove formatting does not claim or discard a
|
||||
handle.
|
||||
|
||||
Run focused root tests and the affected prepared/adapter race tests, followed
|
||||
by repository tests and vet.
|
||||
|
||||
## Stage 6: Correct Engine Construction Edges And Immutable Defaults
|
||||
|
||||
**Findings:** S03-F01, S03-F02, S06-F01.
|
||||
|
||||
Change the shared single-file option helper so trimming is used only for the
|
||||
blank-input check. Perform `Stat`, path decomposition, storage, diagnostics,
|
||||
and later access with the exact caller path for prompt, profile, and schema
|
||||
files. Add one compact table covering existing leading- and trailing-whitespace
|
||||
names through all three options.
|
||||
|
||||
Strengthen engine construction tests with three discriminating cases:
|
||||
|
||||
- reverse the argument order of in-memory, ordinary, fallback, and built-in
|
||||
profile categories while retaining fixed category precedence;
|
||||
- collide `Config.ProfileDir` with an ordinary profile option and prove the
|
||||
option replaces the configuration source; and
|
||||
- place a valid same-category replacement after an invalid option and prove
|
||||
construction still fails at the earlier invalid option.
|
||||
|
||||
Convert `internal/defaults.LLMRequestTimeoutDefault` from a variable to a
|
||||
constant without changing its value or adding a setter. Do not add a test that
|
||||
mutates or pins a noncontractual default; existing client deadline behavior is
|
||||
the verification owner.
|
||||
|
||||
Run focused engine construction, default-client construction, and race tests,
|
||||
then repository tests and vet.
|
||||
|
||||
## Stage 7: Contain And Preserve Prompt Content Paths
|
||||
|
||||
**Finding:** S07-F01.
|
||||
|
||||
Refactor prompt content resolution so both repository forms receive an
|
||||
explicit source-root abstraction. Enforce the path decision fixed by this plan
|
||||
before any content read. Use exact parsed path text after a separate blank
|
||||
check. For OS sources, canonicalize the root and resolved target sufficiently
|
||||
to reject symlink escape; for injected `fs.FS`, use its clean relative path
|
||||
namespace. A parent component that remains inside the root is valid. Absolute,
|
||||
escaping, and symlink-escaping targets are invalid.
|
||||
|
||||
Apply the same behavioral table to an OS directory, `WithPromptFS`, and a
|
||||
single-file source: ordinary sibling, nested parent still within root, parent
|
||||
escape, absolute path, symlink escape where supported, and existing names with
|
||||
leading or trailing whitespace. Prove rejected targets cause no outside read
|
||||
and public operations preserve `ErrPromptLoad`.
|
||||
|
||||
Update the framework format reference and internal source document to make the
|
||||
single-file root and absolute-path rule explicit. Run focused prompt-definition
|
||||
and public source tests, including race tests, then repository tests and vet.
|
||||
|
||||
## Stage 8: Correct Prompt Selection, Strictness, Coverage, And Lookup Cost
|
||||
|
||||
**Findings:** S07-F02, S07-F03, S07-F04, S07-F06.
|
||||
|
||||
Correct both existing prompt repository paths before consolidating them in the
|
||||
next stage:
|
||||
|
||||
- Recover selector metadata from YAML `id` and `version`; never use a filename
|
||||
stem as an identity.
|
||||
- Apply normalized ID and requested-version selection before semantic
|
||||
normalization or `content_file` reads.
|
||||
- Associate strict YAML, semantic, and content errors only with a reliably
|
||||
matching selected definition. An unidentifiable malformed file is unrelated
|
||||
to point lookup; a reliably selected malformed file remains authoritative.
|
||||
- Require exactly one YAML document. Comments and trailing whitespace are
|
||||
allowed; a second empty or populated document and malformed trailing YAML
|
||||
are `ErrInvalidYAML`.
|
||||
- Continue scanning the YAML metadata required for duplicate detection, but
|
||||
open content only for selected candidates. A selected content file is opened
|
||||
once; unrelated and different-version bodies are never opened.
|
||||
|
||||
Add paired OS and `fs.FS` regressions for same-stem/different-ID malformed
|
||||
files, same-ID/different-version invalid files, selected malformed definitions,
|
||||
additional YAML documents, duplicates, and counting filesystem behavior.
|
||||
Add a compact normalization table for the previously uncovered missing
|
||||
version, blank input name, blank message role, invalid output format, negative
|
||||
repair attempts, and explicit blank default profile. Output-contract rows
|
||||
should exercise the shared Stage 2 owner rather than recreate its full table.
|
||||
|
||||
Run focused prompt-definition, use-case inspection, and root source tests,
|
||||
focused race tests, repository tests, and vet.
|
||||
|
||||
## Stage 9: Unify Prompt Repository Semantics
|
||||
|
||||
**Finding:** S07-F05.
|
||||
|
||||
After Stage 8 establishes correct behavior in both paths, replace their
|
||||
duplicated discovery-to-selection algorithms with one source-neutral prompt
|
||||
selection and normalization flow. Introduce only the small internal source
|
||||
adapter needed for YAML discovery, bytes, exact content opening, display paths,
|
||||
and root containment. Keep genuine OS and `fs.FS` mechanics at the adapter
|
||||
edge.
|
||||
|
||||
Move exact selection, version filtering, strict one-document decoding,
|
||||
selected-error classification, normalization, duplicate handling, and
|
||||
not-found behavior into the shared flow. Preserve point-in-time source access;
|
||||
do not cache catalogs or definitions across operations.
|
||||
|
||||
Turn the Stage 8 behavior matrix into a shared suite over both adapters and
|
||||
retain source-specific tests only for distinct path and I/O failures. Delete
|
||||
superseded duplicate helpers and tests only after the shared suite protects
|
||||
their meaningful behavior. Use a counting filesystem and a before/after
|
||||
benchmark over small and large prompt catalogs to confirm unrelated content is
|
||||
not read and the refactor adds no second scan; do not enforce wall-clock
|
||||
thresholds.
|
||||
|
||||
Update the internal source document to describe the unified semantic owner.
|
||||
Run focused package, integration, race, repository test, and vet gates.
|
||||
|
||||
## Stage 10: Correct Profile Source Validation And Identity
|
||||
|
||||
**Findings:** S08-F02, S08-F03, S08-F04, S08-F05. The non-finite scalar
|
||||
symptom S08-F01 is already resolved by Stage 1.
|
||||
|
||||
Make file profile normalization pass `extra_params` through
|
||||
`internal/jsonvalue.CopyMap` before publishing a domain profile. Preserve
|
||||
`ErrInvalidProfile` and source path context for empty keys, non-finite values,
|
||||
nested invalid data, or traversal-budget failures. Do not move reserved
|
||||
OpenAI-compatible field policy into the profile package.
|
||||
|
||||
Use YAML metadata ID as the only selector; never infer authority from a
|
||||
filename. Normalize the decoded ID once according to this plan. Require exactly
|
||||
one YAML document in both metadata and strict selected decoding, so trailing
|
||||
raw credentials, unknown fields, empty documents, and malformed YAML cannot be
|
||||
ignored. Reliably selected malformed definitions must stop overlay fallback;
|
||||
unrelated malformed files must not.
|
||||
|
||||
Add shared OS and `fs.FS` tables for invalid/valid extra parameters,
|
||||
same-stem/different-ID malformed files beside a valid profile, fallback
|
||||
behavior, additional documents, leading/trailing/blank IDs, normalized
|
||||
duplicates, and exact inspection/preparation of the normalized ID. Retain only
|
||||
representative public error-translation cases.
|
||||
|
||||
Update the framework format and internal source documents if needed to state
|
||||
ID normalization and one-document behavior. Run focused profile, use-case,
|
||||
root, and race tests, followed by repository tests and vet.
|
||||
|
||||
## Stage 11: Eliminate Duplicate Profile Decoding
|
||||
|
||||
**Finding:** S08-F06.
|
||||
|
||||
Refactor point lookup so each file receives one metadata pass and only
|
||||
canonical ID matches receive strict full decoding and normalization. Reuse
|
||||
bytes already read for metadata; do not decode every unrelated full profile or
|
||||
turn the repository into a cache. Preserve deterministic duplicate detection,
|
||||
strict selected errors, overlay fallthrough only on not-found, and fresh
|
||||
point-in-time reads on every operation.
|
||||
|
||||
Add counting/parser-observation tests where stable behavior can be observed,
|
||||
plus benchmarks for small and large catalogs reporting time and allocations.
|
||||
Exercise valid selection, unrelated malformed files, selected malformed files,
|
||||
duplicates, overlay fallthrough, and repeated lookup. Do not add brittle exact
|
||||
allocation thresholds to ordinary tests.
|
||||
|
||||
Run focused profile and root integration tests, benchmarks for diagnostic
|
||||
comparison, race tests, repository tests, and vet.
|
||||
|
||||
## Stage 12: Correct Artifact Semantics, Cancellation, And Hash Tests
|
||||
|
||||
**Findings:** S09-F01, S09-F02, S09-F05.
|
||||
|
||||
Treat an explicitly typed empty inline reference as a valid zero-byte artifact,
|
||||
including `InlineWithURI`. Keep absence at the input map/reference boundary and
|
||||
compute the same metadata and opaque equality value used for other bodies.
|
||||
|
||||
For ordinary file references, inspect the target before opening and again
|
||||
after opening; reject anything that is not a regular file under the decision
|
||||
above. Replace unbounded `io.ReadAll` with a normal synchronous chunked read
|
||||
that checks `ctx.Err()` before open, before and after each read, and before
|
||||
publishing the artifact. Do not return partial artifacts, add a hidden size
|
||||
limit, or launch an abandoned reader goroutine.
|
||||
|
||||
Add source-parity cases for empty and nonempty inline, inline-with-URI, and file
|
||||
content. Add a platform-appropriate FIFO regression proving the known FIFO is
|
||||
rejected without requiring an external writer, and cancellation cases for a
|
||||
pre-canceled file and a progressing regular-file read. Run them repeatedly and
|
||||
under the race detector.
|
||||
|
||||
Replace exact SHA-256 literals with relational assertions: nonempty and stable
|
||||
for repeat reads, equal for equal inline/file bodies, unequal for changed
|
||||
bodies, and propagated opaquely through preparation. Do not document or test a
|
||||
specific algorithm.
|
||||
|
||||
Update public GoDoc and the internal source document to describe regular-file
|
||||
support and cancellation checkpoints. Run focused package, use-case, root,
|
||||
race, repository test, and vet gates.
|
||||
|
||||
## Stage 13: Make Rendering Cancellation-Aware And Reuse Artifact Text
|
||||
|
||||
**Findings:** S09-F03, S09-F04.
|
||||
|
||||
Check context before session work, before and after each template parse and
|
||||
execution, before and after every message, and before returning the completed
|
||||
prompt. Make the `input` helper return an error when cancellation is observed.
|
||||
Do not run template execution in a detached goroutine.
|
||||
|
||||
Within one `Render` call, lazily convert each named artifact body to text once
|
||||
and memoize that string for the session and all messages. Build the cached
|
||||
string in 64 KiB chunks with one pre-grown `strings.Builder`, checking the
|
||||
context between chunks. Preserve bytes exactly, including invalid UTF-8; do not
|
||||
cache across render calls or mutate artifacts. Unknown and nil inputs retain
|
||||
their current errors, and a canceled conversion must not publish or cache a
|
||||
partial string.
|
||||
|
||||
Add deterministic tests for pre-cancellation, cancellation during the chunked
|
||||
input conversion, and cancellation observed after final-message execution.
|
||||
Require the context identity and no partial prompt while preserving active-
|
||||
context template errors. Add benchmarks for one and repeated references across
|
||||
session and messages; report allocations without hard-coded timing limits.
|
||||
|
||||
Run focused renderer/use-case/root tests, benchmarks, repeated race tests,
|
||||
repository tests, and vet.
|
||||
|
||||
## Stage 14: Preserve Exact JSON Validation Semantics
|
||||
|
||||
**Findings:** S10-F01, S10-F05.
|
||||
|
||||
Create one helper for decoding exactly one JSON value with
|
||||
`json.Decoder.UseNumber` and required EOF after trailing whitespace. Use it for
|
||||
schema documents and JSON Schema instance values so large integers, precise
|
||||
decimals, and exponents retain exact `json.Number` semantics through
|
||||
compilation, prepared metadata, copying, and validation.
|
||||
|
||||
For plain `ValidationJSON`, use a non-materializing complete-document syntax
|
||||
check such as `json.Valid`; do not build a generic tree. Preserve the current
|
||||
result distinction: malformed generated JSON is a completed failed validation,
|
||||
not an operational error, and original output bytes remain unchanged.
|
||||
|
||||
Add focused OS and `fs.FS` cases around `2^53`, `1e400`, precise decimals,
|
||||
ordinary numbers, malformed syntax, and trailing values. Exercise schema
|
||||
`const`, minimum/maximum, and `multipleOf`, and verify the public structured
|
||||
schema retains exact numeric values. Add benchmarks for scalar, object, and
|
||||
large-array JSON validation with allocation reporting but no wall-clock
|
||||
contract.
|
||||
|
||||
Update format/internal validation documentation only where it currently
|
||||
implies float64-limited semantics. Run focused validator/use-case/root tests,
|
||||
benchmarks, race tests, repository tests, and vet.
|
||||
|
||||
## Stage 15: Escape Schema Resources And Compile Once Per Operation
|
||||
|
||||
**Findings:** S10-F02, S10-F03.
|
||||
|
||||
Represent schema compiler resources with `url.URL` rather than string
|
||||
concatenation. Use canonical escaped file URLs for OS paths and a private
|
||||
scheme URL whose path segments are escaped for `fs.FS`. Preserve separators,
|
||||
decode resource paths exactly once at the loader boundary, and continue
|
||||
rejecting remote and escaping references. Legal filenames containing percent,
|
||||
space, `#`, `?`, or Unicode must compile, including contained relative
|
||||
references.
|
||||
|
||||
Unify JSON Schema preparation around `validate.PreparedValidation`:
|
||||
|
||||
- the shared preparation pipeline must create one operation-local compiled
|
||||
plan and derive provider-facing root schema metadata from that plan;
|
||||
- `Prepare` may discard the plan after returning metadata;
|
||||
- `Run` must retain and use the plan for its one operation so the schema graph
|
||||
is not loaded or compiled again during validation; and
|
||||
- `PrepareExecution` must retain the same plan in its frozen payload.
|
||||
|
||||
Remove the document-only `SchemaDocumentLoader` capability if it has no
|
||||
remaining production caller. Do not add an engine-wide or cross-operation
|
||||
schema cache. Keep a clear private preparation carrier in `internal/usecase`
|
||||
if needed so public `domain.PreparedRun` remains free of validator interfaces.
|
||||
|
||||
Replace the existing legal-filename expected failure with valid behavior and
|
||||
retain a genuine compiler-registration failure only if reachable through a
|
||||
valid source. Add public parity tests for invalid keywords, malformed and
|
||||
missing direct/second-level references, unsupported dialects, escapes, remote
|
||||
references, and valid multi-document graphs. A counting source must show each
|
||||
document read once per operation and fresh reads across separate operations.
|
||||
|
||||
Update internal source, validator, and runner documentation for the unified
|
||||
plan lifetime. Run focused validator/use-case/root tests, race tests,
|
||||
repository tests, and vet.
|
||||
|
||||
## Stage 16: Make Validation Cancellation Authoritative
|
||||
|
||||
**Finding:** S10-F04.
|
||||
|
||||
Apply the cancellation decision fixed above. Thread context through schema
|
||||
resource loaders and all Promptkit-controlled read/decode helpers. Read opened
|
||||
schema files in context-checked chunks. Check the context immediately before
|
||||
and after JSON decoding, schema compilation, and schema execution; if
|
||||
cancellation occurred during a synchronous dependency call, return the context
|
||||
error instead of a schema or successful validation result. Do not publish a
|
||||
partial plan or validation result.
|
||||
|
||||
Do not place arbitrary `fs.FS` calls or JSON Schema work in goroutines merely
|
||||
to race them against `ctx.Done()`. Tests must therefore distinguish:
|
||||
|
||||
- prompt cancellation before work;
|
||||
- cancellation between controlled read chunks;
|
||||
- cancellation that becomes authoritative immediately after a synchronous
|
||||
compile or validation call returns; and
|
||||
- the documented limitation that Promptkit cannot preempt a dependency method
|
||||
that never returns.
|
||||
|
||||
Use deterministic controlled readers/contexts rather than sleeps. Assert no
|
||||
goroutine growth or leaked work and preserve operational validation and public
|
||||
context identities. Update validator GoDoc and internal source/runner documents
|
||||
to state the synchronous cancellation boundary accurately.
|
||||
|
||||
Run focused cancellation tests normally, repeatedly, and under the race
|
||||
detector, followed by repository tests and vet.
|
||||
|
||||
## Stage 17: Repair And Protect The Retained Internal Repair Path
|
||||
|
||||
**Findings:** S12-F01, S12-F02, S12-F03.
|
||||
|
||||
Retain the internal repair architecture. Extend `RepairRequest` with
|
||||
`ExecutionTargetPresence` and carry the resolved presence bits unchanged into
|
||||
the default repairer's `GenerateRequest`. Factor one use-case-local constructor
|
||||
for common initial/repair generation fields—effective target, presence,
|
||||
credential, backend identity, session, and structured output—while keeping the
|
||||
initial and repair prompts intentionally separate.
|
||||
|
||||
Accumulate every completed generation response's five token-usage fields into
|
||||
run-level usage. The final content/raw output/artifact continues to come from
|
||||
the last candidate, while usage includes initial generation and every completed
|
||||
repair exactly once. A repair call that returns an error still returns no
|
||||
partial public result under current error semantics.
|
||||
|
||||
Replace the one-attempt-only repair coverage with a compact state-machine
|
||||
table for:
|
||||
|
||||
- initial success with no repair;
|
||||
- ineligible basic validation despite a positive budget;
|
||||
- explicit zero and inherited-zero presence across initial and repair calls;
|
||||
- success before a larger budget is exhausted;
|
||||
- exact exhaustion of a larger budget; and
|
||||
- advancement of attempt number, maximum, prior output, diagnostics, final
|
||||
status, cumulative usage, and collaborator call count.
|
||||
|
||||
Retain the distinct capacity integration proving initial and repair generation
|
||||
use the same backend pool and one whole-run admission lease, but simplify it if
|
||||
the new focused table makes repair-state assertions redundant.
|
||||
|
||||
Update the internal runner and capacity documents for presence fidelity and
|
||||
cumulative usage. Do not alter `NewRunner` to install a repairer, public GoDoc
|
||||
that says the engine is single-pass, or the future public repair roadmap.
|
||||
|
||||
Run focused use-case, prepared, capacity, and race tests, followed by repository
|
||||
tests and vet.
|
||||
|
||||
## Stage 18: Preserve Transport Error Identities And Test Deterministically
|
||||
|
||||
**Findings:** S14-F01, S04-F01, S16-F02. Timeout overflow S14-F02 is already
|
||||
resolved through Stage 1's shared bound.
|
||||
|
||||
Preserve the underlying `http.Client.Do` error in the chain while retaining
|
||||
`internal/llm.ErrRequestFailed` and the public generation category. Do not add
|
||||
headers, request content, endpoints, or provider bodies to error text.
|
||||
Cancellation and deadline identities must survive caller cancellation, caller
|
||||
deadline, generation deadline, and whole-request client timeout.
|
||||
|
||||
Replace the port-9999 test with a controlled round tripper or `httptest`
|
||||
endpoint that records the selected URL and returns a deliberate result. It must
|
||||
make no host-dependent connection and must separately prove empty configured
|
||||
base acceptance and request-endpoint precedence.
|
||||
|
||||
Extend the existing external ordinary-run cancellation test to require both
|
||||
`ErrLLMGenerate` and `context.Canceled`; retain lower-layer tests only for their
|
||||
distinct error owners.
|
||||
|
||||
Update the OpenAI-compatible integration and internal LLM documents for error
|
||||
identity behavior. Run focused LLM and root tests with repetition and the race
|
||||
detector, followed by repository tests and vet.
|
||||
|
||||
## Stage 19: Validate And Compose Effective Provider Endpoints
|
||||
|
||||
**Finding:** S14-F03.
|
||||
|
||||
Add one source-neutral OpenAI-compatible base-endpoint validator in
|
||||
`internal/domain`, alongside the effective execution target invariant. It must
|
||||
trim surrounding configuration whitespace, require absolute HTTP or HTTPS with
|
||||
a host, and reject user information, query, and fragment. Use it from backend
|
||||
registration, in-memory and file profiles, resolved request overrides, and the
|
||||
built-in client defense while preserving each boundary's existing config,
|
||||
profile-load, invalid-request, or LLM error category.
|
||||
|
||||
An empty configured base URL remains valid for a built-in client because a
|
||||
resolved request endpoint may supply it later. Validate only a nonempty
|
||||
configured base at construction, and always validate the final selected
|
||||
endpoint before transport. Backends and endpoint-only profiles retain their
|
||||
existing nonempty endpoint requirements.
|
||||
|
||||
Compose the completion URL through parsed URL operations (prefer
|
||||
`url.JoinPath`) so nested paths and trailing slashes reach exactly one
|
||||
`/chat/completions` suffix. Never append to a raw string.
|
||||
|
||||
Add endpoint tables for HTTP and HTTPS, hosts, nested paths, repeated trailing
|
||||
slashes, queries, fragments, user information, relative paths, missing hosts,
|
||||
unsupported schemes, and request/profile/config error mapping. Require every
|
||||
invalid selected endpoint to fail before transport.
|
||||
|
||||
Update the architecture/internal overview for domain endpoint invariants and
|
||||
the OpenAI-compatible integration and internal LLM documents for URL behavior.
|
||||
Run focused domain/backend/profile/use-case/LLM/root tests, repetition and race
|
||||
tests where ownership crosses packages, followed by repository tests and vet.
|
||||
|
||||
## Stage 20: Bound And Strictly Frame Successful Provider Responses
|
||||
|
||||
**Findings:** S14-F04, S14-F05.
|
||||
|
||||
Enforce the 16 MiB successful-response decision without first copying the
|
||||
entire body. Reject an over-limit `Content-Length` immediately, but also wrap
|
||||
the body in a counting/limited reader that reads at most one byte beyond the
|
||||
limit so chunked or dishonest responses cannot bypass it. Exactly-limit bodies
|
||||
remain valid. Always close the body; do not drain an unbounded oversized
|
||||
stream.
|
||||
|
||||
Decode exactly one response object. After the first decode, require only
|
||||
trailing JSON whitespace and EOF. A second value, non-whitespace suffix,
|
||||
truncated body, malformed JSON, or size overflow returns
|
||||
`ErrMalformedResponse` with no partial response and no provider content in the
|
||||
error.
|
||||
|
||||
Add streaming tests just below, at, and one byte over the limit with and
|
||||
without `Content-Length`, plus a continuing oversized stream. Assert bounded
|
||||
bytes read, timely return, no partial result, and closure. Add trailing
|
||||
whitespace success and trailing garbage/second-value failures. Retain ordinary
|
||||
response mapping and redaction cases.
|
||||
|
||||
Document the fixed successful-response boundary and strict one-document rule
|
||||
in the integration and internal LLM documents. Explicitly leave bounded
|
||||
non-success error-envelope parsing to the structured-generation-error roadmap.
|
||||
Run focused transport tests normally, repeatedly, and under race, followed by
|
||||
repository tests and vet.
|
||||
|
||||
## Stage 21: Consolidate Transport Test Scaffolding
|
||||
|
||||
**Finding:** S14-F06.
|
||||
|
||||
After transport behavior is stable, introduce one small recording-provider
|
||||
fixture for common request capture and successful/error response setup.
|
||||
Organize focused tables around request mapping, authentication, endpoint
|
||||
composition, timeout/error identity, and response framing. Keep specialized
|
||||
round trippers/readers for cancellation, deadlines, byte counts, continuing
|
||||
streams, and body closure.
|
||||
|
||||
Retain every existing durable assertion for method, path, headers,
|
||||
authentication, omission and explicit presence, reserved fields, cache
|
||||
control, structured output, response mapping, usage, error redaction, and
|
||||
timeout precedence. Retain all Stage 18 through 20 regressions. Delete repeated
|
||||
servers, generic-map decoding, and response literals only where the fixture
|
||||
makes the owning behavior clearer; do not replace wire assertions with a broad
|
||||
snapshot.
|
||||
|
||||
Run the LLM suite normally, with shuffle/repetition, and under the race
|
||||
detector. Deliberately inspect the resulting test inventory against the audit's
|
||||
transport matrix before running repository tests and vet.
|
||||
|
||||
## Stage 22: Restore One Canonical Maintainer Validation Workflow
|
||||
|
||||
**Finding:** S16-F01.
|
||||
|
||||
Make `docs/development.md` the canonical owner of the complete local maintainer
|
||||
workflow, as assigned by the documentation policy. Its validation section must
|
||||
include, from the repository root:
|
||||
|
||||
```sh
|
||||
go test ./...
|
||||
go test -race ./...
|
||||
go vet ./...
|
||||
go build ./...
|
||||
go run ./examples/go-library/prepare
|
||||
go run ./examples/go-library/run
|
||||
```
|
||||
|
||||
It must also own the Go formatting, local Markdown link, `git diff --check`,
|
||||
workspace/vendor/replacement, generated-output, credential, and working-tree
|
||||
hygiene checks used before accepting changes.
|
||||
|
||||
Change the testing policy to state the semantic requirements and link to that
|
||||
canonical workflow instead of maintaining a partial competing command list.
|
||||
Change the release procedure to invoke the development-guide validation as a
|
||||
release prerequisite rather than presenting a separately maintained copy;
|
||||
retain release-specific metadata, candidate, tag, and publication commands in
|
||||
the release document.
|
||||
|
||||
Run both examples offline and confirm that a missing or invalid Run example
|
||||
fixture makes its command fail. Validate all changed Markdown links and ensure
|
||||
current-state documentation describes only the implemented workflow.
|
||||
|
||||
## Stage 23: Complete Traceability And Final Validation
|
||||
|
||||
This final stage introduces no new behavior. Review the final tree against the
|
||||
finding-to-stage table below and the evidence in `audit.md`. Confirm every
|
||||
canonical group is implemented and every source-specific symptom retains its
|
||||
required regression and error boundary. Do not mark a finding resolved merely
|
||||
because a nearby refactor landed.
|
||||
|
||||
Run the complete development-guide workflow, including both examples, all
|
||||
formatting and link checks, and repository hygiene. Also run shuffled ordinary
|
||||
tests and repeated race-enabled tests for the changed concurrency,
|
||||
cancellation, prepared, validation, repair, and transport packages. Run the
|
||||
accepted performance benchmarks for prompt lookup, profile lookup, rendering,
|
||||
and JSON validation and record only qualitative before/after conclusions; do
|
||||
not establish release timing promises.
|
||||
|
||||
Inspect canonical GoDoc, formats, integration, architecture, and internal
|
||||
documents against the final implementation. Confirm the public engine still
|
||||
performs no output repair and the future repair entry remains future work.
|
||||
Confirm the structured-generation-error feature was not implemented as part of
|
||||
transport remediation.
|
||||
|
||||
Leave `audit-sequence.md`, `audit.md`, and this plan in place for maintainer
|
||||
review. Retire them only in a separately authorized roadmap-cleanup pass after
|
||||
the remediation has been reviewed and accepted.
|
||||
|
||||
## Finding-To-Stage Traceability
|
||||
|
||||
| Stage | Canonical findings | Historical or source-specific records handled with the canonical owner |
|
||||
| ---: | --- | --- |
|
||||
| 1 | S05-F01, S17-F01, S14-F02 | S02-F02, S08-F01, S11-F01 |
|
||||
| 2 | S17-F02 | S11-F02 |
|
||||
| 3 | S05-F02, S05-F03, S05-F04 | None |
|
||||
| 4 | S02-F01, S02-F05, S05-F05 | None |
|
||||
| 5 | S01-F01, S02-F03, S02-F04, S13-F01 | None |
|
||||
| 6 | S03-F01, S03-F02, S06-F01 | None |
|
||||
| 7 | S07-F01 | None |
|
||||
| 8 | S07-F02, S07-F03, S07-F04, S07-F06 | None |
|
||||
| 9 | S07-F05 | None |
|
||||
| 10 | S08-F02, S08-F03, S08-F04, S08-F05 | S08-F01 was handled in Stage 1 |
|
||||
| 11 | S08-F06 | None |
|
||||
| 12 | S09-F01, S09-F02, S09-F05 | None |
|
||||
| 13 | S09-F03, S09-F04 | None |
|
||||
| 14 | S10-F01, S10-F05 | None |
|
||||
| 15 | S10-F02, S10-F03 | None |
|
||||
| 16 | S10-F04 | None |
|
||||
| 17 | S12-F01, S12-F02, S12-F03 | None |
|
||||
| 18 | S14-F01, S04-F01, S16-F02 | S14-F02 was handled in Stage 1 |
|
||||
| 19 | S14-F03 | None |
|
||||
| 20 | S14-F04, S14-F05 | None |
|
||||
| 21 | S14-F06 | None |
|
||||
| 22 | S16-F01 | None |
|
||||
|
||||
The table maps all 49 canonical remediation groups exactly once. S02-F02 is
|
||||
the one superseded historical finding retained as evidence under S17-F01;
|
||||
S08-F01, S11-F01, and S11-F02 retain their source-specific regression
|
||||
responsibilities without being double-counted as canonical groups.
|
||||
|
||||
## Open Questions
|
||||
|
||||
None. The numeric contract, resource bounds, path and identity rules,
|
||||
validation-cancellation limitation, repair retention, transport response
|
||||
limit, and documentation ownership required to implement these stages are
|
||||
fixed above.
|
||||
71
docs/roadmap/structured-generation-errors.md
Normal file
71
docs/roadmap/structured-generation-errors.md
Normal file
@@ -0,0 +1,71 @@
|
||||
# Structured Generation Errors
|
||||
|
||||
## Purpose
|
||||
|
||||
Promptkit should give downstream applications actionable, machine-readable
|
||||
details when the built-in OpenAI-compatible client receives a non-success HTTP
|
||||
response. Today the client reports only the status code and discards the
|
||||
provider response body. This makes ordinary configuration failures—such as an
|
||||
unsupported strict JSON Schema keyword—unnecessarily difficult to diagnose.
|
||||
|
||||
## Target End State
|
||||
|
||||
Failures from the built-in transport are available through a public typed error
|
||||
that works with `errors.As` while continuing to match `ErrLLMGenerate` through
|
||||
`errors.Is`. The error should expose:
|
||||
|
||||
- the HTTP status code;
|
||||
- a normalized provider error code or type when supplied; and
|
||||
- a bounded provider message extracted from a recognized OpenAI-compatible
|
||||
JSON error envelope.
|
||||
|
||||
The ordinary `Error()` string should remain safe and concise: it should include
|
||||
the status and provider code or type, but not automatically include the
|
||||
provider message. Consumers that deliberately want the provider's diagnostic
|
||||
text can retrieve it from the typed error and apply their own disclosure and
|
||||
logging policy.
|
||||
|
||||
This contract should be available for both ordinary and prepared execution.
|
||||
Errors returned by injected model clients must continue to preserve their own
|
||||
identity and should not be converted into fabricated HTTP details.
|
||||
|
||||
## Safety And Compatibility Boundaries
|
||||
|
||||
- Never expose the raw response body, response headers, endpoint, credentials,
|
||||
request messages, schema document, or generated content through this API.
|
||||
- Read only a small fixed maximum response body, reject malformed or
|
||||
unrecognized envelopes, normalize invalid UTF-8 and control characters, and
|
||||
cap every retained diagnostic field independently.
|
||||
- Treat the extracted provider message as untrusted and potentially sensitive:
|
||||
its GoDoc must tell consumers not to log or display it without applying their
|
||||
own policy.
|
||||
- Preserve the existing generic behavior when a response is empty, non-JSON,
|
||||
oversized, or does not match a recognized error envelope.
|
||||
- Do not assign retryability from an HTTP status. Promptkit supplies facts;
|
||||
downstream applications retain retry and presentation policy.
|
||||
|
||||
## Recommended API Direction
|
||||
|
||||
Prefer one immutable public `GenerationError` value, constructed internally and
|
||||
carrying accessors for HTTP status, provider code or type, and provider message.
|
||||
This keeps the exact representation evolvable while giving consumers an
|
||||
idiomatic `errors.As` contract. Public Go declarations and GoDoc should own the
|
||||
final exact names and semantics.
|
||||
|
||||
The internal OpenAI-compatible client should parse only the conventional
|
||||
top-level `error` envelope and pass normalized details through the use-case and
|
||||
public error-mapping layers. The integration documentation should continue to
|
||||
own wire behavior; the public declarations should own the consumer contract.
|
||||
|
||||
## Acceptance Criteria
|
||||
|
||||
- A downstream consumer can distinguish a provider HTTP 400 from other
|
||||
generation failures and obtain a bounded provider explanation when present.
|
||||
- The typed error still satisfies `errors.Is(err, ErrLLMGenerate)`.
|
||||
- Existing cancellation, capacity, validation, and injected-client error
|
||||
identities remain unchanged.
|
||||
- Tests cover recognized string and numeric provider codes, absent and malformed
|
||||
envelopes, oversized bodies and fields, control characters, and error-chain
|
||||
behavior without making live provider requests.
|
||||
- Current-state GoDoc and the OpenAI-compatible integration and internal-client
|
||||
documents are updated only when the implementation lands.
|
||||
457
engine.go
457
engine.go
@@ -12,7 +12,10 @@ import (
|
||||
"time"
|
||||
|
||||
artifactadapter "gitea.maximumdirect.net/eric/promptkit/internal/artifact"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/backend"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/capacity"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/defaults"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/llm"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/profile"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/profile/builtin"
|
||||
@@ -22,42 +25,107 @@ import (
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/validate"
|
||||
)
|
||||
|
||||
// ErrInvalidConfig indicates invalid public engine configuration.
|
||||
// ErrInvalidConfig identifies invalid engine construction, including missing
|
||||
// required configuration, invalid options or backend registrations, and a nil
|
||||
// Engine receiver.
|
||||
var ErrInvalidConfig = errors.New("invalid engine configuration")
|
||||
|
||||
var (
|
||||
ErrInvalidRequest = errors.New("invalid run request")
|
||||
ErrPromptNotFound = errors.New("prompt not found")
|
||||
ErrProfileNotFound = errors.New("profile not found")
|
||||
ErrProfileRequired = errors.New("profile selection is required")
|
||||
ErrPromptLoad = errors.New("failed to load prompt definition")
|
||||
ErrProfileLoad = errors.New("failed to load execution profile")
|
||||
// ErrInvalidRequest identifies a request whose required values, overrides,
|
||||
// credentials, or effective settings are invalid.
|
||||
ErrInvalidRequest = errors.New("invalid run request")
|
||||
// ErrPromptNotFound identifies a requested prompt ID or version that is not
|
||||
// present in the selected prompt source. It does not also match
|
||||
// ErrPromptLoad.
|
||||
ErrPromptNotFound = errors.New("prompt not found")
|
||||
// ErrProfileNotFound identifies a selected profile ID that is absent from
|
||||
// every configured profile source. It does not also match ErrProfileLoad.
|
||||
ErrProfileNotFound = errors.New("profile not found")
|
||||
// ErrProfileRequired identifies a request for which neither RunRequest.ProfileID
|
||||
// nor the selected prompt's default profile is present. Such an error also
|
||||
// matches ErrInvalidRequest.
|
||||
ErrProfileRequired = errors.New("profile selection is required")
|
||||
// ErrPromptLoad identifies a failure to read, decode, validate, select, or
|
||||
// hash a prompt definition, except for the not-found case represented by
|
||||
// ErrPromptNotFound.
|
||||
ErrPromptLoad = errors.New("failed to load prompt definition")
|
||||
// ErrProfileLoad identifies a failure to read, decode, validate, or select
|
||||
// an execution profile or resolve its backend, except for the profile
|
||||
// not-found case represented by ErrProfileNotFound.
|
||||
ErrProfileLoad = errors.New("failed to load execution profile")
|
||||
// ErrAPIKeyEnvMissing identifies an APIKeyEnv whose environment variable is
|
||||
// unset or empty when no direct RunRequest.APIKey takes precedence. Such an
|
||||
// error also matches ErrInvalidRequest.
|
||||
ErrAPIKeyEnvMissing = errors.New("api_key_env points to an unset environment variable")
|
||||
ErrArtifactLoad = errors.New("failed to load artifact")
|
||||
ErrPromptRender = errors.New("failed to render prompt")
|
||||
ErrLLMGenerate = errors.New("failed to generate output")
|
||||
ErrValidation = errors.New("failed to validate output")
|
||||
// ErrArtifactLoad identifies a failure to resolve an input artifact. Errors
|
||||
// returned by an injected ArtifactReader remain available through errors.Is.
|
||||
ErrArtifactLoad = errors.New("failed to load artifact")
|
||||
// ErrPromptRender identifies a failure to render prompt messages or the
|
||||
// session ID from the resolved inputs and variables.
|
||||
ErrPromptRender = errors.New("failed to render prompt")
|
||||
// ErrCapacityExceeded identifies a Run or RunPrepared rejected because the
|
||||
// selected backend already admitted ConcurrencyLimit + QueueCapacity calls.
|
||||
// A [CapacityError] reports the selected backend ID. It is not an invalid
|
||||
// request, an LLM or provider rate-limit response, or ErrLLMGenerate.
|
||||
ErrCapacityExceeded = errors.New("backend capacity exceeded")
|
||||
// ErrLLMGenerate identifies a model-client failure or a nil successful
|
||||
// response. Errors returned by an injected LLMClient remain available
|
||||
// through errors.Is.
|
||||
ErrLLMGenerate = errors.New("failed to generate output")
|
||||
// ErrValidation identifies an operational failure to load or compile a
|
||||
// schema or validate output. A completed validation whose Status is
|
||||
// ValidationFailed is returned in RunResult without this error.
|
||||
ErrValidation = errors.New("failed to validate output")
|
||||
)
|
||||
|
||||
// Engine prepares and runs Promptkit prompt requests.
|
||||
// Engine inspects prompts and profiles and prepares and runs Promptkit prompt
|
||||
// requests.
|
||||
//
|
||||
// An Engine is safe for concurrent calls to [Engine.InspectPrompt],
|
||||
// [Engine.InspectProfile], [Engine.Prepare], [Engine.PrepareExecution],
|
||||
// [Engine.Run], and [Engine.RunPrepared]. Each Engine owns independent
|
||||
// backend-capacity pools that coordinate Run and RunPrepared admission and
|
||||
// model generation. Injected collaborators may still be invoked concurrently
|
||||
// across different backend pools or for unlimited backends.
|
||||
type Engine struct {
|
||||
runner *usecase.Runner
|
||||
}
|
||||
|
||||
// Config configures a public Promptkit engine.
|
||||
// Config selects the directory-backed sources and built-in model-client
|
||||
// transport used by [NewEngine]. Config has no stable JSON representation.
|
||||
type Config struct {
|
||||
PromptDir string
|
||||
// PromptDir is the directory searched recursively for prompt definitions.
|
||||
// It is required unless a WithPromptFS or WithPromptFile option supplies the
|
||||
// prompt source.
|
||||
PromptDir string
|
||||
// ProfileDir is an optional ordinary configured source whose profiles take
|
||||
// precedence over application fallback and embedded built-in profiles. An
|
||||
// empty value selects the lower-precedence sources unless a profile-source
|
||||
// option supplies the ordinary source.
|
||||
ProfileDir string
|
||||
SchemaDir string
|
||||
// SchemaDir is the root for JSON Schema files. An empty value uses the
|
||||
// current directory. WithSchemaFS or WithSchemaFile replaces this source.
|
||||
SchemaDir string
|
||||
// Timeout is the transport-wide safety cap for the built-in LLM client
|
||||
// when HTTPClient is absent or has a non-positive timeout.
|
||||
// when HTTPClient is absent or has a non-positive timeout. A zero or negative
|
||||
// value selects the 10-minute default.
|
||||
Timeout time.Duration
|
||||
// HTTPClient is cloned for the built-in LLM client. Its positive Timeout
|
||||
// takes precedence over Config.Timeout as the transport-wide safety cap.
|
||||
// takes precedence over Timeout. A zero or negative client Timeout inherits
|
||||
// Timeout or the 10-minute default. The supplied client is not mutated. This
|
||||
// field is ignored when WithLLMClient is used.
|
||||
HTTPClient *http.Client
|
||||
}
|
||||
|
||||
// Option customizes engine construction.
|
||||
//
|
||||
// NewEngine applies options in argument order and ignores nil options. Within
|
||||
// each prompt-source, ordinary-profile-source, fallback-profile-source,
|
||||
// in-memory-profile, schema-source, model-client, and artifact-reader
|
||||
// category, the last non-nil valid option replaces earlier options in that
|
||||
// category. WithBackend is the additive exception: unique registrations
|
||||
// accumulate, and a repeated backend ID is an error rather than a replacement.
|
||||
// An invalid option fails construction even if a later option would replace it.
|
||||
type Option interface {
|
||||
apply(*engineOptions) error
|
||||
}
|
||||
@@ -69,20 +137,30 @@ func (f optionFunc) apply(options *engineOptions) error {
|
||||
}
|
||||
|
||||
type engineOptions struct {
|
||||
llmClient llm.Client
|
||||
artifactReader artifactadapter.Reader
|
||||
promptDefs promptdef.Repository
|
||||
profiles profile.Repository
|
||||
memoryProfiles profile.Repository
|
||||
validator validate.Validator
|
||||
promptSource bool
|
||||
profileSource bool
|
||||
memorySource bool
|
||||
validatorSource bool
|
||||
artifactSource bool
|
||||
llmClient llm.Client
|
||||
artifactReader artifactadapter.Reader
|
||||
promptDefs promptdef.Repository
|
||||
profiles profile.Repository
|
||||
fallbackProfiles profile.Repository
|
||||
memoryProfiles profile.Repository
|
||||
backends []domain.Backend
|
||||
validator validate.Validator
|
||||
promptSource bool
|
||||
profileSource bool
|
||||
fallbackProfileSource bool
|
||||
memorySource bool
|
||||
validatorSource bool
|
||||
artifactSource bool
|
||||
}
|
||||
|
||||
// WithLLMClient injects a custom LLM client for execution.
|
||||
// WithLLMClient replaces the built-in model client used by [Engine.Run] and
|
||||
// [Engine.RunPrepared].
|
||||
//
|
||||
// A nil client makes NewEngine fail with ErrInvalidConfig. The Engine schedules
|
||||
// Generate calls according to the selected backend's capacity policy, but the
|
||||
// client may still be called concurrently across different backend pools or for
|
||||
// unlimited backends. The client is not used by [Engine.Prepare] or
|
||||
// [Engine.PrepareExecution].
|
||||
func WithLLMClient(client LLMClient) Option {
|
||||
return optionFunc(func(options *engineOptions) error {
|
||||
if client == nil {
|
||||
@@ -93,7 +171,11 @@ func WithLLMClient(client LLMClient) Option {
|
||||
})
|
||||
}
|
||||
|
||||
// WithArtifactReader injects a reader for every input artifact reference.
|
||||
// WithArtifactReader replaces the default reader for every input artifact
|
||||
// reference, regardless of its ArtifactRef.Type.
|
||||
//
|
||||
// A nil reader makes NewEngine fail with ErrInvalidConfig. The reader may be
|
||||
// called concurrently.
|
||||
func WithArtifactReader(reader ArtifactReader) Option {
|
||||
return optionFunc(func(options *engineOptions) error {
|
||||
if reader == nil {
|
||||
@@ -109,6 +191,9 @@ func WithArtifactReader(reader ArtifactReader) Option {
|
||||
//
|
||||
// The source uses the same strict prompt YAML rules as configured prompt
|
||||
// directories, and prompt content_file paths resolve within this source.
|
||||
// fsys must be non-nil and root must be non-empty; otherwise NewEngine fails
|
||||
// with ErrInvalidConfig. This option replaces Config.PromptDir and earlier
|
||||
// prompt-source options.
|
||||
func WithPromptFS(fsys fs.FS, root string) Option {
|
||||
return optionFunc(func(options *engineOptions) error {
|
||||
if fsys == nil {
|
||||
@@ -125,7 +210,9 @@ func WithPromptFS(fsys fs.FS, root string) Option {
|
||||
|
||||
// WithPromptFile loads prompt definitions from the single prompt file at path.
|
||||
//
|
||||
// Relative prompt content_file paths resolve from the file's directory.
|
||||
// Relative prompt content_file paths resolve from the file's directory. path
|
||||
// must name an existing non-directory file when NewEngine applies the option.
|
||||
// This option replaces Config.PromptDir and earlier prompt-source options.
|
||||
func WithPromptFile(path string) Option {
|
||||
return optionFunc(func(options *engineOptions) error {
|
||||
fsys, root, err := fileSource(path)
|
||||
@@ -140,8 +227,12 @@ func WithPromptFile(path string) Option {
|
||||
|
||||
// WithProfileFS loads execution profiles from fsys under root.
|
||||
//
|
||||
// Profiles from this source overlay built-in profiles. Profile YAML must use
|
||||
// api_key_env for environment-based credentials; raw API keys are rejected.
|
||||
// Profiles from this ordinary configured source take precedence over
|
||||
// application fallback and built-in profiles. Profile YAML must use api_key_env
|
||||
// for environment-based credentials; raw API keys are rejected. fsys must be
|
||||
// non-nil and root must be non-empty; otherwise NewEngine fails with
|
||||
// ErrInvalidConfig. This option replaces Config.ProfileDir and earlier file or
|
||||
// FS profile-source options, but remains below WithProfiles in precedence.
|
||||
func WithProfileFS(fsys fs.FS, root string) Option {
|
||||
return optionFunc(func(options *engineOptions) error {
|
||||
if fsys == nil {
|
||||
@@ -158,8 +249,12 @@ func WithProfileFS(fsys fs.FS, root string) Option {
|
||||
|
||||
// WithProfileFile loads execution profiles from the single profile file at path.
|
||||
//
|
||||
// The profile overlays built-in profiles. Profile YAML must use api_key_env for
|
||||
// environment-based credentials; raw API keys are rejected.
|
||||
// The profile takes precedence over application fallback and built-in profiles.
|
||||
// Profile YAML must use api_key_env for environment-based credentials; raw API
|
||||
// keys are rejected. path must name an existing non-directory file when
|
||||
// NewEngine applies the option. This option replaces Config.ProfileDir and
|
||||
// earlier file or FS profile-source options, but remains below WithProfiles in
|
||||
// precedence.
|
||||
func WithProfileFile(path string) Option {
|
||||
return optionFunc(func(options *engineOptions) error {
|
||||
fsys, root, err := fileSource(path)
|
||||
@@ -172,8 +267,46 @@ func WithProfileFile(path string) Option {
|
||||
})
|
||||
}
|
||||
|
||||
// WithFallbackProfileFS supplies application-owned fallback profile
|
||||
// definitions from fsys under root.
|
||||
//
|
||||
// Profile lookup checks, in order, profiles supplied by WithProfiles; the
|
||||
// ordinary configured source selected by WithProfileFile, WithProfileFS, or
|
||||
// Config.ProfileDir; this fallback source; and Promptkit's embedded built-in
|
||||
// profiles. Each source supplies a complete profile definition; profile fields
|
||||
// are not merged between sources. Only an absent profile ID proceeds to the
|
||||
// next source. A matching read, parse, duplicate, validation, or credential
|
||||
// format failure stops resolution.
|
||||
//
|
||||
// Files use the ordinary strict profile YAML and api_key_env credential rules.
|
||||
// Loading and validation are lazy: NewEngine validates this option's arguments
|
||||
// but does not read profile files. fsys must be non-nil and root must be
|
||||
// nonblank; otherwise NewEngine returns an error matching ErrInvalidConfig.
|
||||
// Repeating this option replaces the earlier valid fallback source.
|
||||
//
|
||||
// This option controls profile-definition lookup, not provider or generation
|
||||
// failover.
|
||||
func WithFallbackProfileFS(fsys fs.FS, root string) Option {
|
||||
return optionFunc(func(options *engineOptions) error {
|
||||
if fsys == nil {
|
||||
return ErrInvalidConfig
|
||||
}
|
||||
if strings.TrimSpace(root) == "" {
|
||||
return ErrInvalidConfig
|
||||
}
|
||||
options.fallbackProfiles = profile.NewFSRepository(fsys, root)
|
||||
options.fallbackProfileSource = true
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// WithProfiles configures in-memory profiles that take precedence over
|
||||
// configured profile files and built-in profiles.
|
||||
// ordinary configured, application fallback, and built-in profiles.
|
||||
//
|
||||
// NewEngine validates and copies every profile. IDs must be unique within one
|
||||
// call. An invalid profile, duplicate ID, or unsupported ExtraParams value
|
||||
// makes construction fail with ErrInvalidConfig. Repeating WithProfiles
|
||||
// replaces the complete earlier in-memory set rather than merging it.
|
||||
func WithProfiles(profiles ...Profile) Option {
|
||||
return optionFunc(func(options *engineOptions) error {
|
||||
repo, err := newMemoryProfileRepository(profiles)
|
||||
@@ -189,7 +322,9 @@ func WithProfiles(profiles ...Profile) Option {
|
||||
// WithSchemaFS loads JSON Schema documents from fsys under root.
|
||||
//
|
||||
// Prompt schema_path values resolve within this source when schema validation
|
||||
// or structured output is requested.
|
||||
// or structured output is requested. fsys must be non-nil and root must be
|
||||
// non-empty; otherwise NewEngine fails with ErrInvalidConfig. This option
|
||||
// replaces Config.SchemaDir and earlier schema-source options.
|
||||
func WithSchemaFS(fsys fs.FS, root string) Option {
|
||||
return optionFunc(func(options *engineOptions) error {
|
||||
if fsys == nil {
|
||||
@@ -206,7 +341,9 @@ func WithSchemaFS(fsys fs.FS, root string) Option {
|
||||
|
||||
// WithSchemaFile loads JSON Schema documents from the single schema file at path.
|
||||
//
|
||||
// Prompt schema_path values refer to the file's base name.
|
||||
// Prompt schema_path values refer to the file's base name. path must name an
|
||||
// existing non-directory file when NewEngine applies the option. This option
|
||||
// replaces Config.SchemaDir and earlier schema-source options.
|
||||
func WithSchemaFile(path string) Option {
|
||||
return optionFunc(func(options *engineOptions) error {
|
||||
fsys, root, err := fileSource(path)
|
||||
@@ -220,6 +357,17 @@ func WithSchemaFile(path string) Option {
|
||||
}
|
||||
|
||||
// NewEngine constructs an Engine from configuration and options.
|
||||
//
|
||||
// Options are applied in order according to [Option]. PromptDir is required
|
||||
// unless a prompt-source option is present. Construction validates option
|
||||
// arguments, in-memory profiles, and backend registrations but defers reading
|
||||
// and validating prompt, file-backed profile, and schema contents until Prepare
|
||||
// or Run needs them.
|
||||
//
|
||||
// NewEngine returns an error matching ErrInvalidConfig for invalid
|
||||
// configuration, options, or backend-capacity policies. Each constructed
|
||||
// Engine has independent backend-capacity pools. Construction does not perform
|
||||
// model requests or require credentials.
|
||||
func NewEngine(cfg Config, opts ...Option) (*Engine, error) {
|
||||
var options engineOptions
|
||||
for _, opt := range opts {
|
||||
@@ -239,12 +387,16 @@ func NewEngine(cfg Config, opts ...Option) (*Engine, error) {
|
||||
promptDefs = promptdef.NewFilesystemRepository(cfg.PromptDir)
|
||||
}
|
||||
|
||||
profiles := builtin.NewRepositoryWithDirectory(cfg.ProfileDir)
|
||||
if options.profileSource {
|
||||
profiles = builtin.NewRepositoryWithPrimary(options.profiles)
|
||||
profiles := newProfileRepository(cfg.ProfileDir, options)
|
||||
|
||||
backendRegistry, err := backend.NewRegistry(options.backends)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: failed to construct backend registry: %v", ErrInvalidConfig, err)
|
||||
}
|
||||
if options.memorySource {
|
||||
profiles = profile.NewOverlayRepository(options.memoryProfiles, profiles)
|
||||
|
||||
capacityManager, err := capacity.NewManager(backendRegistry.CapacityPolicies())
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: failed to construct backend capacity manager: %v", ErrInvalidConfig, err)
|
||||
}
|
||||
|
||||
validator := options.validator
|
||||
@@ -267,6 +419,7 @@ func NewEngine(cfg Config, opts ...Option) (*Engine, error) {
|
||||
return nil, fmt.Errorf("%w: %v", ErrInvalidConfig, err)
|
||||
}
|
||||
}
|
||||
llmClient = capacity.NewClient(capacityManager, llmClient)
|
||||
|
||||
artifacts := options.artifactReader
|
||||
if !options.artifactSource {
|
||||
@@ -277,14 +430,36 @@ func NewEngine(cfg Config, opts ...Option) (*Engine, error) {
|
||||
runner: usecase.NewRunner(
|
||||
promptDefs,
|
||||
profiles,
|
||||
backendRegistry,
|
||||
artifacts,
|
||||
prompt.NewGoRenderer(),
|
||||
llmClient,
|
||||
validator,
|
||||
capacityManager,
|
||||
),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func newProfileRepository(profileDir string, options engineOptions) profile.Repository {
|
||||
repository := builtin.NewRepository()
|
||||
|
||||
if options.fallbackProfileSource {
|
||||
repository = profile.NewOverlayRepository(options.fallbackProfiles, repository)
|
||||
}
|
||||
|
||||
if options.profileSource {
|
||||
repository = profile.NewOverlayRepository(options.profiles, repository)
|
||||
} else if strings.TrimSpace(profileDir) != "" {
|
||||
repository = profile.NewOverlayRepository(profile.NewFilesystemRepository(profileDir), repository)
|
||||
}
|
||||
|
||||
if options.memorySource {
|
||||
repository = profile.NewOverlayRepository(options.memoryProfiles, repository)
|
||||
}
|
||||
|
||||
return repository
|
||||
}
|
||||
|
||||
func fileSource(name string) (fs.FS, string, error) {
|
||||
cleanName := strings.TrimSpace(name)
|
||||
if cleanName == "" {
|
||||
@@ -305,7 +480,110 @@ func fileSource(name string) (fs.FS, string, error) {
|
||||
return os.DirFS(dir), filepath.ToSlash(base), nil
|
||||
}
|
||||
|
||||
// Prepare resolves a prompt request without calling an LLM.
|
||||
// InspectPrompt resolves one explicit prompt definition without selecting a
|
||||
// profile or starting execution work.
|
||||
//
|
||||
// InspectPrompt requires a nonblank promptID. It passes nonblank promptID and
|
||||
// promptVersion values unchanged to the engine's ordinary, case-sensitive
|
||||
// prompt selection. An empty version succeeds only when that source has one
|
||||
// selected ID; a nonempty version selects one exact ID/version pair. The
|
||||
// configured prompt source is used without merging, fallback, or enumeration.
|
||||
//
|
||||
// A successful result proves that the selected definition and any referenced
|
||||
// message content files were structurally loaded. Inputs are returned in
|
||||
// definition order. DefaultProfileID is declared metadata only and is not
|
||||
// resolved. OutputContract is the normalized declared contract, with a JSON
|
||||
// Schema path when declared but without loading or compiling that schema.
|
||||
// PromptHash is the same opaque equality value as PreparedRun.PromptHash for
|
||||
// the selected definition and observed source state; its spelling, length,
|
||||
// encoding, algorithm, and security properties are not contracts.
|
||||
//
|
||||
// This method does not return prompt bodies, templates, source paths, schemas,
|
||||
// rendered messages, or execution settings. It does not resolve a profile or
|
||||
// credential, read artifacts or schemas, render, validate, admit capacity,
|
||||
// contact a provider, or generate model output. The returned PromptInspection
|
||||
// and its input slice are caller-owned. Filesystem-backed inspection is a
|
||||
// point-in-time lookup and does not freeze a definition for later execution.
|
||||
//
|
||||
// A nil Engine returns an error matching ErrInvalidConfig. A blank prompt ID
|
||||
// matches ErrInvalidRequest. An absent exact ID or version matches
|
||||
// ErrPromptNotFound and not ErrPromptLoad. Malformed, unreadable, duplicate,
|
||||
// ambiguous, referenced-content, or hashing failures match ErrPromptLoad.
|
||||
// Cancellation during lookup matches ErrPromptLoad while preserving the
|
||||
// context error. InspectPrompt returns no partial result on error.
|
||||
func (e *Engine) InspectPrompt(
|
||||
ctx context.Context,
|
||||
promptID string,
|
||||
promptVersion string,
|
||||
) (*PromptInspection, error) {
|
||||
if e == nil || e.runner == nil {
|
||||
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
|
||||
}
|
||||
|
||||
inspection, err := e.runner.InspectPrompt(ctx, promptID, promptVersion)
|
||||
if err != nil {
|
||||
return nil, mapPublicError(err)
|
||||
}
|
||||
return fromDomainPromptInspection(inspection), nil
|
||||
}
|
||||
|
||||
// InspectProfile resolves one explicit profile without selecting a prompt or
|
||||
// starting execution work.
|
||||
//
|
||||
// InspectProfile trims surrounding whitespace from profileID and looks up the
|
||||
// resulting nonblank ID exactly and case-sensitively through the engine's
|
||||
// in-memory, ordinary configured-source, application fallback, and built-in
|
||||
// profile precedence. It applies the framework timeout baseline, selected
|
||||
// backend, and then selected profile to EffectiveModelParams without a request
|
||||
// override. BackendID is empty for an endpoint-only profile.
|
||||
//
|
||||
// APIKeyEnv in the returned target is an environment-variable name, never its
|
||||
// value. APIKeyRequired instead reports a direct credential requirement and is
|
||||
// mutually exclusive with a nonblank APIKeyEnv. InspectProfile neither derives
|
||||
// an ID from a prompt default_profile nor checks credential availability, so an
|
||||
// absent or blank named environment variable is not an error.
|
||||
//
|
||||
// The returned ProfileInspection and all nested mutable values are
|
||||
// caller-owned. Filesystem-backed inspection is a point-in-time lookup and
|
||||
// does not freeze the profile for a later execution. This method does not load
|
||||
// a prompt, render, read artifacts or schemas, admit backend capacity, contact
|
||||
// a provider, or generate model output.
|
||||
//
|
||||
// A nil Engine returns an error matching ErrInvalidConfig. A blank profile ID
|
||||
// matches ErrInvalidRequest. An absent exact ID matches ErrProfileNotFound and
|
||||
// not ErrProfileLoad. Malformed or unreadable profile data, an unknown backend,
|
||||
// or an invalid resolved target matches ErrProfileLoad. Cancellation during
|
||||
// profile loading matches ErrProfileLoad while preserving the context error.
|
||||
// InspectProfile returns no partial result on error.
|
||||
func (e *Engine) InspectProfile(ctx context.Context, profileID string) (*ProfileInspection, error) {
|
||||
if e == nil || e.runner == nil {
|
||||
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
|
||||
}
|
||||
|
||||
inspection, err := e.runner.InspectProfile(ctx, profileID)
|
||||
if err != nil {
|
||||
return nil, mapPublicError(err)
|
||||
}
|
||||
return fromDomainProfileInspection(inspection), nil
|
||||
}
|
||||
|
||||
// Prepare resolves and renders a prompt request without calling an LLM.
|
||||
//
|
||||
// Prepare selects the prompt and profile, resolves any selected backend and
|
||||
// effective execution settings, resolves the output contract, loads and hashes
|
||||
// inputs, loads structured-output schema metadata when required, and renders
|
||||
// the session ID and messages. The returned PreparedRun is owned by the caller
|
||||
// and never contains a resolved API-key value, model output, or validation
|
||||
// result.
|
||||
//
|
||||
// A nil Engine returns an error matching ErrInvalidConfig. Request and
|
||||
// preparation failures may match ErrInvalidRequest, ErrPromptNotFound,
|
||||
// ErrPromptLoad, ErrProfileNotFound, ErrProfileLoad, ErrProfileRequired,
|
||||
// ErrAPIKeyEnvMissing, ErrArtifactLoad, ErrPromptRender, or ErrValidation as
|
||||
// applicable. Cancellation is passed to the active collaborator and is
|
||||
// reported in the applicable operation category; no general errors.Is
|
||||
// relationship to ctx.Err is promised. Prepare returns no partial result on
|
||||
// error.
|
||||
func (e *Engine) Prepare(ctx context.Context, req RunRequest) (*PreparedRun, error) {
|
||||
if e == nil || e.runner == nil {
|
||||
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
|
||||
@@ -323,7 +601,57 @@ func (e *Engine) Prepare(ctx context.Context, req RunRequest) (*PreparedRun, err
|
||||
return fromDomainPreparedRun(prepared), nil
|
||||
}
|
||||
|
||||
// Run executes a prompt request and returns the generated artifact and metadata.
|
||||
// PrepareExecution completely prepares a prompt request without calling the
|
||||
// configured LLMClient or reserving backend admission capacity.
|
||||
//
|
||||
// The returned opaque handle is bound to this Engine and permits one
|
||||
// [Engine.RunPrepared] invocation. Preparation freezes the selected sources,
|
||||
// rendered messages, effective settings, inputs, provider structured-output
|
||||
// metadata, and validation resources needed by that invocation. The handle
|
||||
// retains a direct RunRequest.APIKey only in private execution state;
|
||||
// [PreparedExecution.Details] is credential-redacted.
|
||||
//
|
||||
// The context governs preparation only. Cancellation after this method
|
||||
// returns does not invalidate the handle or propagate to RunPrepared.
|
||||
// PrepareExecution returns the same error categories as [Engine.Prepare] and
|
||||
// returns no handle on error. A nil Engine returns an error matching
|
||||
// ErrInvalidConfig.
|
||||
func (e *Engine) PrepareExecution(ctx context.Context, req RunRequest) (*PreparedExecution, error) {
|
||||
if e == nil || e.runner == nil {
|
||||
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
|
||||
}
|
||||
|
||||
domainReq, err := toDomainRunRequest(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: %v", ErrInvalidRequest, err)
|
||||
}
|
||||
|
||||
prepared, err := e.runner.PrepareExecution(ctx, domainReq)
|
||||
if err != nil {
|
||||
return nil, mapPublicError(err)
|
||||
}
|
||||
return &PreparedExecution{internal: prepared}, nil
|
||||
}
|
||||
|
||||
// Run prepares a request, invokes the configured LLMClient, and validates the
|
||||
// generated output.
|
||||
//
|
||||
// A content-validation failure is a successful run whose
|
||||
// RunResult.Validation has Status ValidationFailed. An inability to perform
|
||||
// validation returns an error matching ErrValidation and no partial result.
|
||||
// The public Engine does not perform output repair, so validation is
|
||||
// single-pass even when OutputContract.RepairAttempts is positive.
|
||||
//
|
||||
// Run can return every error category documented by [Engine.Prepare], plus
|
||||
// ErrCapacityExceeded and ErrLLMGenerate. An engine admission rejection is
|
||||
// discoverable as [CapacityError] and still matches ErrCapacityExceeded. It
|
||||
// occurs before artifacts, schemas, rendering, or model generation because the
|
||||
// selected backend's admission capacity is full; it does not match
|
||||
// ErrInvalidRequest or ErrLLMGenerate. Errors from injected clients remain
|
||||
// available through errors.Is. Cancellation while waiting for model-generation
|
||||
// capacity matches both ErrLLMGenerate and the context error. Cancellation
|
||||
// otherwise follows the active collaborator's documented behavior. A nil
|
||||
// Engine returns ErrInvalidConfig. Run returns no partial result on error.
|
||||
func (e *Engine) Run(ctx context.Context, req RunRequest) (*RunResult, error) {
|
||||
if e == nil || e.runner == nil {
|
||||
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
|
||||
@@ -340,3 +668,42 @@ func (e *Engine) Run(ctx context.Context, req RunRequest) (*RunResult, error) {
|
||||
}
|
||||
return fromDomainRunResult(result), nil
|
||||
}
|
||||
|
||||
// RunPrepared atomically claims and executes a handle created by
|
||||
// [Engine.PrepareExecution].
|
||||
//
|
||||
// A valid owning-Engine invocation consumes the handle's one attempt before
|
||||
// credential revalidation, backend admission, generation, or validation.
|
||||
// Cancellation, capacity rejection, generation failure, operational
|
||||
// validation failure, and success all leave the handle unusable. A nil,
|
||||
// zero-value, foreign-Engine, discarded, claimed, or used handle returns an
|
||||
// error matching ErrInvalidRequest; a nil Engine returns ErrInvalidConfig and
|
||||
// does not claim the handle.
|
||||
//
|
||||
// The supplied context governs this execution attempt independently of the
|
||||
// preparation context. It covers credential revalidation, admission,
|
||||
// generation, validation, and any internal repair. Result timing begins after
|
||||
// the claim and excludes preparation and consumer-held delay.
|
||||
//
|
||||
// RunPrepared can return ErrInvalidRequest, ErrAPIKeyEnvMissing,
|
||||
// ErrCapacityExceeded, ErrLLMGenerate, or ErrValidation as applicable while
|
||||
// preserving documented collaborator and context identities. An engine
|
||||
// admission rejection is discoverable as [CapacityError] and still matches
|
||||
// ErrCapacityExceeded. A completed content-validation rejection is returned
|
||||
// in RunResult, not as an operational error. An operational error returns no
|
||||
// partial RunResult.
|
||||
func (e *Engine) RunPrepared(ctx context.Context, prepared *PreparedExecution) (*RunResult, error) {
|
||||
if e == nil || e.runner == nil {
|
||||
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
|
||||
}
|
||||
|
||||
var internal *usecase.PreparedExecution
|
||||
if prepared != nil {
|
||||
internal = prepared.internal
|
||||
}
|
||||
result, err := e.runner.RunPrepared(ctx, internal)
|
||||
if err != nil {
|
||||
return nil, mapPublicError(err)
|
||||
}
|
||||
return fromDomainRunResult(result), nil
|
||||
}
|
||||
|
||||
249
engine_test.go
249
engine_test.go
@@ -281,6 +281,9 @@ func TestEngineExecutionSettingPrecedence(t *testing.T) {
|
||||
intPointer := func(value int) *int {
|
||||
return &value
|
||||
}
|
||||
stringPointer := func(value string) *string {
|
||||
return &value
|
||||
}
|
||||
|
||||
defaultsProfile := executionProfileFixture{
|
||||
id: "settings-defaults",
|
||||
@@ -357,14 +360,14 @@ func TestEngineExecutionSettingPrecedence(t *testing.T) {
|
||||
wantPresence promptkit.ExecutionTargetPresence
|
||||
}{
|
||||
{
|
||||
name: "framework defaults fill zero-valued profile settings",
|
||||
name: "unspecified provider controls retain framework timeout",
|
||||
profile: defaultsProfile,
|
||||
want: promptkit.ExecutionTarget{
|
||||
Endpoint: defaultsProfile.endpoint,
|
||||
Model: defaultsProfile.model,
|
||||
Temperature: 0,
|
||||
MaxTokens: 0,
|
||||
TopP: 1,
|
||||
TopP: 0,
|
||||
TimeoutSeconds: 600,
|
||||
ServiceTier: defaultsProfile.serviceTier,
|
||||
ReasoningEffort: defaultsProfile.reasoningEffort,
|
||||
@@ -388,7 +391,7 @@ func TestEngineExecutionSettingPrecedence(t *testing.T) {
|
||||
TopP: floatPointer(requestTarget.TopP),
|
||||
TimeoutSeconds: intPointer(requestTarget.TimeoutSeconds),
|
||||
ServiceTier: requestTarget.ServiceTier,
|
||||
ReasoningEffort: requestTarget.ReasoningEffort,
|
||||
ReasoningEffort: stringPointer(requestTarget.ReasoningEffort),
|
||||
APIKeyEnv: requestTarget.APIKeyEnv,
|
||||
ExtraParams: requestTarget.ExtraParams,
|
||||
},
|
||||
@@ -452,6 +455,43 @@ func TestEngineExecutionSettingPrecedence(t *testing.T) {
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
t.Run("blank request reasoning clears profile setting", func(t *testing.T) {
|
||||
profile := executionProfileFixture{
|
||||
id: "settings-reasoning-clear",
|
||||
endpoint: "http://profile-reasoning.test/v1",
|
||||
model: "profile-reasoning-model",
|
||||
reasoningEffort: "medium",
|
||||
}
|
||||
profileDir := t.TempDir()
|
||||
writeExecutionProfileFixture(t, profileDir, profile)
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{
|
||||
PromptDir: frameworkPromptDir,
|
||||
ProfileDir: profileDir,
|
||||
SchemaDir: frameworkSchemaDir,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("construct engine: %v", err)
|
||||
}
|
||||
|
||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||
PromptID: frameworkMarkdownSummaryPromptID,
|
||||
ProfileID: profile.id,
|
||||
Inputs: map[string]promptkit.ArtifactRef{
|
||||
"transcript": promptkit.Inline("Nia labels the archive."),
|
||||
"glossary": promptkit.Inline("archive: A catalogued collection."),
|
||||
},
|
||||
Execution: &promptkit.ExecutionTargetOverride{
|
||||
ReasoningEffort: stringPointer(" \t "),
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("prepare engine: %v", err)
|
||||
}
|
||||
if prepared.EffectiveModelParams.ReasoningEffort != "" {
|
||||
t.Fatalf("expected blank request reasoning to clear profile value, got %q", prepared.EffectiveModelParams.ReasoningEffort)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestRunSucceedsWithInjectedLLMClient(t *testing.T) {
|
||||
@@ -571,19 +611,26 @@ func TestEngineRunWithDirectorySourcesAndFileInputs(t *testing.T) {
|
||||
|
||||
func TestRunPassesPreparedRequestToInjectedLLMClient(t *testing.T) {
|
||||
const directKey = "direct-injected-key"
|
||||
const directSession = "assembled-session"
|
||||
fake := &fakeLLMClient{
|
||||
response: &promptkit.GenerateResponse{Content: `{"events":[{"title":"Archive labelled"}]}`},
|
||||
}
|
||||
engine := newContractEngineWithOptions(t, frameworkSchemaDir, promptkit.WithLLMClient(fake))
|
||||
|
||||
_, err := engine.Run(context.Background(), promptkit.RunRequest{
|
||||
PromptID: frameworkStructuredEventsPromptID,
|
||||
APIKey: directKey,
|
||||
runRequest := promptkit.RunRequest{
|
||||
PromptID: frameworkStructuredEventsPromptID,
|
||||
SessionID: " " + directSession + " ",
|
||||
APIKey: directKey,
|
||||
Inputs: map[string]promptkit.ArtifactRef{
|
||||
"transcript": promptkit.Inline("Rin opens the gate."),
|
||||
"glossary": promptkit.Inline("gate: A guarded passage."),
|
||||
},
|
||||
})
|
||||
}
|
||||
prepared, err := engine.Prepare(context.Background(), runRequest)
|
||||
if err != nil {
|
||||
t.Fatalf("expected prepare to succeed, got %v", err)
|
||||
}
|
||||
result, err := engine.Run(context.Background(), runRequest)
|
||||
if err != nil {
|
||||
t.Fatalf("expected run to succeed, got %v", err)
|
||||
}
|
||||
@@ -594,6 +641,16 @@ func TestRunPassesPreparedRequestToInjectedLLMClient(t *testing.T) {
|
||||
if len(req.Prompt.Messages) != 2 || !strings.Contains(req.Prompt.Messages[1].Content, "Rin opens the gate.") {
|
||||
t.Fatalf("expected rendered prompt in generate request, got %+v", req.Prompt)
|
||||
}
|
||||
if prepared.SessionID != directSession ||
|
||||
req.Prompt.SessionID != directSession ||
|
||||
result.SessionID != directSession {
|
||||
t.Fatalf(
|
||||
"direct session did not propagate consistently: prepared=%q generated=%q result=%q",
|
||||
prepared.SessionID,
|
||||
req.Prompt.SessionID,
|
||||
result.SessionID,
|
||||
)
|
||||
}
|
||||
if req.StructuredOutput == nil || req.StructuredOutput.Type != promptkit.StructuredOutputJSONSchema || req.StructuredOutput.JSONSchema == nil {
|
||||
t.Fatalf("expected structured output handoff, got %+v", req.StructuredOutput)
|
||||
}
|
||||
@@ -755,6 +812,92 @@ func TestRunUsesDirectAPIKeyWithDefaultLLMClient(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunUsesResolvedBackendWithBuiltInLLMClient(t *testing.T) {
|
||||
const (
|
||||
backendID = "local-test"
|
||||
envName = "PROMPTKIT_BACKEND_TRANSPORT_KEY"
|
||||
apiKey = "synthetic-backend-key"
|
||||
)
|
||||
t.Setenv(envName, apiKey)
|
||||
|
||||
var (
|
||||
gotAuth string
|
||||
gotBody map[string]any
|
||||
)
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotAuth = r.Header.Get("Authorization")
|
||||
if r.URL.Path != "/v1/chat/completions" {
|
||||
t.Errorf("unexpected path: %s", r.URL.Path)
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&gotBody); err != nil {
|
||||
t.Errorf("decode request body: %v", err)
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{
|
||||
"choices": [{"message": {"role": "assistant", "content": "# Summary\n\nDone."}}],
|
||||
"usage": {"prompt_tokens": 3, "completion_tokens": 4, "total_tokens": 7}
|
||||
}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{
|
||||
PromptDir: frameworkPromptDir,
|
||||
SchemaDir: frameworkSchemaDir,
|
||||
},
|
||||
promptkit.WithBackend(promptkit.Backend{
|
||||
ID: backendID,
|
||||
Endpoint: server.URL + "/v1",
|
||||
APIKeyEnv: envName,
|
||||
ExtraParams: map[string]any{
|
||||
"provider": "synthetic",
|
||||
},
|
||||
}),
|
||||
promptkit.WithProfiles(promptkit.Profile{
|
||||
ID: "backend-transport",
|
||||
BackendID: backendID,
|
||||
Model: "test-model",
|
||||
}),
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("construct engine: %v", err)
|
||||
}
|
||||
|
||||
result, err := engine.Run(context.Background(), promptkit.RunRequest{
|
||||
PromptID: frameworkMarkdownSummaryPromptID,
|
||||
ProfileID: "backend-transport",
|
||||
Inputs: map[string]promptkit.ArtifactRef{
|
||||
"transcript": promptkit.Inline("Rin opens the gate."),
|
||||
"glossary": promptkit.Inline("gate: A guarded passage."),
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("run with resolved backend: %v", err)
|
||||
}
|
||||
if gotAuth != "Bearer "+apiKey {
|
||||
t.Fatalf("unexpected Authorization header: %q", gotAuth)
|
||||
}
|
||||
if gotBody["model"] != "test-model" || gotBody["provider"] != "synthetic" {
|
||||
t.Fatalf("backend defaults did not reach provider payload: %#v", gotBody)
|
||||
}
|
||||
for _, field := range []string{"backend_id", "api_key_env"} {
|
||||
if _, ok := gotBody[field]; ok {
|
||||
t.Fatalf("internal metadata field %q was serialized to provider payload: %#v", field, gotBody)
|
||||
}
|
||||
}
|
||||
bodyJSON, err := json.Marshal(gotBody)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal captured provider payload: %v", err)
|
||||
}
|
||||
if strings.Contains(string(bodyJSON), apiKey) {
|
||||
t.Fatalf("credential value was serialized to provider payload: %s", bodyJSON)
|
||||
}
|
||||
if result.SelectedBackendID != backendID ||
|
||||
result.EffectiveModelParams.Endpoint != server.URL+"/v1" ||
|
||||
result.EffectiveModelParams.APIKeyEnv != envName {
|
||||
t.Fatalf("unexpected resolved backend metadata: %+v", result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrepareDirectAPIKeyBypassesMissingEnvWithoutLeakingOrHashing(t *testing.T) {
|
||||
const missingEnv = "PROMPTKIT_PUBLIC_PREPARE_MISSING"
|
||||
const firstKey = "first-direct-key"
|
||||
@@ -1020,6 +1163,7 @@ func TestPublicErrorsSupportErrorsIs(t *testing.T) {
|
||||
client promptkit.LLMClient
|
||||
schemaDir string
|
||||
want error
|
||||
notWant error
|
||||
}{
|
||||
{
|
||||
name: "invalid request",
|
||||
@@ -1028,10 +1172,11 @@ func TestPublicErrorsSupportErrorsIs(t *testing.T) {
|
||||
want: promptkit.ErrInvalidRequest,
|
||||
},
|
||||
{
|
||||
name: "prompt not found",
|
||||
req: promptkit.RunRequest{PromptID: "missing.prompt"},
|
||||
client: &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}},
|
||||
want: promptkit.ErrPromptNotFound,
|
||||
name: "prompt not found",
|
||||
req: promptkit.RunRequest{PromptID: "missing.prompt"},
|
||||
client: &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}},
|
||||
want: promptkit.ErrPromptNotFound,
|
||||
notWant: promptkit.ErrPromptLoad,
|
||||
},
|
||||
{
|
||||
name: "profile not found",
|
||||
@@ -1042,8 +1187,9 @@ func TestPublicErrorsSupportErrorsIs(t *testing.T) {
|
||||
"transcript": promptkit.Inline("Rin opens the gate."),
|
||||
},
|
||||
},
|
||||
client: &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}},
|
||||
want: promptkit.ErrProfileNotFound,
|
||||
client: &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}},
|
||||
want: promptkit.ErrProfileNotFound,
|
||||
notWant: promptkit.ErrProfileLoad,
|
||||
},
|
||||
{
|
||||
name: "artifact load",
|
||||
@@ -1114,6 +1260,9 @@ func TestPublicErrorsSupportErrorsIs(t *testing.T) {
|
||||
if !errors.Is(err, tc.want) {
|
||||
t.Fatalf("expected errors.Is(%v), got %v", tc.want, err)
|
||||
}
|
||||
if tc.notWant != nil && errors.Is(err, tc.notWant) {
|
||||
t.Fatalf("did not expect errors.Is(%v), got %v", tc.notWant, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1265,6 +1414,18 @@ func TestPrepareUsesBuiltInProfileWithoutProfileDir(t *testing.T) {
|
||||
if prepared.SelectedProfileID != "mistral-small-3" {
|
||||
t.Fatalf("unexpected selected profile: %q", prepared.SelectedProfileID)
|
||||
}
|
||||
if prepared.SelectedBackendID != promptkit.BackendOpenRouter {
|
||||
t.Fatalf("unexpected selected backend: %q", prepared.SelectedBackendID)
|
||||
}
|
||||
if prepared.EffectiveModelParams.BackendID != promptkit.BackendOpenRouter {
|
||||
t.Fatalf("unexpected effective backend: %q", prepared.EffectiveModelParams.BackendID)
|
||||
}
|
||||
if prepared.EffectiveModelParams.Endpoint != "https://openrouter.ai/api/v1" {
|
||||
t.Fatalf("unexpected built-in endpoint: %q", prepared.EffectiveModelParams.Endpoint)
|
||||
}
|
||||
if prepared.EffectiveModelParams.APIKeyEnv != "OPENROUTER_API_KEY" {
|
||||
t.Fatalf("unexpected built-in api key environment name: %q", prepared.EffectiveModelParams.APIKeyEnv)
|
||||
}
|
||||
if prepared.EffectiveModelParams.Model != "mistralai/mistral-small-3.2-24b-instruct" {
|
||||
t.Fatalf("unexpected built-in model: %q", prepared.EffectiveModelParams.Model)
|
||||
}
|
||||
@@ -1635,6 +1796,7 @@ func TestOpenAICompatibleProfileRunsThroughNormalProfilePath(t *testing.T) {
|
||||
fake := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}}
|
||||
prof := promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
|
||||
ID: "template-profile",
|
||||
BackendID: " openrouter ",
|
||||
Endpoint: "http://template/v1",
|
||||
Model: "template-model",
|
||||
APIKeyRequired: true,
|
||||
@@ -1666,7 +1828,9 @@ func TestOpenAICompatibleProfileRunsThroughNormalProfilePath(t *testing.T) {
|
||||
if len(fake.requests) != 1 {
|
||||
t.Fatalf("expected one request, got %d", len(fake.requests))
|
||||
}
|
||||
if fake.requests[0].Target.Model != "template-model" || fake.requests[0].APIKey != "template-key" {
|
||||
if fake.requests[0].Target.BackendID != promptkit.BackendOpenRouter ||
|
||||
fake.requests[0].Target.Model != "template-model" ||
|
||||
fake.requests[0].APIKey != "template-key" {
|
||||
t.Fatalf("unexpected generated request: %+v", fake.requests[0])
|
||||
}
|
||||
if !reflect.DeepEqual(fake.requests[0].Target.ExtraParams, map[string]any{"provider": "template"}) {
|
||||
@@ -1699,6 +1863,16 @@ func TestEngineRunLayersTransportAndGenerationTimeouts(t *testing.T) {
|
||||
configTimeout: 5 * time.Second,
|
||||
wantRemainingAtRequest: 5 * time.Second,
|
||||
},
|
||||
{
|
||||
name: "zero configuration uses ten minute transport default",
|
||||
wantRemainingAtRequest: 10 * time.Minute,
|
||||
},
|
||||
{
|
||||
name: "negative configuration uses ten minute transport default",
|
||||
configTimeout: -2 * time.Second,
|
||||
suppliedClientTimeout: -3 * time.Second,
|
||||
wantRemainingAtRequest: 10 * time.Minute,
|
||||
},
|
||||
{
|
||||
name: "profile deadline is shorter than transport cap",
|
||||
suppliedClientTimeout: 6 * time.Second,
|
||||
@@ -2127,6 +2301,8 @@ func TestSourceOptionsRejectInvalidInputs(t *testing.T) {
|
||||
{name: "profile fs nil", opt: promptkit.WithProfileFS(nil, "profiles")},
|
||||
{name: "profile fs empty root", opt: promptkit.WithProfileFS(fstest.MapFS{}, "")},
|
||||
{name: "profile file empty", opt: promptkit.WithProfileFile("")},
|
||||
{name: "fallback profile fs nil", opt: promptkit.WithFallbackProfileFS(nil, "profiles")},
|
||||
{name: "fallback profile fs empty root", opt: promptkit.WithFallbackProfileFS(fstest.MapFS{}, "")},
|
||||
{name: "schema fs nil", opt: promptkit.WithSchemaFS(nil, "schemas")},
|
||||
{name: "schema fs empty root", opt: promptkit.WithSchemaFS(fstest.MapFS{}, "")},
|
||||
{name: "schema file empty", opt: promptkit.WithSchemaFile("")},
|
||||
@@ -2240,6 +2416,11 @@ func TestExtraParamsTypedNestedValuesAreCopiedAcrossPublicBoundary(t *testing.T)
|
||||
}
|
||||
|
||||
func TestRunRejectsInvalidExtraParams(t *testing.T) {
|
||||
cyclicMap := map[string]any{}
|
||||
cyclicMap["self"] = cyclicMap
|
||||
cyclicSlice := []any{nil}
|
||||
cyclicSlice[0] = cyclicSlice
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
extraParams map[string]any
|
||||
@@ -2251,42 +2432,10 @@ func TestRunRejectsInvalidExtraParams(t *testing.T) {
|
||||
{name: "nan", extraParams: map[string]any{"bad": math.NaN()}},
|
||||
{name: "positive infinity", extraParams: map[string]any{"bad": math.Inf(1)}},
|
||||
{name: "negative infinity", extraParams: map[string]any{"bad": math.Inf(-1)}},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
fake := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}}
|
||||
engine := newContractEngineWithOptions(t, frameworkSchemaDir, promptkit.WithLLMClient(fake))
|
||||
|
||||
_, err := engine.Run(context.Background(), promptkit.RunRequest{
|
||||
PromptID: frameworkMarkdownSummaryPromptID,
|
||||
Inputs: map[string]promptkit.ArtifactRef{
|
||||
"transcript": promptkit.Inline("Rin opens the gate."),
|
||||
"glossary": promptkit.Inline("gate: A guarded passage."),
|
||||
},
|
||||
Execution: &promptkit.ExecutionTargetOverride{ExtraParams: tc.extraParams},
|
||||
})
|
||||
if !errors.Is(err, promptkit.ErrInvalidRequest) {
|
||||
t.Fatalf("expected ErrInvalidRequest, got %v", err)
|
||||
}
|
||||
if len(fake.requests) != 0 {
|
||||
t.Fatalf("expected invalid request to fail before LLM call, got %d requests", len(fake.requests))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunRejectsCyclicExtraParams(t *testing.T) {
|
||||
cyclicMap := map[string]any{}
|
||||
cyclicMap["self"] = cyclicMap
|
||||
cyclicSlice := []any{nil}
|
||||
cyclicSlice[0] = cyclicSlice
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
extraParams map[string]any
|
||||
}{
|
||||
{name: "map", extraParams: cyclicMap},
|
||||
{name: "slice", extraParams: map[string]any{"cycle": cyclicSlice}},
|
||||
{name: "cyclic map", extraParams: cyclicMap},
|
||||
{name: "cyclic slice", extraParams: map[string]any{"cycle": cyclicSlice}},
|
||||
{name: "malformed JSON number", extraParams: map[string]any{"value": json.Number("+1")}},
|
||||
{name: "empty nested key", extraParams: map[string]any{"nested": map[string]any{"": true}}},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
|
||||
@@ -3,7 +3,9 @@ package promptkit
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/capacity"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/profile"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/promptdef"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/usecase"
|
||||
@@ -13,6 +15,11 @@ func mapPublicError(err error) error {
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
var internalCapacityError *usecase.CapacityError
|
||||
if errors.As(err, &internalCapacityError) && internalCapacityError != nil &&
|
||||
strings.TrimSpace(internalCapacityError.BackendID) != "" {
|
||||
return &CapacityError{BackendID: internalCapacityError.BackendID}
|
||||
}
|
||||
publicErr := publicErrorFor(err)
|
||||
if publicErr == nil {
|
||||
return err
|
||||
@@ -38,6 +45,8 @@ func publicErrorFor(err error) error {
|
||||
return ErrProfileLoad
|
||||
case errors.Is(err, usecase.ErrAPIKeyEnvMissing):
|
||||
return errors.Join(ErrInvalidRequest, ErrAPIKeyEnvMissing)
|
||||
case errors.Is(err, capacity.ErrCapacityExceeded):
|
||||
return ErrCapacityExceeded
|
||||
case errors.Is(err, usecase.ErrArtifactLoad):
|
||||
return ErrArtifactLoad
|
||||
case errors.Is(err, usecase.ErrPromptRender):
|
||||
|
||||
50
errors_internal_test.go
Normal file
50
errors_internal_test.go
Normal file
@@ -0,0 +1,50 @@
|
||||
package promptkit
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/usecase"
|
||||
)
|
||||
|
||||
func TestMapPublicErrorPreservesGenerationCancellation(t *testing.T) {
|
||||
internalErr := fmt.Errorf("%w: %w", usecase.ErrLLMGenerate, context.Canceled)
|
||||
|
||||
err := mapPublicError(internalErr)
|
||||
if !errors.Is(err, ErrLLMGenerate) {
|
||||
t.Fatalf("mapped error=%v, want ErrLLMGenerate", err)
|
||||
}
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("mapped error=%v, want context.Canceled", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMapPublicErrorTranslatesCapacityError(t *testing.T) {
|
||||
internalErr := &usecase.CapacityError{BackendID: "limited"}
|
||||
|
||||
err := mapPublicError(internalErr)
|
||||
var publicErr *CapacityError
|
||||
if !errors.As(err, &publicErr) || publicErr == nil {
|
||||
t.Fatalf("mapped error=%v, want public CapacityError", err)
|
||||
}
|
||||
if publicErr.BackendID != "limited" {
|
||||
t.Fatalf("mapped backend ID=%q, want limited", publicErr.BackendID)
|
||||
}
|
||||
if !errors.Is(err, ErrCapacityExceeded) {
|
||||
t.Fatalf("mapped error=%v, want ErrCapacityExceeded", err)
|
||||
}
|
||||
if errors.Is(err, ErrInvalidRequest) || errors.Is(err, ErrLLMGenerate) {
|
||||
t.Fatalf("mapped capacity error has an unrelated category: %v", err)
|
||||
}
|
||||
var leakedInternalErr *usecase.CapacityError
|
||||
if errors.As(err, &leakedInternalErr) {
|
||||
t.Fatalf("mapped error exposes internal CapacityError: %v", err)
|
||||
}
|
||||
|
||||
internalErr.BackendID = "changed"
|
||||
if publicErr.BackendID != "limited" {
|
||||
t.Fatalf("mapped backend ID changed with source error: %q", publicErr.BackendID)
|
||||
}
|
||||
}
|
||||
81
examples/go-library/run/main.go
Normal file
81
examples/go-library/run/main.go
Normal file
@@ -0,0 +1,81 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
type deterministicClient struct{}
|
||||
|
||||
func (deterministicClient) Generate(
|
||||
ctx context.Context,
|
||||
_ promptkit.GenerateRequest,
|
||||
) (*promptkit.GenerateResponse, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &promptkit.GenerateResponse{
|
||||
Content: "Ada finished the migration review.",
|
||||
Usage: promptkit.TokenUsage{
|
||||
PromptTokens: 12,
|
||||
CompletionTokens: 6,
|
||||
TotalTokens: 18,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
type summary struct {
|
||||
Output string `json:"output"`
|
||||
ValidationStatus promptkit.ValidationStatus `json:"validation_status"`
|
||||
IsValid bool `json:"is_valid"`
|
||||
Model string `json:"model"`
|
||||
TotalTokens int `json:"total_tokens"`
|
||||
}
|
||||
|
||||
func main() {
|
||||
engine, err := promptkit.NewEngine(
|
||||
promptkit.Config{},
|
||||
promptkit.WithPromptFile("examples/go-library/run/prompt.yaml"),
|
||||
promptkit.WithProfiles(promptkit.Profile{
|
||||
ID: "offline-example",
|
||||
Endpoint: "https://example.invalid/v1",
|
||||
Model: "offline-model",
|
||||
}),
|
||||
promptkit.WithLLMClient(deterministicClient{}),
|
||||
)
|
||||
if err != nil {
|
||||
exit(err)
|
||||
}
|
||||
|
||||
result, err := engine.Run(context.Background(), promptkit.RunRequest{
|
||||
PromptID: "example.run",
|
||||
Inputs: map[string]promptkit.ArtifactRef{
|
||||
"note": promptkit.Inline("Ada finished the migration review."),
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
exit(err)
|
||||
}
|
||||
|
||||
encoder := json.NewEncoder(os.Stdout)
|
||||
encoder.SetIndent("", " ")
|
||||
if err := encoder.Encode(summary{
|
||||
Output: result.RawOutput,
|
||||
ValidationStatus: result.Validation.Status,
|
||||
IsValid: result.Validation.IsValid,
|
||||
Model: result.ModelName,
|
||||
TotalTokens: result.Usage.TotalTokens,
|
||||
}); err != nil {
|
||||
exit(err)
|
||||
}
|
||||
}
|
||||
|
||||
func exit(err error) {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
16
examples/go-library/run/prompt.yaml
Normal file
16
examples/go-library/run/prompt.yaml
Normal file
@@ -0,0 +1,16 @@
|
||||
id: example.run
|
||||
version: "1.0.0"
|
||||
default_profile: offline-example
|
||||
description: Run a prompt with a deterministic injected model client.
|
||||
inputs:
|
||||
- name: note
|
||||
required: true
|
||||
content_type: text/plain
|
||||
messages:
|
||||
- role: system
|
||||
content: Summarize the note in one sentence.
|
||||
- role: user
|
||||
content: '{{input "note"}}'
|
||||
output:
|
||||
format: text
|
||||
validation_mode: basic
|
||||
@@ -2,19 +2,23 @@ package promptkit
|
||||
|
||||
import "fmt"
|
||||
|
||||
// String returns a concise request summary without exposing direct API keys.
|
||||
// String returns a concise request summary without exposing the direct API key
|
||||
// or input and variable contents. Reflection-based formatting does not carry
|
||||
// this guarantee.
|
||||
func (r RunRequest) String() string {
|
||||
return r.redactedString()
|
||||
}
|
||||
|
||||
// GoString returns a concise request summary without exposing direct API keys.
|
||||
// GoString returns a concise request summary without exposing the direct API
|
||||
// key or input and variable contents. Reflection-based formatting does not
|
||||
// carry this guarantee.
|
||||
func (r RunRequest) GoString() string {
|
||||
return r.redactedString()
|
||||
}
|
||||
|
||||
func (r RunRequest) redactedString() string {
|
||||
return fmt.Sprintf(
|
||||
"promptkit.RunRequest{PromptID:%q PromptVersion:%q ProfileID:%q APIKeySet:%t Inputs:%d Vars:%d ExecutionSet:%t ValidationSet:%t Metadata:%d}",
|
||||
"promptkit.RunRequest{PromptID:%q PromptVersion:%q ProfileID:%q APIKeySet:%t Inputs:%d Vars:%d ExecutionSet:%t ValidationSet:%t}",
|
||||
r.PromptID,
|
||||
r.PromptVersion,
|
||||
r.ProfileID,
|
||||
@@ -23,18 +27,19 @@ func (r RunRequest) redactedString() string {
|
||||
len(r.Vars),
|
||||
r.Execution != nil,
|
||||
r.Validation != nil,
|
||||
len(r.Metadata),
|
||||
)
|
||||
}
|
||||
|
||||
// String returns a concise request summary without exposing direct API keys or
|
||||
// rendered prompt content.
|
||||
// rendered prompt content. Reflection-based formatting does not carry this
|
||||
// guarantee.
|
||||
func (r GenerateRequest) String() string {
|
||||
return r.redactedString()
|
||||
}
|
||||
|
||||
// GoString returns a concise request summary without exposing direct API keys or
|
||||
// rendered prompt content.
|
||||
// rendered prompt content. Reflection-based formatting does not carry this
|
||||
// guarantee.
|
||||
func (r GenerateRequest) GoString() string {
|
||||
return r.redactedString()
|
||||
}
|
||||
|
||||
212
internal/backend/registry.go
Normal file
212
internal/backend/registry.go
Normal file
@@ -0,0 +1,212 @@
|
||||
// Package backend owns validated, immutable OpenAI-compatible backend
|
||||
// definitions.
|
||||
package backend
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/jsonvalue"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/llm"
|
||||
)
|
||||
|
||||
const (
|
||||
// OpenRouterID is the reserved ID of Promptkit's built-in OpenRouter
|
||||
// backend.
|
||||
OpenRouterID = "openrouter"
|
||||
|
||||
openRouterEndpoint = "https://openrouter.ai/api/v1"
|
||||
openRouterAPIKeyEnv = "OPENROUTER_API_KEY"
|
||||
|
||||
openRouterConcurrencyLimit = 16
|
||||
defaultQueueCapacity = 1024
|
||||
)
|
||||
|
||||
// ErrBackendNotFound identifies a registry lookup for an unknown backend ID.
|
||||
var ErrBackendNotFound = errors.New("backend not found")
|
||||
|
||||
var environmentVariableName = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`)
|
||||
|
||||
// Registry is an immutable collection of validated backend definitions.
|
||||
type Registry struct {
|
||||
backends map[string]domain.Backend
|
||||
}
|
||||
|
||||
// NewRegistry constructs a registry containing the built-in OpenRouter
|
||||
// definition followed by the supplied additions. Every ID must be unique.
|
||||
func NewRegistry(additions []domain.Backend) (*Registry, error) {
|
||||
registry := &Registry{
|
||||
backends: make(map[string]domain.Backend, len(additions)+1),
|
||||
}
|
||||
|
||||
definitions := make([]domain.Backend, 0, len(additions)+1)
|
||||
definitions = append(definitions, domain.Backend{
|
||||
ID: OpenRouterID,
|
||||
Endpoint: openRouterEndpoint,
|
||||
APIKeyEnv: openRouterAPIKeyEnv,
|
||||
ConcurrencyLimit: openRouterConcurrencyLimit,
|
||||
})
|
||||
definitions = append(definitions, additions...)
|
||||
|
||||
for _, definition := range definitions {
|
||||
definition.ID = strings.TrimSpace(definition.ID)
|
||||
if definition.ID == "" {
|
||||
return nil, errors.New("backend ID must not be blank")
|
||||
}
|
||||
if _, exists := registry.backends[definition.ID]; exists {
|
||||
return nil, fmt.Errorf("backend ID %q is already registered", definition.ID)
|
||||
}
|
||||
|
||||
normalized, err := normalizeBackend(definition)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
registry.backends[normalized.ID] = normalized
|
||||
}
|
||||
|
||||
return registry, nil
|
||||
}
|
||||
|
||||
// GetBackend returns a defensive copy of the backend registered with id.
|
||||
func (r *Registry) GetBackend(id string) (domain.Backend, error) {
|
||||
if r == nil {
|
||||
return domain.Backend{}, fmt.Errorf("%w: %q", ErrBackendNotFound, id)
|
||||
}
|
||||
definition, ok := r.backends[id]
|
||||
if !ok {
|
||||
return domain.Backend{}, fmt.Errorf("%w: %q", ErrBackendNotFound, id)
|
||||
}
|
||||
extraParams, err := jsonvalue.CopyMap(definition.ExtraParams)
|
||||
if err != nil {
|
||||
return domain.Backend{}, fmt.Errorf("copy backend %q: %w", id, err)
|
||||
}
|
||||
definition.ExtraParams = extraParams
|
||||
return definition, nil
|
||||
}
|
||||
|
||||
// CapacityPolicies returns a copy of the normalized policies for limited
|
||||
// backends.
|
||||
func (r *Registry) CapacityPolicies() map[string]domain.BackendCapacityPolicy {
|
||||
policies := make(map[string]domain.BackendCapacityPolicy)
|
||||
if r == nil {
|
||||
return policies
|
||||
}
|
||||
for id, definition := range r.backends {
|
||||
if definition.ConcurrencyLimit == 0 {
|
||||
continue
|
||||
}
|
||||
policies[id] = domain.BackendCapacityPolicy{
|
||||
ConcurrencyLimit: definition.ConcurrencyLimit,
|
||||
QueueCapacity: definition.QueueCapacity,
|
||||
}
|
||||
}
|
||||
return policies
|
||||
}
|
||||
|
||||
func normalizeBackend(definition domain.Backend) (domain.Backend, error) {
|
||||
definition.Endpoint = strings.TrimSpace(definition.Endpoint)
|
||||
if err := validateEndpoint(definition.Endpoint); err != nil {
|
||||
return domain.Backend{}, fmt.Errorf("backend %q endpoint: %w", definition.ID, err)
|
||||
}
|
||||
|
||||
definition.APIKeyEnv = strings.TrimSpace(definition.APIKeyEnv)
|
||||
if definition.APIKeyEnv != "" && !environmentVariableName.MatchString(definition.APIKeyEnv) {
|
||||
return domain.Backend{}, fmt.Errorf(
|
||||
"backend %q api key environment variable %q is invalid",
|
||||
definition.ID,
|
||||
definition.APIKeyEnv,
|
||||
)
|
||||
}
|
||||
|
||||
if definition.ConcurrencyLimit < 0 {
|
||||
return domain.Backend{}, fmt.Errorf(
|
||||
"backend %q concurrency limit must not be negative",
|
||||
definition.ID,
|
||||
)
|
||||
}
|
||||
if definition.QueueCapacity < 0 {
|
||||
return domain.Backend{}, fmt.Errorf(
|
||||
"backend %q queue capacity must not be negative",
|
||||
definition.ID,
|
||||
)
|
||||
}
|
||||
if definition.ConcurrencyLimit == 0 {
|
||||
if definition.QueueCapacitySet {
|
||||
return domain.Backend{}, fmt.Errorf(
|
||||
"backend %q queue capacity requires a positive concurrency limit",
|
||||
definition.ID,
|
||||
)
|
||||
}
|
||||
definition.QueueCapacity = 0
|
||||
} else {
|
||||
if !definition.QueueCapacitySet {
|
||||
definition.QueueCapacity = defaultQueueCapacity
|
||||
definition.QueueCapacitySet = true
|
||||
}
|
||||
maxInt := int(^uint(0) >> 1)
|
||||
if definition.QueueCapacity > maxInt-definition.ConcurrencyLimit {
|
||||
return domain.Backend{}, fmt.Errorf(
|
||||
"backend %q total capacity overflows int",
|
||||
definition.ID,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
keys := make([]string, 0, len(definition.ExtraParams))
|
||||
for key := range definition.ExtraParams {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, key := range keys {
|
||||
if key == "" {
|
||||
return domain.Backend{}, fmt.Errorf("backend %q extra parameter key must not be empty", definition.ID)
|
||||
}
|
||||
if llm.IsReservedOpenAIChatRequestField(key) {
|
||||
return domain.Backend{}, fmt.Errorf(
|
||||
"backend %q extra parameter %q collides with a reserved request field",
|
||||
definition.ID,
|
||||
key,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
extraParams, err := jsonvalue.CopyMap(definition.ExtraParams)
|
||||
if err != nil {
|
||||
return domain.Backend{}, fmt.Errorf("backend %q extra parameters: %w", definition.ID, err)
|
||||
}
|
||||
definition.ExtraParams = extraParams
|
||||
return definition, nil
|
||||
}
|
||||
|
||||
func validateEndpoint(endpoint string) error {
|
||||
if endpoint == "" {
|
||||
return errors.New("must not be blank")
|
||||
}
|
||||
if strings.Contains(endpoint, "#") {
|
||||
return errors.New("must not contain a fragment")
|
||||
}
|
||||
|
||||
parsed, err := url.Parse(endpoint)
|
||||
if err != nil {
|
||||
return fmt.Errorf("must be a valid URL: %w", err)
|
||||
}
|
||||
scheme := strings.ToLower(parsed.Scheme)
|
||||
if scheme != "http" && scheme != "https" {
|
||||
return errors.New("must use http or https")
|
||||
}
|
||||
if !parsed.IsAbs() || parsed.Hostname() == "" {
|
||||
return errors.New("must be absolute and include a host")
|
||||
}
|
||||
if parsed.User != nil {
|
||||
return errors.New("must not contain user information")
|
||||
}
|
||||
if parsed.RawQuery != "" || parsed.ForceQuery {
|
||||
return errors.New("must not contain a query string")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
368
internal/backend/registry_test.go
Normal file
368
internal/backend/registry_test.go
Normal file
@@ -0,0 +1,368 @@
|
||||
package backend_test
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/backend"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
)
|
||||
|
||||
const validEndpoint = "https://backend.example/v1"
|
||||
|
||||
func TestRegistryIncludesExactOpenRouterDefinition(t *testing.T) {
|
||||
registry, err := backend.NewRegistry(nil)
|
||||
if err != nil {
|
||||
t.Fatalf("construct registry: %v", err)
|
||||
}
|
||||
|
||||
definition, err := registry.GetBackend(backend.OpenRouterID)
|
||||
if err != nil {
|
||||
t.Fatalf("look up OpenRouter: %v", err)
|
||||
}
|
||||
if definition.ID != "openrouter" ||
|
||||
definition.Endpoint != "https://openrouter.ai/api/v1" ||
|
||||
definition.APIKeyEnv != "OPENROUTER_API_KEY" ||
|
||||
definition.ConcurrencyLimit != 16 ||
|
||||
definition.QueueCapacity != 1024 ||
|
||||
!definition.QueueCapacitySet ||
|
||||
definition.ExtraParams != nil {
|
||||
t.Fatalf("unexpected OpenRouter definition: %#v", definition)
|
||||
}
|
||||
policies := registry.CapacityPolicies()
|
||||
if len(policies) != 1 ||
|
||||
policies["openrouter"] != (domain.BackendCapacityPolicy{
|
||||
ConcurrencyLimit: 16,
|
||||
QueueCapacity: 1024,
|
||||
}) {
|
||||
t.Fatalf("unexpected OpenRouter capacity policies: %#v", policies)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegistryNormalizesUniqueAdditionsAndIsolatesMutations(t *testing.T) {
|
||||
nested := map[string]int{"limit": 2}
|
||||
extraParams := map[string]any{
|
||||
"count": int64(7),
|
||||
"nested": nested,
|
||||
}
|
||||
registry, err := backend.NewRegistry([]domain.Backend{
|
||||
{
|
||||
ID: " custom ",
|
||||
Endpoint: " https://custom.example/openai/v1 ",
|
||||
APIKeyEnv: " CUSTOM_API_KEY ",
|
||||
ExtraParams: extraParams,
|
||||
ConcurrencyLimit: 3,
|
||||
QueueCapacity: 2,
|
||||
QueueCapacitySet: true,
|
||||
},
|
||||
{
|
||||
ID: "Custom",
|
||||
Endpoint: validEndpoint,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("construct registry: %v", err)
|
||||
}
|
||||
|
||||
nested["limit"] = 99
|
||||
extraParams["added"] = true
|
||||
|
||||
got, err := registry.GetBackend("custom")
|
||||
if err != nil {
|
||||
t.Fatalf("look up custom backend: %v", err)
|
||||
}
|
||||
if got.ID != "custom" ||
|
||||
got.Endpoint != "https://custom.example/openai/v1" ||
|
||||
got.APIKeyEnv != "CUSTOM_API_KEY" ||
|
||||
got.ConcurrencyLimit != 3 ||
|
||||
got.QueueCapacity != 2 ||
|
||||
!got.QueueCapacitySet {
|
||||
t.Fatalf("unexpected normalized definition: %#v", got)
|
||||
}
|
||||
if count, ok := got.ExtraParams["count"].(int64); !ok || count != 7 {
|
||||
t.Fatalf("integer type or value changed: %#v", got.ExtraParams["count"])
|
||||
}
|
||||
gotNested, ok := got.ExtraParams["nested"].(map[string]int)
|
||||
if !ok || gotNested["limit"] != 2 {
|
||||
t.Fatalf("container type or value changed: %#v", got.ExtraParams["nested"])
|
||||
}
|
||||
if _, exists := got.ExtraParams["added"]; exists {
|
||||
t.Fatalf("registry retained caller map: %#v", got.ExtraParams)
|
||||
}
|
||||
|
||||
gotNested["limit"] = 100
|
||||
got.ExtraParams["added"] = true
|
||||
again, err := registry.GetBackend("custom")
|
||||
if err != nil {
|
||||
t.Fatalf("look up custom backend again: %v", err)
|
||||
}
|
||||
if again.ExtraParams["nested"].(map[string]int)["limit"] != 2 {
|
||||
t.Fatalf("lookup exposed registry nested map: %#v", again.ExtraParams)
|
||||
}
|
||||
if _, exists := again.ExtraParams["added"]; exists {
|
||||
t.Fatalf("lookup exposed registry map: %#v", again.ExtraParams)
|
||||
}
|
||||
|
||||
if _, err := registry.GetBackend("Custom"); err != nil {
|
||||
t.Fatalf("backend IDs should be case-sensitive: %v", err)
|
||||
}
|
||||
|
||||
policies := registry.CapacityPolicies()
|
||||
if len(policies) != 2 {
|
||||
t.Fatalf("unexpected capacity policy count: %#v", policies)
|
||||
}
|
||||
policies["custom"] = domain.BackendCapacityPolicy{}
|
||||
delete(policies, backend.OpenRouterID)
|
||||
againPolicies := registry.CapacityPolicies()
|
||||
if againPolicies["custom"] != (domain.BackendCapacityPolicy{
|
||||
ConcurrencyLimit: 3,
|
||||
QueueCapacity: 2,
|
||||
}) {
|
||||
t.Fatalf("capacity policy map mutated registry state: %#v", againPolicies)
|
||||
}
|
||||
if _, ok := againPolicies[backend.OpenRouterID]; !ok {
|
||||
t.Fatalf("capacity policy deletion mutated registry state: %#v", againPolicies)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewRegistryNormalizesCapacityPolicy(t *testing.T) {
|
||||
maxInt := int(^uint(0) >> 1)
|
||||
tests := []struct {
|
||||
name string
|
||||
definition domain.Backend
|
||||
want domain.BackendCapacityPolicy
|
||||
wantSet bool
|
||||
wantError bool
|
||||
}{
|
||||
{
|
||||
name: "unlimited when omitted",
|
||||
definition: domain.Backend{},
|
||||
},
|
||||
{
|
||||
name: "default queue",
|
||||
definition: domain.Backend{
|
||||
ConcurrencyLimit: 2,
|
||||
},
|
||||
want: domain.BackendCapacityPolicy{
|
||||
ConcurrencyLimit: 2,
|
||||
QueueCapacity: 1024,
|
||||
},
|
||||
wantSet: true,
|
||||
},
|
||||
{
|
||||
name: "explicit zero queue",
|
||||
definition: domain.Backend{
|
||||
ConcurrencyLimit: 2,
|
||||
QueueCapacitySet: true,
|
||||
},
|
||||
want: domain.BackendCapacityPolicy{
|
||||
ConcurrencyLimit: 2,
|
||||
},
|
||||
wantSet: true,
|
||||
},
|
||||
{
|
||||
name: "negative concurrency limit",
|
||||
definition: domain.Backend{
|
||||
ConcurrencyLimit: -1,
|
||||
},
|
||||
wantError: true,
|
||||
},
|
||||
{
|
||||
name: "negative queue capacity",
|
||||
definition: domain.Backend{
|
||||
ConcurrencyLimit: 1,
|
||||
QueueCapacity: -1,
|
||||
QueueCapacitySet: true,
|
||||
},
|
||||
wantError: true,
|
||||
},
|
||||
{
|
||||
name: "queue without limit",
|
||||
definition: domain.Backend{
|
||||
QueueCapacitySet: true,
|
||||
},
|
||||
wantError: true,
|
||||
},
|
||||
{
|
||||
name: "total overflow",
|
||||
definition: domain.Backend{
|
||||
ConcurrencyLimit: maxInt,
|
||||
QueueCapacity: 1,
|
||||
QueueCapacitySet: true,
|
||||
},
|
||||
wantError: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
tc.definition.ID = "custom"
|
||||
tc.definition.Endpoint = validEndpoint
|
||||
registry, err := backend.NewRegistry([]domain.Backend{tc.definition})
|
||||
if tc.wantError {
|
||||
if err == nil {
|
||||
t.Fatal("expected invalid capacity policy error")
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("construct registry: %v", err)
|
||||
}
|
||||
|
||||
definition, err := registry.GetBackend("custom")
|
||||
if err != nil {
|
||||
t.Fatalf("look up custom backend: %v", err)
|
||||
}
|
||||
if definition.ConcurrencyLimit != tc.want.ConcurrencyLimit ||
|
||||
definition.QueueCapacity != tc.want.QueueCapacity ||
|
||||
definition.QueueCapacitySet != tc.wantSet {
|
||||
t.Fatalf("normalized capacity=(%d, %d, %t), want (%d, %d, %t)",
|
||||
definition.ConcurrencyLimit,
|
||||
definition.QueueCapacity,
|
||||
definition.QueueCapacitySet,
|
||||
tc.want.ConcurrencyLimit,
|
||||
tc.want.QueueCapacity,
|
||||
tc.wantSet,
|
||||
)
|
||||
}
|
||||
policies := registry.CapacityPolicies()
|
||||
got, ok := policies["custom"]
|
||||
if ok != tc.wantSet || got != tc.want {
|
||||
t.Fatalf("capacity policy=(%#v, %t), want (%#v, %t)", got, ok, tc.want, tc.wantSet)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewRegistryRejectsDuplicateIDs(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
additions []domain.Backend
|
||||
wantID string
|
||||
}{
|
||||
{
|
||||
name: "built-in collision after normalization",
|
||||
additions: []domain.Backend{{
|
||||
ID: " openrouter ",
|
||||
}},
|
||||
wantID: "openrouter",
|
||||
},
|
||||
{
|
||||
name: "consumer collision after normalization",
|
||||
additions: []domain.Backend{
|
||||
{ID: "custom", Endpoint: validEndpoint},
|
||||
{ID: " custom ", Endpoint: "https://other.example/v1"},
|
||||
},
|
||||
wantID: "custom",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
_, err := backend.NewRegistry(tc.additions)
|
||||
if err == nil {
|
||||
t.Fatal("expected duplicate ID error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), tc.wantID) {
|
||||
t.Fatalf("expected error to identify %q, got %v", tc.wantID, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewRegistryValidatesIDs(t *testing.T) {
|
||||
for _, id := range []string{"", " \t\n "} {
|
||||
t.Run(id, func(t *testing.T) {
|
||||
_, err := backend.NewRegistry([]domain.Backend{{
|
||||
ID: id,
|
||||
Endpoint: validEndpoint,
|
||||
}})
|
||||
if err == nil {
|
||||
t.Fatal("expected blank ID error")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewRegistryValidatesEndpoints(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
endpoint string
|
||||
}{
|
||||
{name: "blank", endpoint: ""},
|
||||
{name: "relative", endpoint: "/v1"},
|
||||
{name: "missing host", endpoint: "https:///v1"},
|
||||
{name: "unsupported scheme", endpoint: "ftp://backend.example/v1"},
|
||||
{name: "user information", endpoint: "https://user@backend.example/v1"},
|
||||
{name: "query", endpoint: "https://backend.example/v1?mode=chat"},
|
||||
{name: "empty query", endpoint: "https://backend.example/v1?"},
|
||||
{name: "fragment", endpoint: "https://backend.example/v1#chat"},
|
||||
{name: "empty fragment", endpoint: "https://backend.example/v1#"},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
_, err := backend.NewRegistry([]domain.Backend{{
|
||||
ID: "custom",
|
||||
Endpoint: tc.endpoint,
|
||||
}})
|
||||
if err == nil {
|
||||
t.Fatal("expected invalid endpoint error")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewRegistryValidatesEnvironmentVariableNames(t *testing.T) {
|
||||
for _, name := range []string{"1API_KEY", "API-KEY", "API KEY", "ÅPI_KEY"} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
_, err := backend.NewRegistry([]domain.Backend{{
|
||||
ID: "custom",
|
||||
Endpoint: validEndpoint,
|
||||
APIKeyEnv: name,
|
||||
}})
|
||||
if err == nil {
|
||||
t.Fatal("expected invalid environment-variable name error")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewRegistryRejectsInvalidAndReservedExtraParameters(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
extraParams map[string]any
|
||||
}{
|
||||
{name: "unsupported value", extraParams: map[string]any{"value": make(chan int)}},
|
||||
{name: "reserved key", extraParams: map[string]any{"model": "override"}},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
_, err := backend.NewRegistry([]domain.Backend{{
|
||||
ID: "custom",
|
||||
Endpoint: validEndpoint,
|
||||
ExtraParams: tc.extraParams,
|
||||
}})
|
||||
if err == nil {
|
||||
t.Fatal("expected invalid extra parameters error")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegistryLookupReportsNotFound(t *testing.T) {
|
||||
registry, err := backend.NewRegistry(nil)
|
||||
if err != nil {
|
||||
t.Fatalf("construct registry: %v", err)
|
||||
}
|
||||
|
||||
_, err = registry.GetBackend("missing")
|
||||
if !errors.Is(err, backend.ErrBackendNotFound) {
|
||||
t.Fatalf("expected ErrBackendNotFound, got %v", err)
|
||||
}
|
||||
if !strings.Contains(err.Error(), "missing") {
|
||||
t.Fatalf("expected error to identify backend, got %v", err)
|
||||
}
|
||||
}
|
||||
40
internal/capacity/client.go
Normal file
40
internal/capacity/client.go
Normal file
@@ -0,0 +1,40 @@
|
||||
package capacity
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/llm"
|
||||
)
|
||||
|
||||
type client struct {
|
||||
manager *Manager
|
||||
next llm.Client
|
||||
}
|
||||
|
||||
// NewClient wraps next with configured active-generation limits. A nil manager
|
||||
// leaves next unchanged.
|
||||
func NewClient(manager *Manager, next llm.Client) llm.Client {
|
||||
if manager == nil {
|
||||
return next
|
||||
}
|
||||
return &client{
|
||||
manager: manager,
|
||||
next: next,
|
||||
}
|
||||
}
|
||||
|
||||
func (c *client) Generate(
|
||||
ctx context.Context,
|
||||
req domain.GenerateRequest,
|
||||
) (*domain.GenerateResponse, error) {
|
||||
pool := c.manager.getPool(req.Target.BackendID)
|
||||
if pool == nil {
|
||||
return c.next.Generate(ctx, req)
|
||||
}
|
||||
if err := pool.acquire(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer pool.releaseActive()
|
||||
return c.next.Generate(ctx, req)
|
||||
}
|
||||
517
internal/capacity/client_test.go
Normal file
517
internal/capacity/client_test.go
Normal file
@@ -0,0 +1,517 @@
|
||||
package capacity
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"reflect"
|
||||
"runtime"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/llm"
|
||||
)
|
||||
|
||||
type generateResult struct {
|
||||
response *domain.GenerateResponse
|
||||
err error
|
||||
}
|
||||
|
||||
type clientFunc func(
|
||||
context.Context,
|
||||
domain.GenerateRequest,
|
||||
) (*domain.GenerateResponse, error)
|
||||
|
||||
func (f clientFunc) Generate(
|
||||
ctx context.Context,
|
||||
req domain.GenerateRequest,
|
||||
) (*domain.GenerateResponse, error) {
|
||||
return f(ctx, req)
|
||||
}
|
||||
|
||||
type blockingClient struct {
|
||||
mu sync.Mutex
|
||||
active int
|
||||
peak int
|
||||
calls map[string]int
|
||||
started chan string
|
||||
releases map[string]chan struct{}
|
||||
}
|
||||
|
||||
func newBlockingClient(releases map[string]chan struct{}) *blockingClient {
|
||||
return &blockingClient{
|
||||
calls: make(map[string]int),
|
||||
started: make(chan string, 64),
|
||||
releases: releases,
|
||||
}
|
||||
}
|
||||
|
||||
func (c *blockingClient) Generate(
|
||||
ctx context.Context,
|
||||
req domain.GenerateRequest,
|
||||
) (*domain.GenerateResponse, error) {
|
||||
id := req.Prompt.SessionID
|
||||
c.mu.Lock()
|
||||
c.active++
|
||||
if c.active > c.peak {
|
||||
c.peak = c.active
|
||||
}
|
||||
c.calls[id]++
|
||||
c.mu.Unlock()
|
||||
defer func() {
|
||||
c.mu.Lock()
|
||||
c.active--
|
||||
c.mu.Unlock()
|
||||
}()
|
||||
|
||||
c.started <- id
|
||||
if release := c.releases[id]; release != nil {
|
||||
select {
|
||||
case <-release:
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
}
|
||||
return &domain.GenerateResponse{Content: id}, nil
|
||||
}
|
||||
|
||||
func (c *blockingClient) callCount(id string) int {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
return c.calls[id]
|
||||
}
|
||||
|
||||
func (c *blockingClient) peakConcurrency() int {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
return c.peak
|
||||
}
|
||||
|
||||
func generateAsync(
|
||||
client llm.Client,
|
||||
ctx context.Context,
|
||||
backendID string,
|
||||
id string,
|
||||
) <-chan generateResult {
|
||||
result := make(chan generateResult, 1)
|
||||
go func() {
|
||||
response, err := client.Generate(ctx, domain.GenerateRequest{
|
||||
Prompt: domain.RenderedPrompt{SessionID: id},
|
||||
Target: domain.ExecutionTarget{BackendID: backendID},
|
||||
})
|
||||
result <- generateResult{response: response, err: err}
|
||||
}()
|
||||
return result
|
||||
}
|
||||
|
||||
func waitForWaiterCount(t *testing.T, manager *Manager, backendID string, want int) {
|
||||
t.Helper()
|
||||
pool := manager.pools[backendID]
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for {
|
||||
pool.mu.Lock()
|
||||
got := pool.waiters.Len()
|
||||
pool.mu.Unlock()
|
||||
if got == want {
|
||||
return
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("waiter count=%d, want %d", got, want)
|
||||
}
|
||||
runtime.Gosched()
|
||||
}
|
||||
}
|
||||
|
||||
func receiveStarted(t *testing.T, started <-chan string) string {
|
||||
t.Helper()
|
||||
select {
|
||||
case id := <-started:
|
||||
return id
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("timed out waiting for wrapped client invocation")
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
func receiveResult(t *testing.T, result <-chan generateResult) generateResult {
|
||||
t.Helper()
|
||||
select {
|
||||
case got := <-result:
|
||||
return got
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("timed out waiting for generation result")
|
||||
return generateResult{}
|
||||
}
|
||||
}
|
||||
|
||||
func newTestManager(t *testing.T, policies map[string]domain.BackendCapacityPolicy) *Manager {
|
||||
t.Helper()
|
||||
manager, err := NewManager(policies)
|
||||
if err != nil {
|
||||
t.Fatalf("construct manager: %v", err)
|
||||
}
|
||||
return manager
|
||||
}
|
||||
|
||||
func TestClientLimitsPeakConcurrencyAndServesWaitersFIFO(t *testing.T) {
|
||||
manager := newTestManager(t, map[string]domain.BackendCapacityPolicy{
|
||||
"limited": {ConcurrencyLimit: 1},
|
||||
})
|
||||
firstRelease := make(chan struct{})
|
||||
secondRelease := make(chan struct{})
|
||||
thirdRelease := make(chan struct{})
|
||||
next := newBlockingClient(map[string]chan struct{}{
|
||||
"first": firstRelease,
|
||||
"second": secondRelease,
|
||||
"third": thirdRelease,
|
||||
})
|
||||
client := NewClient(manager, next)
|
||||
|
||||
first := generateAsync(client, context.Background(), "limited", "first")
|
||||
if got := receiveStarted(t, next.started); got != "first" {
|
||||
t.Fatalf("first invocation=%q, want first", got)
|
||||
}
|
||||
second := generateAsync(client, context.Background(), "limited", "second")
|
||||
waitForWaiterCount(t, manager, "limited", 1)
|
||||
third := generateAsync(client, context.Background(), "limited", "third")
|
||||
waitForWaiterCount(t, manager, "limited", 2)
|
||||
|
||||
close(firstRelease)
|
||||
if got := receiveResult(t, first); got.err != nil {
|
||||
t.Fatalf("first generation: %v", got.err)
|
||||
}
|
||||
if got := receiveStarted(t, next.started); got != "second" {
|
||||
t.Fatalf("second invocation=%q, want second", got)
|
||||
}
|
||||
close(secondRelease)
|
||||
if got := receiveResult(t, second); got.err != nil {
|
||||
t.Fatalf("second generation: %v", got.err)
|
||||
}
|
||||
if got := receiveStarted(t, next.started); got != "third" {
|
||||
t.Fatalf("third invocation=%q, want third", got)
|
||||
}
|
||||
close(thirdRelease)
|
||||
if got := receiveResult(t, third); got.err != nil {
|
||||
t.Fatalf("third generation: %v", got.err)
|
||||
}
|
||||
if peak := next.peakConcurrency(); peak != 1 {
|
||||
t.Fatalf("peak concurrency=%d, want 1", peak)
|
||||
}
|
||||
}
|
||||
|
||||
func TestClientPeakConcurrencyDoesNotExceedConfiguredLimit(t *testing.T) {
|
||||
const limit = 2
|
||||
manager := newTestManager(t, map[string]domain.BackendCapacityPolicy{
|
||||
"limited": {ConcurrencyLimit: limit},
|
||||
})
|
||||
gate := make(chan struct{})
|
||||
releases := make(map[string]chan struct{})
|
||||
for i := range 5 {
|
||||
releases[string(rune('a'+i))] = gate
|
||||
}
|
||||
next := newBlockingClient(releases)
|
||||
client := NewClient(manager, next)
|
||||
|
||||
results := make([]<-chan generateResult, 0, len(releases))
|
||||
for id := range releases {
|
||||
results = append(results, generateAsync(client, context.Background(), "limited", id))
|
||||
}
|
||||
for range limit {
|
||||
receiveStarted(t, next.started)
|
||||
}
|
||||
waitForWaiterCount(t, manager, "limited", len(releases)-limit)
|
||||
|
||||
close(gate)
|
||||
for _, result := range results {
|
||||
if got := receiveResult(t, result); got.err != nil {
|
||||
t.Fatalf("generation: %v", got.err)
|
||||
}
|
||||
}
|
||||
if peak := next.peakConcurrency(); peak != limit {
|
||||
t.Fatalf("peak concurrency=%d, want %d", peak, limit)
|
||||
}
|
||||
}
|
||||
|
||||
func TestClientRemovesCanceledWaiters(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
cancelID string
|
||||
wantOrder []string
|
||||
}{
|
||||
{name: "first waiter", cancelID: "one", wantOrder: []string{"two", "three"}},
|
||||
{name: "middle waiter", cancelID: "two", wantOrder: []string{"one", "three"}},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
manager := newTestManager(t, map[string]domain.BackendCapacityPolicy{
|
||||
"limited": {ConcurrencyLimit: 1},
|
||||
})
|
||||
holderRelease := make(chan struct{})
|
||||
releases := map[string]chan struct{}{
|
||||
"holder": holderRelease,
|
||||
"one": make(chan struct{}),
|
||||
"two": make(chan struct{}),
|
||||
"three": make(chan struct{}),
|
||||
}
|
||||
next := newBlockingClient(releases)
|
||||
client := NewClient(manager, next)
|
||||
|
||||
holder := generateAsync(client, context.Background(), "limited", "holder")
|
||||
if got := receiveStarted(t, next.started); got != "holder" {
|
||||
t.Fatalf("initial invocation=%q, want holder", got)
|
||||
}
|
||||
|
||||
contexts := make(map[string]context.Context)
|
||||
cancels := make(map[string]context.CancelFunc)
|
||||
results := make(map[string]<-chan generateResult)
|
||||
for _, id := range []string{"one", "two", "three"} {
|
||||
contexts[id], cancels[id] = context.WithCancel(context.Background())
|
||||
results[id] = generateAsync(client, contexts[id], "limited", id)
|
||||
waitForWaiterCount(t, manager, "limited", len(results))
|
||||
}
|
||||
|
||||
cancels[tc.cancelID]()
|
||||
if got := receiveResult(t, results[tc.cancelID]); !errors.Is(got.err, context.Canceled) {
|
||||
t.Fatalf("canceled waiter error=%v, want context.Canceled", got.err)
|
||||
}
|
||||
waitForWaiterCount(t, manager, "limited", 2)
|
||||
|
||||
close(holderRelease)
|
||||
if got := receiveResult(t, holder); got.err != nil {
|
||||
t.Fatalf("holder generation: %v", got.err)
|
||||
}
|
||||
for _, id := range tc.wantOrder {
|
||||
if got := receiveStarted(t, next.started); got != id {
|
||||
t.Fatalf("next invocation=%q, want %q", got, id)
|
||||
}
|
||||
close(releases[id])
|
||||
if got := receiveResult(t, results[id]); got.err != nil {
|
||||
t.Fatalf("%s generation: %v", id, got.err)
|
||||
}
|
||||
}
|
||||
if calls := next.callCount(tc.cancelID); calls != 0 {
|
||||
t.Fatalf("canceled waiter invoked wrapped client %d times", calls)
|
||||
}
|
||||
for _, cancel := range cancels {
|
||||
cancel()
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestClientGrantCancellationRaceDoesNotLeakPermit(t *testing.T) {
|
||||
const iterations = 200
|
||||
for i := range iterations {
|
||||
manager := newTestManager(t, map[string]domain.BackendCapacityPolicy{
|
||||
"limited": {ConcurrencyLimit: 1},
|
||||
})
|
||||
holderRelease := make(chan struct{})
|
||||
var waiterCalls atomic.Int64
|
||||
next := clientFunc(func(
|
||||
_ context.Context,
|
||||
req domain.GenerateRequest,
|
||||
) (*domain.GenerateResponse, error) {
|
||||
if req.Prompt.SessionID == "holder" {
|
||||
<-holderRelease
|
||||
} else if req.Prompt.SessionID == "waiter" {
|
||||
waiterCalls.Add(1)
|
||||
}
|
||||
return &domain.GenerateResponse{Content: req.Prompt.SessionID}, nil
|
||||
})
|
||||
client := NewClient(manager, next)
|
||||
|
||||
holder := generateAsync(client, context.Background(), "limited", "holder")
|
||||
waitForActiveCount(t, manager, "limited", 1)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
waiterResult := generateAsync(client, ctx, "limited", "waiter")
|
||||
waitForWaiterCount(t, manager, "limited", 1)
|
||||
|
||||
start := make(chan struct{})
|
||||
var race sync.WaitGroup
|
||||
race.Add(2)
|
||||
go func() {
|
||||
defer race.Done()
|
||||
<-start
|
||||
cancel()
|
||||
}()
|
||||
go func() {
|
||||
defer race.Done()
|
||||
<-start
|
||||
close(holderRelease)
|
||||
}()
|
||||
close(start)
|
||||
race.Wait()
|
||||
|
||||
if got := receiveResult(t, holder); got.err != nil {
|
||||
t.Fatalf("iteration %d holder generation: %v", i, got.err)
|
||||
}
|
||||
got := receiveResult(t, waiterResult)
|
||||
switch calls := waiterCalls.Load(); {
|
||||
case calls == 0 && errors.Is(got.err, context.Canceled):
|
||||
case calls == 1 && got.err == nil:
|
||||
default:
|
||||
t.Fatalf("iteration %d waiter calls=%d error=%v", i, calls, got.err)
|
||||
}
|
||||
|
||||
probe := generateAsync(client, context.Background(), "limited", "probe")
|
||||
if got := receiveResult(t, probe); got.err != nil {
|
||||
t.Fatalf("iteration %d probe generation: %v", i, got.err)
|
||||
}
|
||||
waitForActiveCount(t, manager, "limited", 0)
|
||||
waitForWaiterCount(t, manager, "limited", 0)
|
||||
}
|
||||
}
|
||||
|
||||
func waitForActiveCount(t *testing.T, manager *Manager, backendID string, want int) {
|
||||
t.Helper()
|
||||
pool := manager.pools[backendID]
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for {
|
||||
pool.mu.Lock()
|
||||
got := pool.active
|
||||
pool.mu.Unlock()
|
||||
if got == want {
|
||||
return
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("active count=%d, want %d", got, want)
|
||||
}
|
||||
runtime.Gosched()
|
||||
}
|
||||
}
|
||||
|
||||
func TestClientUsesIndependentPoolsAndUnlimitedFastPaths(t *testing.T) {
|
||||
manager := newTestManager(t, map[string]domain.BackendCapacityPolicy{
|
||||
"alpha": {ConcurrencyLimit: 1},
|
||||
"beta": {ConcurrencyLimit: 1},
|
||||
})
|
||||
alphaRelease := make(chan struct{})
|
||||
betaRelease := make(chan struct{})
|
||||
next := newBlockingClient(map[string]chan struct{}{
|
||||
"alpha": alphaRelease,
|
||||
"beta": betaRelease,
|
||||
})
|
||||
client := NewClient(manager, next)
|
||||
|
||||
alpha := generateAsync(client, context.Background(), "alpha", "alpha")
|
||||
beta := generateAsync(client, context.Background(), "beta", "beta")
|
||||
started := map[string]bool{
|
||||
receiveStarted(t, next.started): true,
|
||||
receiveStarted(t, next.started): true,
|
||||
}
|
||||
if !started["alpha"] || !started["beta"] {
|
||||
t.Fatalf("independent pools did not both start: %#v", started)
|
||||
}
|
||||
close(alphaRelease)
|
||||
close(betaRelease)
|
||||
if got := receiveResult(t, alpha); got.err != nil {
|
||||
t.Fatalf("alpha generation: %v", got.err)
|
||||
}
|
||||
if got := receiveResult(t, beta); got.err != nil {
|
||||
t.Fatalf("beta generation: %v", got.err)
|
||||
}
|
||||
|
||||
for _, backendID := range []string{"", "unknown"} {
|
||||
response, err := client.Generate(context.Background(), domain.GenerateRequest{
|
||||
Prompt: domain.RenderedPrompt{SessionID: backendID},
|
||||
Target: domain.ExecutionTarget{BackendID: backendID},
|
||||
})
|
||||
if err != nil || response == nil {
|
||||
t.Fatalf("unlimited backend %q response=(%#v, %v)", backendID, response, err)
|
||||
}
|
||||
}
|
||||
if got := NewClient(nil, next); got != next {
|
||||
t.Fatal("nil manager did not return the wrapped client unchanged")
|
||||
}
|
||||
}
|
||||
|
||||
func TestClientPreservesRequestsResponsesAndErrors(t *testing.T) {
|
||||
manager := newTestManager(t, map[string]domain.BackendCapacityPolicy{
|
||||
"limited": {ConcurrencyLimit: 1},
|
||||
})
|
||||
request := domain.GenerateRequest{
|
||||
Prompt: domain.RenderedPrompt{
|
||||
SessionID: "session",
|
||||
Messages: []domain.RenderedMessage{
|
||||
{Role: "user", Content: "content"},
|
||||
},
|
||||
},
|
||||
Target: domain.ExecutionTarget{
|
||||
BackendID: "limited",
|
||||
Model: "model",
|
||||
ExtraParams: map[string]any{"key": "value"},
|
||||
},
|
||||
}
|
||||
response := &domain.GenerateResponse{
|
||||
Content: "output",
|
||||
Usage: domain.TokenUsage{TotalTokens: 7},
|
||||
}
|
||||
collaboratorErr := errors.New("collaborator failure")
|
||||
tests := []struct {
|
||||
name string
|
||||
response *domain.GenerateResponse
|
||||
err error
|
||||
}{
|
||||
{name: "successful response", response: response},
|
||||
{name: "nil response"},
|
||||
{name: "collaborator error", response: response, err: collaboratorErr},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
var captured domain.GenerateRequest
|
||||
next := clientFunc(func(
|
||||
_ context.Context,
|
||||
req domain.GenerateRequest,
|
||||
) (*domain.GenerateResponse, error) {
|
||||
captured = req
|
||||
return tc.response, tc.err
|
||||
})
|
||||
gotResponse, gotErr := NewClient(manager, next).Generate(context.Background(), request)
|
||||
if !reflect.DeepEqual(captured, request) {
|
||||
t.Fatalf("request changed: %#v", captured)
|
||||
}
|
||||
if gotResponse != tc.response || gotErr != tc.err {
|
||||
t.Fatalf("response=(%p, %v), want (%p, %v)",
|
||||
gotResponse, gotErr, tc.response, tc.err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestClientReleasesPermitDuringPanicUnwinding(t *testing.T) {
|
||||
manager := newTestManager(t, map[string]domain.BackendCapacityPolicy{
|
||||
"limited": {ConcurrencyLimit: 1},
|
||||
})
|
||||
var calls atomic.Int64
|
||||
next := clientFunc(func(
|
||||
_ context.Context,
|
||||
_ domain.GenerateRequest,
|
||||
) (*domain.GenerateResponse, error) {
|
||||
if calls.Add(1) == 1 {
|
||||
panic("test panic")
|
||||
}
|
||||
return &domain.GenerateResponse{Content: "recovered"}, nil
|
||||
})
|
||||
client := NewClient(manager, next)
|
||||
request := domain.GenerateRequest{
|
||||
Target: domain.ExecutionTarget{BackendID: "limited"},
|
||||
}
|
||||
|
||||
func() {
|
||||
defer func() {
|
||||
if recover() == nil {
|
||||
t.Fatal("expected wrapped client panic")
|
||||
}
|
||||
}()
|
||||
_, _ = client.Generate(context.Background(), request)
|
||||
}()
|
||||
|
||||
response, err := client.Generate(context.Background(), request)
|
||||
if err != nil || response == nil || response.Content != "recovered" {
|
||||
t.Fatalf("generation after panic=(%#v, %v)", response, err)
|
||||
}
|
||||
}
|
||||
160
internal/capacity/manager.go
Normal file
160
internal/capacity/manager.go
Normal file
@@ -0,0 +1,160 @@
|
||||
// Package capacity coordinates engine-local run admission and model-generation
|
||||
// concurrency for configured backends.
|
||||
package capacity
|
||||
|
||||
import (
|
||||
"container/list"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
)
|
||||
|
||||
// ErrCapacityExceeded identifies an admission rejected because a backend's
|
||||
// configured run capacity is full.
|
||||
var ErrCapacityExceeded = errors.New("backend capacity exceeded")
|
||||
|
||||
// Manager owns independent backend capacity pools with immutable limits.
|
||||
type Manager struct {
|
||||
pools map[string]*pool
|
||||
}
|
||||
|
||||
type pool struct {
|
||||
mu sync.Mutex
|
||||
concurrencyLimit int
|
||||
totalCapacity int
|
||||
admitted int
|
||||
active int
|
||||
waiters list.List
|
||||
}
|
||||
|
||||
type waiter struct {
|
||||
ready chan struct{}
|
||||
element *list.Element
|
||||
granted bool
|
||||
}
|
||||
|
||||
// NewManager constructs independent pools from normalized backend policies.
|
||||
func NewManager(policies map[string]domain.BackendCapacityPolicy) (*Manager, error) {
|
||||
manager := &Manager{
|
||||
pools: make(map[string]*pool, len(policies)),
|
||||
}
|
||||
maxInt := int(^uint(0) >> 1)
|
||||
for id, policy := range policies {
|
||||
if strings.TrimSpace(id) == "" {
|
||||
return nil, errors.New("backend capacity policy ID must not be blank")
|
||||
}
|
||||
if policy.ConcurrencyLimit <= 0 {
|
||||
return nil, fmt.Errorf(
|
||||
"backend %q concurrency limit must be positive",
|
||||
id,
|
||||
)
|
||||
}
|
||||
if policy.QueueCapacity < 0 {
|
||||
return nil, fmt.Errorf(
|
||||
"backend %q queue capacity must not be negative",
|
||||
id,
|
||||
)
|
||||
}
|
||||
if policy.QueueCapacity > maxInt-policy.ConcurrencyLimit {
|
||||
return nil, fmt.Errorf("backend %q total capacity overflows int", id)
|
||||
}
|
||||
manager.pools[id] = &pool{
|
||||
concurrencyLimit: policy.ConcurrencyLimit,
|
||||
totalCapacity: policy.ConcurrencyLimit + policy.QueueCapacity,
|
||||
}
|
||||
}
|
||||
return manager, nil
|
||||
}
|
||||
|
||||
// Admit immediately reserves one configured backend run slot. Backends without
|
||||
// a configured pool are unlimited.
|
||||
func (m *Manager) Admit(ctx context.Context, backendID string) (func(), error) {
|
||||
pool := m.getPool(backendID)
|
||||
if pool == nil {
|
||||
return releaseNothing, nil
|
||||
}
|
||||
|
||||
pool.mu.Lock()
|
||||
defer pool.mu.Unlock()
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if pool.admitted >= pool.totalCapacity {
|
||||
return nil, ErrCapacityExceeded
|
||||
}
|
||||
pool.admitted++
|
||||
|
||||
var once sync.Once
|
||||
return func() {
|
||||
once.Do(func() {
|
||||
pool.mu.Lock()
|
||||
pool.admitted--
|
||||
pool.mu.Unlock()
|
||||
})
|
||||
}, nil
|
||||
}
|
||||
|
||||
func releaseNothing() {}
|
||||
|
||||
func (m *Manager) getPool(backendID string) *pool {
|
||||
if m == nil || backendID == "" {
|
||||
return nil
|
||||
}
|
||||
return m.pools[backendID]
|
||||
}
|
||||
|
||||
func (p *pool) acquire(ctx context.Context) error {
|
||||
p.mu.Lock()
|
||||
if err := ctx.Err(); err != nil {
|
||||
p.mu.Unlock()
|
||||
return err
|
||||
}
|
||||
if p.active < p.concurrencyLimit && p.waiters.Len() == 0 {
|
||||
p.active++
|
||||
p.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
waiter := &waiter{ready: make(chan struct{})}
|
||||
waiter.element = p.waiters.PushBack(waiter)
|
||||
p.mu.Unlock()
|
||||
|
||||
select {
|
||||
case <-waiter.ready:
|
||||
return nil
|
||||
case <-ctx.Done():
|
||||
p.mu.Lock()
|
||||
if !waiter.granted {
|
||||
p.waiters.Remove(waiter.element)
|
||||
waiter.element = nil
|
||||
p.mu.Unlock()
|
||||
return ctx.Err()
|
||||
}
|
||||
p.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func (p *pool) releaseActive() {
|
||||
var ready chan struct{}
|
||||
|
||||
p.mu.Lock()
|
||||
if element := p.waiters.Front(); element != nil {
|
||||
waiter := element.Value.(*waiter)
|
||||
p.waiters.Remove(element)
|
||||
waiter.element = nil
|
||||
waiter.granted = true
|
||||
ready = waiter.ready
|
||||
} else {
|
||||
p.active--
|
||||
}
|
||||
p.mu.Unlock()
|
||||
|
||||
if ready != nil {
|
||||
close(ready)
|
||||
}
|
||||
}
|
||||
163
internal/capacity/manager_test.go
Normal file
163
internal/capacity/manager_test.go
Normal file
@@ -0,0 +1,163 @@
|
||||
package capacity
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
)
|
||||
|
||||
func TestNewManagerRejectsInvalidPolicies(t *testing.T) {
|
||||
maxInt := int(^uint(0) >> 1)
|
||||
tests := []struct {
|
||||
name string
|
||||
id string
|
||||
policy domain.BackendCapacityPolicy
|
||||
}{
|
||||
{
|
||||
name: "blank ID",
|
||||
id: " \t ",
|
||||
policy: domain.BackendCapacityPolicy{ConcurrencyLimit: 1},
|
||||
},
|
||||
{
|
||||
name: "zero concurrency",
|
||||
id: "backend",
|
||||
policy: domain.BackendCapacityPolicy{},
|
||||
},
|
||||
{
|
||||
name: "negative concurrency",
|
||||
id: "backend",
|
||||
policy: domain.BackendCapacityPolicy{ConcurrencyLimit: -1},
|
||||
},
|
||||
{
|
||||
name: "negative queue",
|
||||
id: "backend",
|
||||
policy: domain.BackendCapacityPolicy{
|
||||
ConcurrencyLimit: 1,
|
||||
QueueCapacity: -1,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "total overflow",
|
||||
id: "backend",
|
||||
policy: domain.BackendCapacityPolicy{
|
||||
ConcurrencyLimit: maxInt,
|
||||
QueueCapacity: 1,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
_, err := NewManager(map[string]domain.BackendCapacityPolicy{
|
||||
tc.id: tc.policy,
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("expected invalid policy error")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestManagerAdmissionIsBoundedAndReleaseIsIdempotent(t *testing.T) {
|
||||
policies := map[string]domain.BackendCapacityPolicy{
|
||||
"limited": {
|
||||
ConcurrencyLimit: 2,
|
||||
QueueCapacity: 1,
|
||||
},
|
||||
"independent": {
|
||||
ConcurrencyLimit: 1,
|
||||
},
|
||||
}
|
||||
manager, err := NewManager(policies)
|
||||
if err != nil {
|
||||
t.Fatalf("construct manager: %v", err)
|
||||
}
|
||||
policies["limited"] = domain.BackendCapacityPolicy{
|
||||
ConcurrencyLimit: 100,
|
||||
QueueCapacity: 100,
|
||||
}
|
||||
|
||||
releases := make([]func(), 0, 3)
|
||||
for range 3 {
|
||||
release, err := manager.Admit(context.Background(), "limited")
|
||||
if err != nil {
|
||||
t.Fatalf("admit within configured capacity: %v", err)
|
||||
}
|
||||
releases = append(releases, release)
|
||||
}
|
||||
if release, err := manager.Admit(context.Background(), "limited"); release != nil ||
|
||||
!errors.Is(err, ErrCapacityExceeded) {
|
||||
t.Fatalf("admission beyond capacity=(release=%t, err=%v), want ErrCapacityExceeded",
|
||||
release != nil, err)
|
||||
}
|
||||
independentRelease, err := manager.Admit(context.Background(), "independent")
|
||||
if err != nil {
|
||||
t.Fatalf("admit independent backend while first is full: %v", err)
|
||||
}
|
||||
independentRelease()
|
||||
|
||||
releases[0]()
|
||||
releases[0]()
|
||||
replacement, err := manager.Admit(context.Background(), "limited")
|
||||
if err != nil {
|
||||
t.Fatalf("admit after release: %v", err)
|
||||
}
|
||||
replacement()
|
||||
releases[1]()
|
||||
releases[2]()
|
||||
|
||||
pool := manager.pools["limited"]
|
||||
pool.mu.Lock()
|
||||
admitted := pool.admitted
|
||||
pool.mu.Unlock()
|
||||
if admitted != 0 {
|
||||
t.Fatalf("admitted runs after releases=%d, want 0", admitted)
|
||||
}
|
||||
}
|
||||
|
||||
func TestManagerAdmissionHonorsContextAndUnlimitedBackends(t *testing.T) {
|
||||
manager, err := NewManager(map[string]domain.BackendCapacityPolicy{
|
||||
"limited": {ConcurrencyLimit: 1},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("construct manager: %v", err)
|
||||
}
|
||||
release, err := manager.Admit(context.Background(), "limited")
|
||||
if err != nil {
|
||||
t.Fatalf("fill limited pool: %v", err)
|
||||
}
|
||||
defer release()
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
if release, err := manager.Admit(ctx, "limited"); release != nil ||
|
||||
!errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("canceled limited admission=(release=%t, err=%v), want context cancellation",
|
||||
release != nil, err)
|
||||
}
|
||||
|
||||
var nilManager *Manager
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
manager *Manager
|
||||
backendID string
|
||||
}{
|
||||
{name: "nil manager", manager: nilManager, backendID: "limited"},
|
||||
{name: "blank ID", manager: manager},
|
||||
{name: "unknown ID", manager: manager, backendID: "unknown"},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
release, err := tc.manager.Admit(ctx, tc.backendID)
|
||||
if err != nil {
|
||||
t.Fatalf("unlimited admission: %v", err)
|
||||
}
|
||||
if release == nil {
|
||||
t.Fatal("unlimited admission returned nil release")
|
||||
}
|
||||
release()
|
||||
release()
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -14,9 +14,6 @@ const (
|
||||
ContentTypeApplicationJSON = "application/json"
|
||||
OpenAIChatCompletionsPath = "/chat/completions"
|
||||
|
||||
ExecutionDefaultTemperature = 0.0
|
||||
ExecutionDefaultMaxTokens = 0
|
||||
ExecutionDefaultTopP = 1.0
|
||||
ExecutionDefaultTimeoutSeconds = 600
|
||||
)
|
||||
|
||||
@@ -26,9 +23,6 @@ var (
|
||||
|
||||
func ExecutionTargetDefault() domain.ExecutionTarget {
|
||||
return domain.ExecutionTarget{
|
||||
Temperature: ExecutionDefaultTemperature,
|
||||
MaxTokens: ExecutionDefaultMaxTokens,
|
||||
TopP: ExecutionDefaultTopP,
|
||||
TimeoutSeconds: ExecutionDefaultTimeoutSeconds,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -63,12 +63,12 @@ type RunRequest struct {
|
||||
PromptID string
|
||||
PromptVersion string
|
||||
ProfileID string
|
||||
SessionID string
|
||||
APIKey string `json:"-" yaml:"-"`
|
||||
Inputs map[string]ArtifactRef
|
||||
Vars map[string]string
|
||||
Execution *ExecutionTargetOverride
|
||||
Validation *OutputContract
|
||||
Metadata map[string]string
|
||||
}
|
||||
|
||||
// RunResult represents the complete result of a prompt execution run.
|
||||
@@ -80,8 +80,10 @@ type RunResult struct {
|
||||
PromptID string
|
||||
PromptVersion string
|
||||
PromptHash string
|
||||
SessionID string
|
||||
RenderedPromptHash string
|
||||
SelectedProfileID string
|
||||
SelectedBackendID string
|
||||
ModelName string
|
||||
Endpoint string
|
||||
EffectiveModelParams ExecutionTarget
|
||||
@@ -99,6 +101,7 @@ type PreparedRun struct {
|
||||
PromptVersion string `json:"prompt_version,omitempty"`
|
||||
PromptHash string `json:"prompt_hash,omitempty"`
|
||||
SelectedProfileID string `json:"selected_profile_id"`
|
||||
SelectedBackendID string `json:"selected_backend_id,omitempty"`
|
||||
EffectiveModelParams ExecutionTarget `json:"effective_model_params"`
|
||||
TargetPresence ExecutionTargetPresence `json:"-"`
|
||||
OutputContract OutputContract `json:"output_contract"`
|
||||
@@ -142,6 +145,16 @@ type PromptDefinition struct {
|
||||
Validation OutputContract `yaml:"validation"`
|
||||
}
|
||||
|
||||
// PromptInspection is the resolved result of exact prompt inspection.
|
||||
type PromptInspection struct {
|
||||
PromptID string
|
||||
PromptVersion string
|
||||
PromptHash string
|
||||
DefaultProfileID string
|
||||
Inputs []PromptInput
|
||||
OutputContract OutputContract
|
||||
}
|
||||
|
||||
// PromptInput describes one named input expected by a prompt definition.
|
||||
type PromptInput struct {
|
||||
Name string `yaml:"name"`
|
||||
@@ -158,9 +171,28 @@ type PromptMessageTemplate struct {
|
||||
CacheControl *CacheControl `yaml:"cache_control,omitempty" json:"cache_control,omitempty"`
|
||||
}
|
||||
|
||||
// Backend describes reusable OpenAI-compatible connection defaults.
|
||||
type Backend struct {
|
||||
ID string
|
||||
Endpoint string
|
||||
APIKeyEnv string
|
||||
ExtraParams map[string]any
|
||||
ConcurrencyLimit int
|
||||
QueueCapacity int
|
||||
QueueCapacitySet bool
|
||||
}
|
||||
|
||||
// BackendCapacityPolicy describes normalized run and generation capacity for
|
||||
// one limited backend.
|
||||
type BackendCapacityPolicy struct {
|
||||
ConcurrencyLimit int
|
||||
QueueCapacity int
|
||||
}
|
||||
|
||||
// ExecutionProfile describes how and where to execute a model.
|
||||
type ExecutionProfile struct {
|
||||
ID string `yaml:"id"`
|
||||
BackendID string `yaml:"backend"`
|
||||
Endpoint string `yaml:"endpoint"`
|
||||
Model string `yaml:"model"`
|
||||
Temperature float64 `yaml:"temperature"`
|
||||
@@ -183,7 +215,7 @@ type ExecutionTargetOverride struct {
|
||||
TopP *float64 `json:"top_p,omitempty"`
|
||||
TimeoutSeconds *int `json:"timeout_seconds,omitempty"`
|
||||
ServiceTier string `json:"service_tier,omitempty"`
|
||||
ReasoningEffort string `json:"reasoning_effort,omitempty"`
|
||||
ReasoningEffort *string `json:"reasoning_effort,omitempty"`
|
||||
APIKeyEnv string `json:"api_key_env,omitempty"`
|
||||
ExtraParams map[string]any `json:"extra_params,omitempty"`
|
||||
}
|
||||
@@ -199,6 +231,7 @@ type ExecutionTargetPresence struct {
|
||||
|
||||
// ExecutionTarget represents effective model runtime settings for a run.
|
||||
type ExecutionTarget struct {
|
||||
BackendID string `yaml:"backend" json:"backend_id,omitempty"`
|
||||
Endpoint string `yaml:"endpoint" json:"endpoint"`
|
||||
Model string `yaml:"model" json:"model"`
|
||||
Temperature float64 `yaml:"temperature" json:"temperature"`
|
||||
@@ -213,6 +246,13 @@ type ExecutionTarget struct {
|
||||
ExtraParams map[string]any `yaml:"extra_params" json:"extra_params"`
|
||||
}
|
||||
|
||||
// ProfileInspection is the resolved result of exact profile inspection.
|
||||
type ProfileInspection struct {
|
||||
ProfileID string
|
||||
EffectiveModelParams ExecutionTarget
|
||||
APIKeyRequired bool
|
||||
}
|
||||
|
||||
// OutputContract defines the requirements for the output artifact.
|
||||
type OutputContract struct {
|
||||
Format OutputFormat `yaml:"format"`
|
||||
|
||||
19
internal/domain/session.go
Normal file
19
internal/domain/session.go
Normal file
@@ -0,0 +1,19 @@
|
||||
package domain
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
// NormalizeSessionID applies the shared session identifier rule.
|
||||
func NormalizeSessionID(raw string) (string, error) {
|
||||
normalized := strings.TrimSpace(raw)
|
||||
if normalized == "" {
|
||||
return "", nil
|
||||
}
|
||||
if length := utf8.RuneCountInString(normalized); length > SessionIDMaxLength {
|
||||
return "", fmt.Errorf("session_id length %d exceeds maximum %d", length, SessionIDMaxLength)
|
||||
}
|
||||
return normalized, nil
|
||||
}
|
||||
57
internal/domain/session_test.go
Normal file
57
internal/domain/session_test.go
Normal file
@@ -0,0 +1,57 @@
|
||||
package domain
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestNormalizeSessionID(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
raw string
|
||||
want string
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "trims surrounding Unicode whitespace",
|
||||
raw: "\u2003 session-123 \u2003",
|
||||
want: "session-123",
|
||||
},
|
||||
{
|
||||
name: "blank input is omitted",
|
||||
raw: " \t\u2003 ",
|
||||
want: "",
|
||||
},
|
||||
{
|
||||
name: "maximum Unicode length is accepted",
|
||||
raw: strings.Repeat("界", SessionIDMaxLength),
|
||||
want: strings.Repeat("界", SessionIDMaxLength),
|
||||
},
|
||||
{
|
||||
name: "one Unicode code point over maximum is rejected",
|
||||
raw: strings.Repeat("界", SessionIDMaxLength+1),
|
||||
wantErr: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := NormalizeSessionID(tt.raw)
|
||||
if tt.wantErr {
|
||||
if err == nil {
|
||||
t.Fatal("expected normalization error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "exceeds maximum") {
|
||||
t.Fatalf("expected useful length diagnostic, got %v", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("normalize session id: %v", err)
|
||||
}
|
||||
if got != tt.want {
|
||||
t.Fatalf("normalized session id = %q, want %q", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,25 +1,36 @@
|
||||
package promptkit
|
||||
// Package jsonvalue validates and defensively copies JSON-compatible value
|
||||
// trees used by configuration, request, and prepared-state boundaries.
|
||||
package jsonvalue
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"math"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
const maxSafeJSONInteger = 1<<53 - 1
|
||||
|
||||
type jsonVisit struct {
|
||||
type visit struct {
|
||||
typ reflect.Type
|
||||
ptr uintptr
|
||||
}
|
||||
|
||||
func copyPublicJSONMap(src map[string]any) (map[string]any, error) {
|
||||
// Copy validates and deeply copies a JSON-compatible value while preserving
|
||||
// compatible concrete map, slice, array, scalar, and number types.
|
||||
func Copy(src any) (any, error) {
|
||||
return copyValue(reflect.ValueOf(src), "value", make(map[visit]struct{}), true)
|
||||
}
|
||||
|
||||
// CopyMap validates and deeply copies an extra-parameter map while preserving
|
||||
// compatible concrete map, slice, array, scalar, and number types.
|
||||
func CopyMap(src map[string]any) (map[string]any, error) {
|
||||
if src == nil {
|
||||
return nil, nil
|
||||
}
|
||||
copied, err := copyPublicJSONValue(reflect.ValueOf(src), "extra_params", make(map[jsonVisit]struct{}))
|
||||
copied, err := copyValue(reflect.ValueOf(src), "extra_params", make(map[visit]struct{}), false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -30,7 +41,12 @@ func copyPublicJSONMap(src map[string]any) (map[string]any, error) {
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func copyPublicJSONValue(value reflect.Value, path string, seen map[jsonVisit]struct{}) (any, error) {
|
||||
func copyValue(
|
||||
value reflect.Value,
|
||||
path string,
|
||||
seen map[visit]struct{},
|
||||
allowEmptyMapKeys bool,
|
||||
) (any, error) {
|
||||
if !value.IsValid() {
|
||||
return nil, nil
|
||||
}
|
||||
@@ -38,12 +54,15 @@ func copyPublicJSONValue(value reflect.Value, path string, seen map[jsonVisit]st
|
||||
if value.IsNil() {
|
||||
return nil, nil
|
||||
}
|
||||
return copyPublicJSONValue(value.Elem(), path, seen)
|
||||
return copyValue(value.Elem(), path, seen, allowEmptyMapKeys)
|
||||
}
|
||||
if !value.CanInterface() {
|
||||
return nil, fmt.Errorf("%s: value cannot be copied", path)
|
||||
}
|
||||
if number, ok := value.Interface().(json.Number); ok {
|
||||
if _, err := json.Marshal(number); err != nil {
|
||||
return nil, fmt.Errorf("%s: invalid JSON number", path)
|
||||
}
|
||||
f, err := strconv.ParseFloat(number.String(), 64)
|
||||
if err != nil || math.IsNaN(f) || math.IsInf(f, 0) {
|
||||
return nil, fmt.Errorf("%s: invalid JSON number", path)
|
||||
@@ -65,8 +84,8 @@ func copyPublicJSONValue(value reflect.Value, path string, seen map[jsonVisit]st
|
||||
}
|
||||
return value.Interface(), nil
|
||||
case reflect.Float32, reflect.Float64:
|
||||
f := value.Convert(reflect.TypeOf(float64(0))).Float()
|
||||
if math.IsNaN(f) || math.IsInf(f, 0) {
|
||||
number := value.Convert(reflect.TypeOf(float64(0))).Float()
|
||||
if math.IsNaN(number) || math.IsInf(number, 0) {
|
||||
return nil, fmt.Errorf("%s: floating-point value must be finite", path)
|
||||
}
|
||||
return value.Interface(), nil
|
||||
@@ -74,28 +93,33 @@ func copyPublicJSONValue(value reflect.Value, path string, seen map[jsonVisit]st
|
||||
if value.IsNil() {
|
||||
return nil, nil
|
||||
}
|
||||
visit := jsonVisit{typ: value.Type(), ptr: value.Pointer()}
|
||||
if _, ok := seen[visit]; ok {
|
||||
current := visit{typ: value.Type(), ptr: value.Pointer()}
|
||||
if _, ok := seen[current]; ok {
|
||||
return nil, fmt.Errorf("%s: cyclic value is not supported", path)
|
||||
}
|
||||
seen[visit] = struct{}{}
|
||||
defer delete(seen, visit)
|
||||
return copyPublicJSONValue(value.Elem(), path, seen)
|
||||
seen[current] = struct{}{}
|
||||
defer delete(seen, current)
|
||||
return copyValue(value.Elem(), path, seen, allowEmptyMapKeys)
|
||||
case reflect.Map:
|
||||
return copyPublicJSONMapValue(value, path, seen)
|
||||
return copyMapValue(value, path, seen, allowEmptyMapKeys)
|
||||
case reflect.Slice:
|
||||
if value.IsNil() {
|
||||
return nil, nil
|
||||
}
|
||||
return copyPublicJSONSequenceValue(value, path, seen)
|
||||
return copySequenceValue(value, path, seen, allowEmptyMapKeys)
|
||||
case reflect.Array:
|
||||
return copyPublicJSONSequenceValue(value, path, seen)
|
||||
return copySequenceValue(value, path, seen, allowEmptyMapKeys)
|
||||
default:
|
||||
return nil, fmt.Errorf("%s: unsupported JSON value type %s", path, value.Type())
|
||||
}
|
||||
}
|
||||
|
||||
func copyPublicJSONMapValue(value reflect.Value, path string, seen map[jsonVisit]struct{}) (any, error) {
|
||||
func copyMapValue(
|
||||
value reflect.Value,
|
||||
path string,
|
||||
seen map[visit]struct{},
|
||||
allowEmptyMapKeys bool,
|
||||
) (any, error) {
|
||||
if value.IsNil() {
|
||||
return nil, nil
|
||||
}
|
||||
@@ -103,37 +127,43 @@ func copyPublicJSONMapValue(value reflect.Value, path string, seen map[jsonVisit
|
||||
return nil, fmt.Errorf("%s: map key type %s is not supported", path, value.Type().Key())
|
||||
}
|
||||
|
||||
visit := jsonVisit{typ: value.Type(), ptr: value.Pointer()}
|
||||
if _, ok := seen[visit]; ok {
|
||||
current := visit{typ: value.Type(), ptr: value.Pointer()}
|
||||
if _, ok := seen[current]; ok {
|
||||
return nil, fmt.Errorf("%s: cyclic value is not supported", path)
|
||||
}
|
||||
seen[visit] = struct{}{}
|
||||
defer delete(seen, visit)
|
||||
seen[current] = struct{}{}
|
||||
defer delete(seen, current)
|
||||
|
||||
keys := value.MapKeys()
|
||||
sort.Slice(keys, func(i, j int) bool {
|
||||
return keys[i].String() < keys[j].String()
|
||||
})
|
||||
|
||||
type entry struct {
|
||||
key reflect.Value
|
||||
name string
|
||||
value any
|
||||
}
|
||||
entries := make([]entry, 0, value.Len())
|
||||
entries := make([]entry, 0, len(keys))
|
||||
preserveType := true
|
||||
elemType := value.Type().Elem()
|
||||
iter := value.MapRange()
|
||||
for iter.Next() {
|
||||
key := iter.Key()
|
||||
elementType := value.Type().Elem()
|
||||
for _, key := range keys {
|
||||
name := key.String()
|
||||
copied, err := copyPublicJSONValue(iter.Value(), path+"."+name, seen)
|
||||
if name == "" && !allowEmptyMapKeys {
|
||||
return nil, fmt.Errorf("%s: map key must not be empty", path)
|
||||
}
|
||||
copied, err := copyValue(value.MapIndex(key), path+"."+name, seen, allowEmptyMapKeys)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
entries = append(entries, entry{key: key, name: name, value: copied})
|
||||
if copied == nil {
|
||||
if !canAssignNil(elemType) {
|
||||
if !canAssignNil(elementType) {
|
||||
preserveType = false
|
||||
}
|
||||
continue
|
||||
}
|
||||
if !reflect.TypeOf(copied).AssignableTo(elemType) {
|
||||
if !reflect.TypeOf(copied).AssignableTo(elementType) {
|
||||
preserveType = false
|
||||
}
|
||||
}
|
||||
@@ -142,7 +172,7 @@ func copyPublicJSONMapValue(value reflect.Value, path string, seen map[jsonVisit
|
||||
out := reflect.MakeMapWithSize(value.Type(), len(entries))
|
||||
for _, entry := range entries {
|
||||
if entry.value == nil {
|
||||
out.SetMapIndex(entry.key, reflect.Zero(elemType))
|
||||
out.SetMapIndex(entry.key, reflect.Zero(elementType))
|
||||
continue
|
||||
}
|
||||
out.SetMapIndex(entry.key, reflect.ValueOf(entry.value))
|
||||
@@ -157,33 +187,43 @@ func copyPublicJSONMapValue(value reflect.Value, path string, seen map[jsonVisit
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func copyPublicJSONSequenceValue(value reflect.Value, path string, seen map[jsonVisit]struct{}) (any, error) {
|
||||
var visit jsonVisit
|
||||
func copySequenceValue(
|
||||
value reflect.Value,
|
||||
path string,
|
||||
seen map[visit]struct{},
|
||||
allowEmptyMapKeys bool,
|
||||
) (any, error) {
|
||||
var current visit
|
||||
if value.Kind() == reflect.Slice {
|
||||
visit = jsonVisit{typ: value.Type(), ptr: value.Pointer()}
|
||||
if _, ok := seen[visit]; ok {
|
||||
current = visit{typ: value.Type(), ptr: value.Pointer()}
|
||||
if _, ok := seen[current]; ok {
|
||||
return nil, fmt.Errorf("%s: cyclic value is not supported", path)
|
||||
}
|
||||
seen[visit] = struct{}{}
|
||||
defer delete(seen, visit)
|
||||
seen[current] = struct{}{}
|
||||
defer delete(seen, current)
|
||||
}
|
||||
|
||||
values := make([]any, value.Len())
|
||||
preserveType := true
|
||||
elemType := value.Type().Elem()
|
||||
elementType := value.Type().Elem()
|
||||
for i := 0; i < value.Len(); i++ {
|
||||
copied, err := copyPublicJSONValue(value.Index(i), fmt.Sprintf("%s[%d]", path, i), seen)
|
||||
copied, err := copyValue(
|
||||
value.Index(i),
|
||||
fmt.Sprintf("%s[%d]", path, i),
|
||||
seen,
|
||||
allowEmptyMapKeys,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
values[i] = copied
|
||||
if copied == nil {
|
||||
if !canAssignNil(elemType) {
|
||||
if !canAssignNil(elementType) {
|
||||
preserveType = false
|
||||
}
|
||||
continue
|
||||
}
|
||||
if !reflect.TypeOf(copied).AssignableTo(elemType) {
|
||||
if !reflect.TypeOf(copied).AssignableTo(elementType) {
|
||||
preserveType = false
|
||||
}
|
||||
}
|
||||
@@ -195,7 +235,7 @@ func copyPublicJSONSequenceValue(value reflect.Value, path string, seen map[json
|
||||
}
|
||||
for i, copied := range values {
|
||||
if copied == nil {
|
||||
out.Index(i).Set(reflect.Zero(elemType))
|
||||
out.Index(i).Set(reflect.Zero(elementType))
|
||||
continue
|
||||
}
|
||||
out.Index(i).Set(reflect.ValueOf(copied))
|
||||
112
internal/jsonvalue/jsonvalue_test.go
Normal file
112
internal/jsonvalue/jsonvalue_test.go
Normal file
@@ -0,0 +1,112 @@
|
||||
package jsonvalue_test
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"math"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/jsonvalue"
|
||||
)
|
||||
|
||||
func TestCopyMapPreservesTypesAndIsolatesMutations(t *testing.T) {
|
||||
nested := map[string]int{"limit": 2}
|
||||
sequence := []string{"one", "two"}
|
||||
input := map[string]any{
|
||||
"count": int64(7),
|
||||
"number": json.Number("-1.25e+2"),
|
||||
"nested": nested,
|
||||
"sequence": sequence,
|
||||
}
|
||||
|
||||
copied, err := jsonvalue.CopyMap(input)
|
||||
if err != nil {
|
||||
t.Fatalf("copy map: %v", err)
|
||||
}
|
||||
nested["limit"] = 99
|
||||
sequence[0] = "changed"
|
||||
input["added"] = true
|
||||
|
||||
if got, ok := copied["count"].(int64); !ok || got != 7 {
|
||||
t.Fatalf("integer type or value changed: %#v", copied["count"])
|
||||
}
|
||||
if got, ok := copied["number"].(json.Number); !ok || got != "-1.25e+2" {
|
||||
t.Fatalf("JSON number type or value changed: %#v", copied["number"])
|
||||
}
|
||||
if got := copied["nested"].(map[string]int)["limit"]; got != 2 {
|
||||
t.Fatalf("nested map was not isolated: %d", got)
|
||||
}
|
||||
if got := copied["sequence"].([]string)[0]; got != "one" {
|
||||
t.Fatalf("sequence was not isolated: %q", got)
|
||||
}
|
||||
if _, ok := copied["added"]; ok {
|
||||
t.Fatalf("top-level map was not isolated: %#v", copied)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCopyAllowsEmptyObjectKeysAndIsolatesMutations(t *testing.T) {
|
||||
nested := map[string]any{"": []any{"original"}}
|
||||
|
||||
copiedValue, err := jsonvalue.Copy(nested)
|
||||
if err != nil {
|
||||
t.Fatalf("copy value: %v", err)
|
||||
}
|
||||
nested[""].([]any)[0] = "changed"
|
||||
|
||||
copied := copiedValue.(map[string]any)
|
||||
if got := copied[""].([]any)[0]; got != "original" {
|
||||
t.Fatalf("copied value was not isolated: %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCopyMapRejectsInvalidValues(t *testing.T) {
|
||||
cyclicMap := map[string]any{}
|
||||
cyclicMap["self"] = cyclicMap
|
||||
cyclicSlice := []any{nil}
|
||||
cyclicSlice[0] = cyclicSlice
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
value any
|
||||
}{
|
||||
{name: "empty nested key", value: map[string]int{"": 1}},
|
||||
{name: "non-string map key", value: map[int]string{1: "one"}},
|
||||
{name: "unsupported value", value: make(chan int)},
|
||||
{name: "cyclic map", value: cyclicMap},
|
||||
{name: "cyclic slice", value: cyclicSlice},
|
||||
{name: "NaN", value: math.NaN()},
|
||||
{name: "positive infinity", value: math.Inf(1)},
|
||||
{name: "unsafe signed integer", value: int64(1 << 53)},
|
||||
{name: "unsafe unsigned integer", value: uint64(1 << 53)},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if _, err := jsonvalue.CopyMap(map[string]any{"value": tc.value}); err == nil {
|
||||
t.Fatal("expected validation error")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCopyMapValidatesJSONNumberSyntaxAndRange(t *testing.T) {
|
||||
for _, number := range []json.Number{"0", "-1", "1.25", "-1.25e+2"} {
|
||||
t.Run("valid "+number.String(), func(t *testing.T) {
|
||||
got, err := jsonvalue.CopyMap(map[string]any{"value": number})
|
||||
if err != nil {
|
||||
t.Fatalf("copy valid JSON number: %v", err)
|
||||
}
|
||||
if !reflect.DeepEqual(got["value"], number) {
|
||||
t.Fatalf("JSON number changed: got %#v want %#v", got["value"], number)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
for _, number := range []json.Number{"", "01", "+1", "1.", ".1", "1e9999", "not-a-number"} {
|
||||
t.Run("invalid "+number.String(), func(t *testing.T) {
|
||||
if _, err := jsonvalue.CopyMap(map[string]any{"value": number}); err == nil {
|
||||
t.Fatal("expected invalid JSON number error")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -12,7 +12,6 @@ import (
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/defaults"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
@@ -177,12 +176,11 @@ func openAIChatRequestFromGenerateRequest(req domain.GenerateRequest, defaultMod
|
||||
wireReq := openAIChatRequest{
|
||||
Model: model,
|
||||
}
|
||||
if sessionID := strings.TrimSpace(req.Prompt.SessionID); sessionID != "" {
|
||||
if n := utf8.RuneCountInString(sessionID); n > domain.SessionIDMaxLength {
|
||||
return openAIChatRequest{}, fmt.Errorf("session_id length %d exceeds maximum %d", n, domain.SessionIDMaxLength)
|
||||
}
|
||||
wireReq.SessionID = sessionID
|
||||
sessionID, err := domain.NormalizeSessionID(req.Prompt.SessionID)
|
||||
if err != nil {
|
||||
return openAIChatRequest{}, err
|
||||
}
|
||||
wireReq.SessionID = sessionID
|
||||
|
||||
wireReq.Messages = make([]openAIChatRequestMessage, 0, len(req.Prompt.Messages))
|
||||
for _, msg := range req.Prompt.Messages {
|
||||
@@ -262,7 +260,7 @@ func openAIChatRequestPayload(req openAIChatRequest) (map[string]any, error) {
|
||||
if key == "" {
|
||||
return nil, errors.New("extra_params key must not be empty")
|
||||
}
|
||||
if _, reserved := reservedOpenAIChatRequestFields[key]; reserved {
|
||||
if IsReservedOpenAIChatRequestField(key) {
|
||||
return nil, fmt.Errorf("extra_params key %q collides with reserved request field", key)
|
||||
}
|
||||
if _, err := json.Marshal(value); err != nil {
|
||||
@@ -274,16 +272,23 @@ func openAIChatRequestPayload(req openAIChatRequest) (map[string]any, error) {
|
||||
return out, nil
|
||||
}
|
||||
|
||||
var reservedOpenAIChatRequestFields = map[string]struct{}{
|
||||
"model": {},
|
||||
"session_id": {},
|
||||
"messages": {},
|
||||
"temperature": {},
|
||||
"max_tokens": {},
|
||||
"top_p": {},
|
||||
"service_tier": {},
|
||||
"reasoning_effort": {},
|
||||
"response_format": {},
|
||||
// IsReservedOpenAIChatRequestField reports whether name is owned by the
|
||||
// standard OpenAI-compatible chat request rather than extra parameters.
|
||||
func IsReservedOpenAIChatRequestField(name string) bool {
|
||||
switch name {
|
||||
case "model",
|
||||
"session_id",
|
||||
"messages",
|
||||
"temperature",
|
||||
"max_tokens",
|
||||
"top_p",
|
||||
"service_tier",
|
||||
"reasoning_effort",
|
||||
"response_format":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
type openAIChatRequestMessage struct {
|
||||
|
||||
@@ -1,9 +1,8 @@
|
||||
id: aion-2
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
backend: openrouter
|
||||
model: aion-labs/aion-2.0
|
||||
temperature: 0.72
|
||||
reasoning_effort: high
|
||||
top_p: 0.95
|
||||
timeout_seconds: 180
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
service_tier: flex
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
id: claude-fable-latest
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
backend: openrouter
|
||||
model: "~anthropic/claude-fable-latest"
|
||||
reasoning_effort: high
|
||||
timeout_seconds: 600
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
service_tier: flex
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
id: claude-haiku-latest
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
backend: openrouter
|
||||
model: "~anthropic/claude-haiku-latest"
|
||||
reasoning_effort: medium
|
||||
timeout_seconds: 240
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
service_tier: flex
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
id: claude-opus-latest
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
backend: openrouter
|
||||
model: "~anthropic/claude-opus-latest"
|
||||
reasoning_effort: high
|
||||
timeout_seconds: 240
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
service_tier: flex
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
id: claude-sonnet-latest
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
backend: openrouter
|
||||
model: "~anthropic/claude-sonnet-latest"
|
||||
reasoning_effort: high
|
||||
timeout_seconds: 240
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
service_tier: flex
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
id: deepseek-3-2
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
backend: openrouter
|
||||
model: deepseek/deepseek-v3.2
|
||||
reasoning_effort: high
|
||||
timeout_seconds: 180
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
service_tier: flex
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
id: deepseek-4-flash
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
backend: openrouter
|
||||
model: deepseek/deepseek-v4-flash
|
||||
#reasoning_effort: medium
|
||||
timeout_seconds: 180
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
service_tier: flex
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
id: deepseek-4-pro
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
backend: openrouter
|
||||
model: deepseek/deepseek-v4-pro
|
||||
reasoning_effort: high
|
||||
timeout_seconds: 180
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
service_tier: flex
|
||||
|
||||
@@ -1,9 +1,8 @@
|
||||
id: gemini-2-flash-lite
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
backend: openrouter
|
||||
model: "google/gemini-2.5-flash-lite"
|
||||
#temperature: 0.15
|
||||
reasoning_effort: high
|
||||
#top_p: 0.98
|
||||
timeout_seconds: 240
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
service_tier: flex
|
||||
|
||||
@@ -1,9 +1,8 @@
|
||||
id: gemini-2-flash
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
backend: openrouter
|
||||
model: "google/gemini-2.5-flash"
|
||||
#temperature: 0.15
|
||||
reasoning_effort: high
|
||||
#top_p: 0.98
|
||||
timeout_seconds: 240
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
service_tier: flex
|
||||
|
||||
@@ -1,9 +1,8 @@
|
||||
id: gemini-2-pro
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
backend: openrouter
|
||||
model: "google/gemini-2.5-pro"
|
||||
#temperature: 0.15
|
||||
reasoning_effort: high
|
||||
#top_p: 0.98
|
||||
timeout_seconds: 240
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
service_tier: flex
|
||||
|
||||
@@ -1,9 +1,8 @@
|
||||
id: gemini-3-flash-lite
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
backend: openrouter
|
||||
model: "google/gemini-3.1-flash-lite"
|
||||
#temperature: 0.15
|
||||
reasoning_effort: high
|
||||
#top_p: 0.98
|
||||
timeout_seconds: 240
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
service_tier: flex
|
||||
|
||||
@@ -1,9 +1,8 @@
|
||||
id: gemini-flash-latest
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
backend: openrouter
|
||||
model: "~google/gemini-flash-latest"
|
||||
#temperature: 0.15
|
||||
reasoning_effort: high
|
||||
#top_p: 0.98
|
||||
timeout_seconds: 240
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
service_tier: flex
|
||||
|
||||
@@ -1,9 +1,8 @@
|
||||
id: gemini-pro-latest
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
backend: openrouter
|
||||
model: "~google/gemini-pro-latest"
|
||||
#temperature: 0.15
|
||||
reasoning_effort: high
|
||||
#top_p: 0.98
|
||||
timeout_seconds: 240
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
service_tier: flex
|
||||
|
||||
@@ -1,9 +1,8 @@
|
||||
id: gemma-4-31b
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
backend: openrouter
|
||||
model: google/gemma-4-31b-it:exacto
|
||||
temperature: 0.15
|
||||
reasoning_effort: high
|
||||
top_p: 0.98
|
||||
timeout_seconds: 240
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
service_tier: flex
|
||||
|
||||
@@ -1,9 +1,8 @@
|
||||
id: minimax-m2
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
backend: openrouter
|
||||
model: minimax/minimax-m2.5
|
||||
temperature: 0.5
|
||||
reasoning_effort: high
|
||||
top_p: 0.95
|
||||
timeout_seconds: 180
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
service_tier: flex
|
||||
|
||||
@@ -1,9 +1,8 @@
|
||||
id: minimax-m3
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
backend: openrouter
|
||||
model: minimax/minimax-m3
|
||||
#temperature: 0.5
|
||||
reasoning_effort: high
|
||||
#top_p: 0.95
|
||||
timeout_seconds: 180
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
service_tier: flex
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
id: mistral-large-2512
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
backend: openrouter
|
||||
model: mistralai/mistral-large-2512
|
||||
temperature: 0.15
|
||||
top_p: 0.98
|
||||
timeout_seconds: 180
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
|
||||
@@ -1,8 +1,7 @@
|
||||
id: mistral-medium-3-5
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
backend: openrouter
|
||||
model: mistralai/mistral-medium-3-5
|
||||
temperature: 0.15
|
||||
reasoning_effort: high
|
||||
top_p: 0.98
|
||||
timeout_seconds: 180
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
id: mistral-small-3
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
backend: openrouter
|
||||
model: mistralai/mistral-small-3.2-24b-instruct
|
||||
temperature: 0.05
|
||||
top_p: 1.0
|
||||
timeout_seconds: 180
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
|
||||
@@ -1,8 +1,7 @@
|
||||
id: mistral-small-4
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
backend: openrouter
|
||||
model: mistralai/mistral-small-2603
|
||||
temperature: 0.1
|
||||
reasoning_effort: high
|
||||
top_p: 0.98
|
||||
timeout_seconds: 180
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
id: nemotron-3-ultra
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
backend: openrouter
|
||||
model: nvidia/nemotron-3-ultra-550b-a55b
|
||||
reasoning_effort: high
|
||||
timeout_seconds: 180
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
service_tier: flex
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
id: gpt-5-mini
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
backend: openrouter
|
||||
model: "openai/gpt-5.4-mini"
|
||||
reasoning_effort: high
|
||||
timeout_seconds: 240
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
service_tier: flex
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
id: gpt-5-nano
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
backend: openrouter
|
||||
model: "openai/gpt-5.4-nano"
|
||||
reasoning_effort: high
|
||||
timeout_seconds: 240
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
service_tier: flex
|
||||
|
||||
@@ -2,7 +2,6 @@ package builtin
|
||||
|
||||
import (
|
||||
"embed"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/profile"
|
||||
)
|
||||
@@ -15,17 +14,3 @@ var assets embed.FS
|
||||
func NewRepository() profile.Repository {
|
||||
return profile.NewFSRepository(assets, assetRoot)
|
||||
}
|
||||
|
||||
func NewRepositoryWithPrimary(primary profile.Repository) profile.Repository {
|
||||
if primary == nil {
|
||||
return NewRepository()
|
||||
}
|
||||
return profile.NewOverlayRepository(primary, NewRepository())
|
||||
}
|
||||
|
||||
func NewRepositoryWithDirectory(dir string) profile.Repository {
|
||||
if strings.TrimSpace(dir) == "" {
|
||||
return NewRepository()
|
||||
}
|
||||
return NewRepositoryWithPrimary(profile.NewFilesystemRepository(dir))
|
||||
}
|
||||
|
||||
@@ -2,13 +2,11 @@ package builtin
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io/fs"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/profile"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/backend"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
@@ -28,6 +26,12 @@ func TestBuiltInProfilesValidateThroughRepository(t *testing.T) {
|
||||
if p.ID != id {
|
||||
t.Fatalf("expected profile id %q, got %q", id, p.ID)
|
||||
}
|
||||
if p.BackendID != backend.OpenRouterID {
|
||||
t.Fatalf("expected profile %q to select %q, got %q", id, backend.OpenRouterID, p.BackendID)
|
||||
}
|
||||
if p.Endpoint != "" || p.APIKeyEnv != "" {
|
||||
t.Fatalf("expected profile %q to inherit backend connection settings, got endpoint=%q api_key_env=%q", id, p.Endpoint, p.APIKeyEnv)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -60,6 +64,15 @@ func loadBuiltInProfileIDs(t *testing.T) map[string]string {
|
||||
if _, ok := raw["api_key"]; ok {
|
||||
t.Fatalf("built-in profile %s contains raw api_key", name)
|
||||
}
|
||||
if raw["backend"] != backend.OpenRouterID {
|
||||
t.Fatalf("built-in profile %s does not select %q", name, backend.OpenRouterID)
|
||||
}
|
||||
if _, ok := raw["endpoint"]; ok {
|
||||
t.Fatalf("built-in profile %s repeats endpoint", name)
|
||||
}
|
||||
if _, ok := raw["api_key_env"]; ok {
|
||||
t.Fatalf("built-in profile %s repeats api_key_env", name)
|
||||
}
|
||||
id, ok := raw["id"].(string)
|
||||
if !ok || strings.TrimSpace(id) == "" {
|
||||
t.Fatalf("built-in profile %s has missing id", name)
|
||||
@@ -75,53 +88,3 @@ func loadBuiltInProfileIDs(t *testing.T) map[string]string {
|
||||
}
|
||||
return ids
|
||||
}
|
||||
|
||||
func TestRepositoryWithPrimaryUsesPrimaryBeforeBuiltIns(t *testing.T) {
|
||||
repo := NewRepositoryWithPrimary(staticProfileRepo{
|
||||
profiles: map[string]string{"mistral-small-3": "custom-model"},
|
||||
})
|
||||
|
||||
p, err := repo.GetProfile(context.Background(), "mistral-small-3")
|
||||
if err != nil {
|
||||
t.Fatalf("expected profile to load, got %v", err)
|
||||
}
|
||||
if p.Model != "custom-model" {
|
||||
t.Fatalf("expected primary profile to override built-in, got %+v", p)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRepositoryWithPrimaryFallsBackToBuiltIns(t *testing.T) {
|
||||
repo := NewRepositoryWithPrimary(staticProfileRepo{})
|
||||
|
||||
p, err := repo.GetProfile(context.Background(), "mistral-small-3")
|
||||
if err != nil {
|
||||
t.Fatalf("expected built-in profile to load, got %v", err)
|
||||
}
|
||||
if p.ID != "mistral-small-3" {
|
||||
t.Fatalf("unexpected profile: %+v", p)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRepositoryWithPrimaryDoesNotFallBackAfterPrimaryError(t *testing.T) {
|
||||
repo := NewRepositoryWithPrimary(staticProfileRepo{err: profile.ErrInvalidProfile})
|
||||
|
||||
_, err := repo.GetProfile(context.Background(), "mistral-small-3")
|
||||
if !errors.Is(err, profile.ErrInvalidProfile) {
|
||||
t.Fatalf("expected primary error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
type staticProfileRepo struct {
|
||||
profiles map[string]string
|
||||
err error
|
||||
}
|
||||
|
||||
func (r staticProfileRepo) GetProfile(_ context.Context, id string) (*domain.ExecutionProfile, error) {
|
||||
if r.err != nil {
|
||||
return nil, r.err
|
||||
}
|
||||
if model, ok := r.profiles[id]; ok {
|
||||
return &domain.ExecutionProfile{ID: id, Endpoint: "http://primary/v1", Model: model}, nil
|
||||
}
|
||||
return nil, profile.ErrProfileNotFound
|
||||
}
|
||||
|
||||
@@ -121,6 +121,7 @@ func loadProfile(ctx context.Context, fsys fs.FS, root string, id string) (*doma
|
||||
if prof.ID != id {
|
||||
continue
|
||||
}
|
||||
prof.BackendID = strings.TrimSpace(prof.BackendID)
|
||||
if err := validateProfile(&prof); err != nil {
|
||||
if errors.Is(err, ErrRawAPIKeyNotAllowed) {
|
||||
return nil, fmt.Errorf("%w: %s", err, relPath)
|
||||
@@ -189,8 +190,8 @@ func validateProfile(p *domain.ExecutionProfile) error {
|
||||
if strings.TrimSpace(p.ID) == "" {
|
||||
return errors.New("id is required")
|
||||
}
|
||||
if strings.TrimSpace(p.Endpoint) == "" {
|
||||
return errors.New("endpoint is required")
|
||||
if strings.TrimSpace(p.BackendID) == "" && strings.TrimSpace(p.Endpoint) == "" {
|
||||
return errors.New("backend or endpoint is required")
|
||||
}
|
||||
if strings.TrimSpace(p.Model) == "" {
|
||||
return errors.New("model is required")
|
||||
|
||||
@@ -52,6 +52,43 @@ func TestFilesystemRepository_GetProfile(t *testing.T) {
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("backend and endpoint connection matrix", func(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
connection string
|
||||
wantBackend string
|
||||
wantEndpoint string
|
||||
wantErr bool
|
||||
}{
|
||||
{name: "backend only", connection: "backend: ' openrouter '", wantBackend: "openrouter"},
|
||||
{name: "endpoint only", connection: "endpoint: http://localhost:8000/v1", wantEndpoint: "http://localhost:8000/v1"},
|
||||
{name: "both", connection: "backend: openrouter\nendpoint: http://localhost:8000/v1", wantBackend: "openrouter", wantEndpoint: "http://localhost:8000/v1"},
|
||||
{name: "neither", wantErr: true},
|
||||
{name: "blank backend", connection: "backend: ' '", wantErr: true},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
id := "connection-" + strings.ReplaceAll(tt.name, " ", "-")
|
||||
writeProfileTestFile(t, filepath.Join(tmpDir, id+".yaml"), "id: "+id+"\nmodel: model\n"+tt.connection+"\n")
|
||||
|
||||
p, err := repo.GetProfile(ctx, id)
|
||||
if tt.wantErr {
|
||||
if !errors.Is(err, ErrInvalidProfile) {
|
||||
t.Fatalf("expected ErrInvalidProfile, got %v", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("expected profile to load, got %v", err)
|
||||
}
|
||||
if p.BackendID != tt.wantBackend || p.Endpoint != tt.wantEndpoint {
|
||||
t.Fatalf("unexpected connection values: backend=%q endpoint=%q", p.BackendID, p.Endpoint)
|
||||
}
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("valid profile with api_key_env", func(t *testing.T) {
|
||||
p, err := repo.GetProfile(ctx, "local-secure")
|
||||
if err != nil {
|
||||
|
||||
@@ -5,10 +5,9 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
"strings"
|
||||
"text/template"
|
||||
"unicode/utf8"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -95,10 +94,6 @@ func (r *goRenderer) Render(ctx context.Context, definition *domain.PromptDefini
|
||||
}
|
||||
|
||||
func renderSessionID(raw string, funcs template.FuncMap, vars map[string]string) (string, error) {
|
||||
if strings.TrimSpace(raw) == "" {
|
||||
return "", nil
|
||||
}
|
||||
|
||||
tmpl, err := template.New("session_id").Funcs(funcs).Option("missingkey=error").Parse(raw)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("%w: session_id: %v", ErrInvalidTemplate, err)
|
||||
@@ -109,9 +104,9 @@ func renderSessionID(raw string, funcs template.FuncMap, vars map[string]string)
|
||||
return "", fmt.Errorf("%w: session_id: %w", ErrRenderFailure, err)
|
||||
}
|
||||
|
||||
sessionID := strings.TrimSpace(buf.String())
|
||||
if n := utf8.RuneCountInString(sessionID); n > domain.SessionIDMaxLength {
|
||||
return "", fmt.Errorf("%w: session_id length %d exceeds maximum %d", ErrRenderFailure, n, domain.SessionIDMaxLength)
|
||||
sessionID, err := domain.NormalizeSessionID(buf.String())
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("%w: session_id: %v", ErrRenderFailure, err)
|
||||
}
|
||||
return sessionID, nil
|
||||
}
|
||||
|
||||
24
internal/usecase/capacity_error.go
Normal file
24
internal/usecase/capacity_error.go
Normal file
@@ -0,0 +1,24 @@
|
||||
package usecase
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/capacity"
|
||||
)
|
||||
|
||||
// CapacityError identifies bounded admission rejected for one selected backend.
|
||||
type CapacityError struct {
|
||||
BackendID string
|
||||
}
|
||||
|
||||
func (e *CapacityError) Error() string {
|
||||
if e == nil || strings.TrimSpace(e.BackendID) == "" {
|
||||
return capacity.ErrCapacityExceeded.Error()
|
||||
}
|
||||
return fmt.Sprintf("backend %q admission: %v", e.BackendID, capacity.ErrCapacityExceeded)
|
||||
}
|
||||
|
||||
func (e *CapacityError) Unwrap() error {
|
||||
return capacity.ErrCapacityExceeded
|
||||
}
|
||||
298
internal/usecase/prepared_execution.go
Normal file
298
internal/usecase/prepared_execution.go
Normal file
@@ -0,0 +1,298 @@
|
||||
package usecase
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/jsonvalue"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/validate"
|
||||
)
|
||||
|
||||
type preparedExecutionState uint8
|
||||
|
||||
const (
|
||||
preparedExecutionReady preparedExecutionState = iota
|
||||
preparedExecutionClaimed
|
||||
preparedExecutionDiscarded
|
||||
)
|
||||
|
||||
// PreparedExecution owns one frozen, single-use runner execution.
|
||||
type PreparedExecution struct {
|
||||
owner *Runner
|
||||
mu sync.Mutex
|
||||
state preparedExecutionState
|
||||
details *domain.PreparedRun
|
||||
payload *preparedExecutionPayload
|
||||
}
|
||||
|
||||
type preparedExecutionPayload struct {
|
||||
prepared *domain.PreparedRun
|
||||
validation validate.PreparedValidation
|
||||
directKey string
|
||||
}
|
||||
|
||||
// PrepareExecution completes preparation without generation or admission and
|
||||
// returns a runner-bound, single-use execution.
|
||||
func (r *Runner) PrepareExecution(ctx context.Context, req domain.RunRequest) (*PreparedExecution, error) {
|
||||
state, err := r.resolvePreparation(ctx, req, time.Now().UTC())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
validationPlan, err := r.prepareValidation(ctx, state.effectiveContract)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
structuredOutput, err := r.structuredOutputFromValidationPlan(
|
||||
state.definition,
|
||||
state.effectiveContract,
|
||||
validationPlan,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
prepared, err := r.completePreparationWithStructuredOutput(ctx, req, state, structuredOutput)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
executionSnapshot, err := clonePreparedRun(prepared)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: failed to copy prepared execution: %v", ErrInvalidRequest, err)
|
||||
}
|
||||
|
||||
details, err := clonePreparedRun(executionSnapshot)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: failed to copy prepared execution details: %v", ErrInvalidRequest, err)
|
||||
}
|
||||
|
||||
return &PreparedExecution{
|
||||
owner: r,
|
||||
state: preparedExecutionReady,
|
||||
details: details,
|
||||
payload: &preparedExecutionPayload{
|
||||
prepared: executionSnapshot,
|
||||
validation: validationPlan,
|
||||
directKey: state.effectiveModel.APIKey,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (r *Runner) prepareValidation(
|
||||
ctx context.Context,
|
||||
contract domain.OutputContract,
|
||||
) (validate.PreparedValidation, error) {
|
||||
if r.validator == nil {
|
||||
return noOpPreparedValidation{contract: contract}, nil
|
||||
}
|
||||
preparer, ok := r.validator.(validate.ValidationPreparer)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("%w: validator does not support prepared validation", ErrValidation)
|
||||
}
|
||||
plan, err := preparer.PrepareValidation(ctx, contract)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: %w", ErrValidation, err)
|
||||
}
|
||||
if plan == nil {
|
||||
return nil, fmt.Errorf("%w: validator returned nil prepared validation", ErrValidation)
|
||||
}
|
||||
return plan, nil
|
||||
}
|
||||
|
||||
func (r *Runner) structuredOutputFromValidationPlan(
|
||||
def *domain.PromptDefinition,
|
||||
contract domain.OutputContract,
|
||||
plan validate.PreparedValidation,
|
||||
) (*domain.StructuredOutputSpec, error) {
|
||||
if contract.ValidationMode != domain.ValidationJSONSchema {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
schemaDocument := plan.SchemaDocument()
|
||||
if schemaDocument == nil {
|
||||
if r.validator == nil {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, fmt.Errorf("%w: prepared json_schema validation has no schema document", ErrValidation)
|
||||
}
|
||||
return structuredOutputSpec(def, schemaDocument), nil
|
||||
}
|
||||
|
||||
// Details returns a fresh credential-redacted copy of the prepared run.
|
||||
func (p *PreparedExecution) Details() *domain.PreparedRun {
|
||||
if p == nil {
|
||||
return nil
|
||||
}
|
||||
p.mu.Lock()
|
||||
detailsSnapshot := p.details
|
||||
p.mu.Unlock()
|
||||
|
||||
details, err := clonePreparedRun(detailsSnapshot)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
return details
|
||||
}
|
||||
|
||||
// Discard invalidates an unclaimed execution and drops its private payload.
|
||||
func (p *PreparedExecution) Discard() {
|
||||
if p == nil {
|
||||
return
|
||||
}
|
||||
|
||||
p.mu.Lock()
|
||||
if p.state != preparedExecutionReady {
|
||||
p.mu.Unlock()
|
||||
return
|
||||
}
|
||||
p.state = preparedExecutionDiscarded
|
||||
payload := p.payload
|
||||
p.payload = nil
|
||||
p.mu.Unlock()
|
||||
|
||||
payload.clear()
|
||||
}
|
||||
|
||||
// RunPrepared claims and executes one prepared execution owned by this runner.
|
||||
func (r *Runner) RunPrepared(ctx context.Context, prepared *PreparedExecution) (*domain.RunResult, error) {
|
||||
payload, err := prepared.claim(r)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer payload.clear()
|
||||
|
||||
runID, err := newRunID()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create run id: %w", err)
|
||||
}
|
||||
start := time.Now().UTC()
|
||||
|
||||
target := payload.prepared.EffectiveModelParams
|
||||
if err := validateAPIKey(target.APIKeyEnv, payload.directKey, target.APIKeyRequired); err != nil {
|
||||
return nil, fmt.Errorf("%w: %w", ErrInvalidRequest, err)
|
||||
}
|
||||
|
||||
release, err := r.admitRun(ctx, target.BackendID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer release()
|
||||
|
||||
return r.executePreparedRun(ctx, payload.prepared, payload.directKey, runID, start, func(
|
||||
ctx context.Context,
|
||||
artifact *domain.Artifact,
|
||||
attemptsUsed int,
|
||||
) (domain.ValidationResult, error) {
|
||||
result, validationErr := payload.validation.Validate(ctx, artifact)
|
||||
if validationErr != nil {
|
||||
return domain.ValidationResult{}, validationErr
|
||||
}
|
||||
result.RepairAttempts = attemptsUsed
|
||||
return result, nil
|
||||
})
|
||||
}
|
||||
|
||||
func (p *PreparedExecution) claim(owner *Runner) (*preparedExecutionPayload, error) {
|
||||
if p == nil || owner == nil || p.owner != owner {
|
||||
return nil, fmt.Errorf("%w: prepared execution does not belong to this runner", ErrInvalidRequest)
|
||||
}
|
||||
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
if p.state != preparedExecutionReady || p.payload == nil {
|
||||
return nil, fmt.Errorf("%w: prepared execution is not ready", ErrInvalidRequest)
|
||||
}
|
||||
p.state = preparedExecutionClaimed
|
||||
payload := p.payload
|
||||
p.payload = nil
|
||||
return payload, nil
|
||||
}
|
||||
|
||||
func (p *preparedExecutionPayload) clear() {
|
||||
if p == nil {
|
||||
return
|
||||
}
|
||||
if p.prepared != nil {
|
||||
p.prepared.EffectiveModelParams.APIKey = ""
|
||||
}
|
||||
p.prepared = nil
|
||||
p.validation = nil
|
||||
p.directKey = ""
|
||||
}
|
||||
|
||||
type noOpPreparedValidation struct {
|
||||
contract domain.OutputContract
|
||||
}
|
||||
|
||||
func (p noOpPreparedValidation) Validate(
|
||||
ctx context.Context,
|
||||
_ *domain.Artifact,
|
||||
) (domain.ValidationResult, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return domain.ValidationResult{}, err
|
||||
}
|
||||
return domain.ValidationResult{
|
||||
Status: domain.ValidationSkipped,
|
||||
Mode: p.contract.ValidationMode,
|
||||
SchemaPath: p.contract.SchemaPath,
|
||||
RepairAttempts: p.contract.RepairAttempts,
|
||||
IsValid: true,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (noOpPreparedValidation) SchemaDocument() any {
|
||||
return nil
|
||||
}
|
||||
|
||||
func clonePreparedRun(source *domain.PreparedRun) (*domain.PreparedRun, error) {
|
||||
if source == nil {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
copied := *source
|
||||
extraParams, err := jsonvalue.CopyMap(source.EffectiveModelParams.ExtraParams)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
copied.EffectiveModelParams.ExtraParams = extraParams
|
||||
|
||||
if source.InputHashes != nil {
|
||||
copied.InputHashes = make(map[string]string, len(source.InputHashes))
|
||||
for name, hash := range source.InputHashes {
|
||||
copied.InputHashes[name] = hash
|
||||
}
|
||||
}
|
||||
if source.Messages != nil {
|
||||
copied.Messages = make([]domain.RenderedMessage, len(source.Messages))
|
||||
for i, message := range source.Messages {
|
||||
copied.Messages[i] = message
|
||||
if message.CacheControl != nil {
|
||||
cacheControl := *message.CacheControl
|
||||
copied.Messages[i].CacheControl = &cacheControl
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if source.StructuredOutput != nil {
|
||||
structuredOutput := *source.StructuredOutput
|
||||
copied.StructuredOutput = &structuredOutput
|
||||
if source.StructuredOutput.JSONSchema != nil {
|
||||
jsonSchema := *source.StructuredOutput.JSONSchema
|
||||
copied.StructuredOutput.JSONSchema = &jsonSchema
|
||||
schema, copyErr := cloneJSONValue(source.StructuredOutput.JSONSchema.Schema)
|
||||
if copyErr != nil {
|
||||
return nil, copyErr
|
||||
}
|
||||
copied.StructuredOutput.JSONSchema.Schema = schema
|
||||
}
|
||||
}
|
||||
return &copied, nil
|
||||
}
|
||||
|
||||
func cloneJSONValue(source any) (any, error) {
|
||||
return jsonvalue.Copy(source)
|
||||
}
|
||||
510
internal/usecase/prepared_execution_test.go
Normal file
510
internal/usecase/prepared_execution_test.go
Normal file
@@ -0,0 +1,510 @@
|
||||
package usecase
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/validate"
|
||||
)
|
||||
|
||||
type recordingPreparedValidation struct {
|
||||
contract domain.OutputContract
|
||||
schemaDocument any
|
||||
results []domain.ValidationResult
|
||||
errs []error
|
||||
artifacts []string
|
||||
}
|
||||
|
||||
func (p *recordingPreparedValidation) Validate(
|
||||
_ context.Context,
|
||||
artifact *domain.Artifact,
|
||||
) (domain.ValidationResult, error) {
|
||||
p.artifacts = append(p.artifacts, string(artifact.Body))
|
||||
index := len(p.artifacts) - 1
|
||||
if index < len(p.errs) && p.errs[index] != nil {
|
||||
return domain.ValidationResult{}, p.errs[index]
|
||||
}
|
||||
if len(p.results) == 0 {
|
||||
return domain.ValidationResult{
|
||||
Status: domain.ValidationPassed,
|
||||
Mode: p.contract.ValidationMode,
|
||||
IsValid: true,
|
||||
}, nil
|
||||
}
|
||||
if index >= len(p.results) {
|
||||
index = len(p.results) - 1
|
||||
}
|
||||
return p.results[index], nil
|
||||
}
|
||||
|
||||
func (p *recordingPreparedValidation) SchemaDocument() any {
|
||||
return p.schemaDocument
|
||||
}
|
||||
|
||||
type recordingValidationPreparer struct {
|
||||
plan *recordingPreparedValidation
|
||||
prepareErr error
|
||||
prepareCalls int
|
||||
directValidateCalls int
|
||||
}
|
||||
|
||||
func (v *recordingValidationPreparer) Validate(
|
||||
context.Context,
|
||||
*domain.Artifact,
|
||||
domain.OutputContract,
|
||||
) (domain.ValidationResult, error) {
|
||||
v.directValidateCalls++
|
||||
return domain.ValidationResult{}, errors.New("live validation must not be used")
|
||||
}
|
||||
|
||||
func (v *recordingValidationPreparer) PrepareValidation(
|
||||
_ context.Context,
|
||||
contract domain.OutputContract,
|
||||
) (validate.PreparedValidation, error) {
|
||||
v.prepareCalls++
|
||||
if v.prepareErr != nil {
|
||||
return nil, v.prepareErr
|
||||
}
|
||||
v.plan.contract = contract
|
||||
return v.plan, nil
|
||||
}
|
||||
|
||||
func TestRunnerPrepareExecutionCompletesWithoutAdmissionOrGeneration(t *testing.T) {
|
||||
schemaDocument := map[string]any{
|
||||
"type": "object",
|
||||
"properties": map[string]any{
|
||||
"value": map[string]any{"type": "string"},
|
||||
"": map[string]any{"type": "boolean"},
|
||||
},
|
||||
}
|
||||
def := promptDef(domain.FormatJSON, domain.ValidationJSONSchema, 0)
|
||||
def.Validation.SchemaPath = "schema.json"
|
||||
reader := defaultArtifactReader()
|
||||
renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{
|
||||
SessionID: "prepared-session",
|
||||
Messages: []domain.RenderedMessage{{
|
||||
Role: "user",
|
||||
Content: "original message",
|
||||
}},
|
||||
}}
|
||||
llmClient := &fakeLLM{forbid: true}
|
||||
validator := &recordingValidationPreparer{
|
||||
plan: &recordingPreparedValidation{schemaDocument: schemaDocument},
|
||||
}
|
||||
admitter := &fakeRunAdmitter{}
|
||||
profile := defaultExecutionProfile()
|
||||
profile.ExtraParams = map[string]any{
|
||||
"metadata": map[string]any{"source": "original"},
|
||||
}
|
||||
runner := NewRunner(
|
||||
&fakePromptRepo{def: def},
|
||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": profile}},
|
||||
nil,
|
||||
reader,
|
||||
renderer,
|
||||
llmClient,
|
||||
validator,
|
||||
admitter,
|
||||
)
|
||||
|
||||
prepared, err := runner.PrepareExecution(context.Background(), domain.RunRequest{
|
||||
PromptID: "p",
|
||||
ProfileID: "exec",
|
||||
APIKey: "direct-test-key",
|
||||
Inputs: singleInputRef(),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("prepare execution: %v", err)
|
||||
}
|
||||
defer prepared.Discard()
|
||||
|
||||
if validator.prepareCalls != 1 || validator.directValidateCalls != 0 {
|
||||
t.Fatalf(
|
||||
"validation calls=(prepare=%d direct=%d), want (1, 0)",
|
||||
validator.prepareCalls,
|
||||
validator.directValidateCalls,
|
||||
)
|
||||
}
|
||||
if reader.calls != 1 || renderer.calls != 1 {
|
||||
t.Fatalf("completion calls=(artifact=%d render=%d), want (1, 1)", reader.calls, renderer.calls)
|
||||
}
|
||||
if len(admitter.backendIDs) != 0 || llmClient.calls != 0 {
|
||||
t.Fatalf("prepare invoked execution collaborators: admission=%v generation=%d", admitter.backendIDs, llmClient.calls)
|
||||
}
|
||||
|
||||
first := prepared.Details()
|
||||
if first == nil {
|
||||
t.Fatal("prepared details are nil")
|
||||
}
|
||||
if first.EffectiveModelParams.APIKey != "" {
|
||||
t.Fatal("prepared details retained the direct API key")
|
||||
}
|
||||
if first.StructuredOutput == nil ||
|
||||
first.StructuredOutput.JSONSchema == nil ||
|
||||
!reflect.DeepEqual(first.StructuredOutput.JSONSchema.Schema, schemaDocument) {
|
||||
t.Fatalf("prepared details have unexpected structured output: %#v", first.StructuredOutput)
|
||||
}
|
||||
|
||||
first.Messages[0].Content = "caller mutation"
|
||||
first.InputHashes["input"] = "caller mutation"
|
||||
first.EffectiveModelParams.ExtraParams["metadata"].(map[string]any)["source"] = "caller mutation"
|
||||
first.StructuredOutput.JSONSchema.Schema.(map[string]any)["type"] = "string"
|
||||
renderer.rendered.Messages[0].Content = "source mutation"
|
||||
|
||||
second := prepared.Details()
|
||||
if second.Messages[0].Content != "original message" ||
|
||||
second.InputHashes["input"] == "caller mutation" ||
|
||||
second.EffectiveModelParams.ExtraParams["metadata"].(map[string]any)["source"] != "original" ||
|
||||
second.StructuredOutput.JSONSchema.Schema.(map[string]any)["type"] != "object" {
|
||||
t.Fatalf("details did not preserve an independent snapshot: %#v", second)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerRunPreparedRechecksEnvironmentCredentialBeforeAdmission(t *testing.T) {
|
||||
const environmentName = "PROMPTKIT_PREPARED_EXECUTION_TEST_KEY"
|
||||
t.Setenv(environmentName, "available-during-preparation")
|
||||
|
||||
profile := defaultExecutionProfile()
|
||||
profile.APIKeyEnv = environmentName
|
||||
validator := &recordingValidationPreparer{plan: &recordingPreparedValidation{}}
|
||||
admitter := &fakeRunAdmitter{}
|
||||
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "unexpected"}}
|
||||
runner := NewRunner(
|
||||
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
|
||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": profile}},
|
||||
nil,
|
||||
defaultArtifactReader(),
|
||||
defaultRenderer(),
|
||||
llmClient,
|
||||
validator,
|
||||
admitter,
|
||||
)
|
||||
prepared, err := runner.PrepareExecution(context.Background(), domain.RunRequest{
|
||||
PromptID: "p",
|
||||
ProfileID: "exec",
|
||||
Inputs: singleInputRef(),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("prepare execution: %v", err)
|
||||
}
|
||||
if err := os.Unsetenv(environmentName); err != nil {
|
||||
t.Fatalf("unset credential environment: %v", err)
|
||||
}
|
||||
|
||||
result, err := runner.RunPrepared(context.Background(), prepared)
|
||||
if result != nil {
|
||||
t.Fatalf("credential failure returned partial result: %+v", result)
|
||||
}
|
||||
if !errors.Is(err, ErrInvalidRequest) || !errors.Is(err, ErrAPIKeyEnvMissing) {
|
||||
t.Fatalf("credential error identities are missing: %v", err)
|
||||
}
|
||||
if len(admitter.backendIDs) != 0 || llmClient.calls != 0 {
|
||||
t.Fatalf("credential failure reached admission or generation: admission=%v generation=%d", admitter.backendIDs, llmClient.calls)
|
||||
}
|
||||
if _, err := runner.RunPrepared(context.Background(), prepared); !errors.Is(err, ErrInvalidRequest) {
|
||||
t.Fatalf("credential failure did not consume execution: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerRunPreparedKeepsDirectCredentialOutOfMetadata(t *testing.T) {
|
||||
const directKey = "direct-prepared-test-key"
|
||||
|
||||
profile := defaultExecutionProfile()
|
||||
profile.APIKeyRequired = true
|
||||
client := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
|
||||
runner := NewRunner(
|
||||
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
|
||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": profile}},
|
||||
nil,
|
||||
defaultArtifactReader(),
|
||||
defaultRenderer(),
|
||||
client,
|
||||
&recordingValidationPreparer{plan: &recordingPreparedValidation{}},
|
||||
nil,
|
||||
)
|
||||
|
||||
prepared, err := runner.PrepareExecution(context.Background(), domain.RunRequest{
|
||||
PromptID: "p",
|
||||
ProfileID: "exec",
|
||||
APIKey: directKey,
|
||||
Inputs: singleInputRef(),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("prepare execution: %v", err)
|
||||
}
|
||||
if details := prepared.Details(); details.EffectiveModelParams.APIKey != "" {
|
||||
t.Fatal("prepared details retained direct credential")
|
||||
}
|
||||
|
||||
result, err := runner.RunPrepared(context.Background(), prepared)
|
||||
if err != nil {
|
||||
t.Fatalf("run prepared: %v", err)
|
||||
}
|
||||
if client.lastReq.Target.APIKey != directKey {
|
||||
t.Fatal("generation did not receive direct credential")
|
||||
}
|
||||
if result.EffectiveModelParams.APIKey != "" {
|
||||
t.Fatal("run result retained direct credential")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerRunPreparedUsesFrozenValidationForInitialAndRepairOutputs(t *testing.T) {
|
||||
validator := &recordingValidationPreparer{
|
||||
plan: &recordingPreparedValidation{
|
||||
results: []domain.ValidationResult{
|
||||
{
|
||||
Status: domain.ValidationFailed,
|
||||
Mode: domain.ValidationJSON,
|
||||
Errors: []string{"invalid"},
|
||||
IsValid: false,
|
||||
},
|
||||
{
|
||||
Status: domain.ValidationPassed,
|
||||
Mode: domain.ValidationJSON,
|
||||
IsValid: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
repairer := &fakeRepairer{
|
||||
responses: []*domain.GenerateResponse{{Content: `{"repaired":true}`}},
|
||||
}
|
||||
admitter := &fakeRunAdmitter{}
|
||||
reader := defaultArtifactReader()
|
||||
renderer := defaultRenderer()
|
||||
runner := NewRunnerWithRepairer(
|
||||
&fakePromptRepo{def: promptDef(domain.FormatJSON, domain.ValidationJSON, 1)},
|
||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
||||
nil,
|
||||
reader,
|
||||
renderer,
|
||||
&fakeLLM{resp: &domain.GenerateResponse{Content: `{"broken":true}`}},
|
||||
validator,
|
||||
repairer,
|
||||
admitter,
|
||||
)
|
||||
prepared, err := runner.PrepareExecution(context.Background(), domain.RunRequest{
|
||||
PromptID: "p",
|
||||
ProfileID: "exec",
|
||||
Inputs: singleInputRef(),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("prepare execution: %v", err)
|
||||
}
|
||||
|
||||
result, err := runner.RunPrepared(context.Background(), prepared)
|
||||
if err != nil {
|
||||
t.Fatalf("run prepared: %v", err)
|
||||
}
|
||||
if !reflect.DeepEqual(validator.plan.artifacts, []string{`{"broken":true}`, `{"repaired":true}`}) {
|
||||
t.Fatalf("prepared validation artifacts=%#v", validator.plan.artifacts)
|
||||
}
|
||||
if validator.directValidateCalls != 0 || repairer.calls != 1 {
|
||||
t.Fatalf("validation/repair calls=(direct=%d repair=%d), want (0, 1)", validator.directValidateCalls, repairer.calls)
|
||||
}
|
||||
if result.Validation.Status != domain.ValidationPassed || result.Validation.RepairAttempts != 1 {
|
||||
t.Fatalf("unexpected repaired validation result: %+v", result.Validation)
|
||||
}
|
||||
if admitter.releaseCalls != 1 {
|
||||
t.Fatalf("admission releases=%d, want 1", admitter.releaseCalls)
|
||||
}
|
||||
if reader.calls != 1 || renderer.calls != 1 {
|
||||
t.Fatalf("execution reopened preparation sources: artifact=%d render=%d", reader.calls, renderer.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerRunPreparedReleasesAdmissionAcrossExecutionErrors(t *testing.T) {
|
||||
generationFailure := errors.New("generation failed")
|
||||
validationFailure := errors.New("validation failed")
|
||||
repairFailure := errors.New("repair failed")
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
generationErr error
|
||||
validation *recordingPreparedValidation
|
||||
repairer *fakeRepairer
|
||||
wantError error
|
||||
}{
|
||||
{
|
||||
name: "generation failure",
|
||||
generationErr: generationFailure,
|
||||
validation: &recordingPreparedValidation{},
|
||||
wantError: ErrLLMGenerate,
|
||||
},
|
||||
{
|
||||
name: "validation failure",
|
||||
validation: &recordingPreparedValidation{
|
||||
errs: []error{validationFailure},
|
||||
},
|
||||
wantError: ErrValidation,
|
||||
},
|
||||
{
|
||||
name: "repair failure",
|
||||
validation: &recordingPreparedValidation{
|
||||
results: []domain.ValidationResult{{
|
||||
Status: domain.ValidationFailed,
|
||||
Mode: domain.ValidationJSON,
|
||||
Errors: []string{"invalid"},
|
||||
IsValid: false,
|
||||
}},
|
||||
},
|
||||
repairer: &fakeRepairer{err: repairFailure},
|
||||
wantError: ErrValidation,
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
def := promptDef(domain.FormatJSON, domain.ValidationJSON, 1)
|
||||
if test.repairer == nil {
|
||||
def.Validation.RepairAttempts = 0
|
||||
}
|
||||
validator := &recordingValidationPreparer{plan: test.validation}
|
||||
admitter := &fakeRunAdmitter{}
|
||||
runner := NewRunnerWithRepairer(
|
||||
&fakePromptRepo{def: def},
|
||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
||||
nil,
|
||||
defaultArtifactReader(),
|
||||
defaultRenderer(),
|
||||
&fakeLLM{
|
||||
resp: &domain.GenerateResponse{Content: `{"value":true}`},
|
||||
err: test.generationErr,
|
||||
},
|
||||
validator,
|
||||
test.repairer,
|
||||
admitter,
|
||||
)
|
||||
prepared, err := runner.PrepareExecution(context.Background(), domain.RunRequest{
|
||||
PromptID: "p",
|
||||
ProfileID: "exec",
|
||||
Inputs: singleInputRef(),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("prepare execution: %v", err)
|
||||
}
|
||||
|
||||
result, err := runner.RunPrepared(context.Background(), prepared)
|
||||
if result != nil || !errors.Is(err, test.wantError) {
|
||||
t.Fatalf("run prepared=(%+v, %v), want %v", result, err, test.wantError)
|
||||
}
|
||||
if len(admitter.backendIDs) != 1 || admitter.releaseCalls != 1 {
|
||||
t.Fatalf(
|
||||
"admission calls=%#v releases=%d, want one each",
|
||||
admitter.backendIDs,
|
||||
admitter.releaseCalls,
|
||||
)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerPreparedExecutionOwnershipUseAndDiscard(t *testing.T) {
|
||||
newRunner := func() *Runner {
|
||||
return NewRunner(
|
||||
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
|
||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
||||
nil,
|
||||
defaultArtifactReader(),
|
||||
defaultRenderer(),
|
||||
&fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}},
|
||||
&recordingValidationPreparer{plan: &recordingPreparedValidation{}},
|
||||
nil,
|
||||
)
|
||||
}
|
||||
request := domain.RunRequest{
|
||||
PromptID: "p",
|
||||
ProfileID: "exec",
|
||||
Inputs: singleInputRef(),
|
||||
}
|
||||
|
||||
owner := newRunner()
|
||||
prepared, err := owner.PrepareExecution(context.Background(), request)
|
||||
if err != nil {
|
||||
t.Fatalf("prepare execution: %v", err)
|
||||
}
|
||||
if _, err := newRunner().RunPrepared(context.Background(), prepared); !errors.Is(err, ErrInvalidRequest) {
|
||||
t.Fatalf("foreign runner error=%v, want ErrInvalidRequest", err)
|
||||
}
|
||||
if _, err := owner.RunPrepared(context.Background(), prepared); err != nil {
|
||||
t.Fatalf("owner run prepared: %v", err)
|
||||
}
|
||||
if _, err := owner.RunPrepared(context.Background(), prepared); !errors.Is(err, ErrInvalidRequest) {
|
||||
t.Fatalf("second owner run error=%v, want ErrInvalidRequest", err)
|
||||
}
|
||||
if prepared.Details() == nil {
|
||||
t.Fatal("details unavailable after execution")
|
||||
}
|
||||
|
||||
discarded, err := owner.PrepareExecution(context.Background(), request)
|
||||
if err != nil {
|
||||
t.Fatalf("prepare discarded execution: %v", err)
|
||||
}
|
||||
discarded.Discard()
|
||||
discarded.Discard()
|
||||
if _, err := owner.RunPrepared(context.Background(), discarded); !errors.Is(err, ErrInvalidRequest) {
|
||||
t.Fatalf("discarded execution error=%v, want ErrInvalidRequest", err)
|
||||
}
|
||||
if discarded.Details() == nil {
|
||||
t.Fatal("details unavailable after discard")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerPreparedExecutionWithoutValidatorSkipsValidation(t *testing.T) {
|
||||
runner := NewRunner(
|
||||
&fakePromptRepo{def: promptDef(domain.FormatJSON, domain.ValidationJSON, 0)},
|
||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
||||
nil,
|
||||
defaultArtifactReader(),
|
||||
defaultRenderer(),
|
||||
&fakeLLM{resp: &domain.GenerateResponse{Content: `{}`}},
|
||||
nil,
|
||||
nil,
|
||||
)
|
||||
prepared, err := runner.PrepareExecution(context.Background(), domain.RunRequest{
|
||||
PromptID: "p",
|
||||
ProfileID: "exec",
|
||||
Inputs: singleInputRef(),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("prepare execution: %v", err)
|
||||
}
|
||||
|
||||
result, err := runner.RunPrepared(context.Background(), prepared)
|
||||
if err != nil {
|
||||
t.Fatalf("run prepared: %v", err)
|
||||
}
|
||||
if result.Validation.Status != domain.ValidationSkipped || !result.Validation.IsValid {
|
||||
t.Fatalf("unexpected no-validator result: %+v", result.Validation)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerPrepareExecutionRequiresValidationPreparer(t *testing.T) {
|
||||
reader := defaultArtifactReader()
|
||||
runner := NewRunner(
|
||||
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationBasic, 0)},
|
||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
||||
nil,
|
||||
reader,
|
||||
defaultRenderer(),
|
||||
&fakeLLM{forbid: true},
|
||||
&fakeValidator{},
|
||||
nil,
|
||||
)
|
||||
|
||||
prepared, err := runner.PrepareExecution(context.Background(), domain.RunRequest{
|
||||
PromptID: "p",
|
||||
ProfileID: "exec",
|
||||
Inputs: singleInputRef(),
|
||||
})
|
||||
if prepared != nil || !errors.Is(err, ErrValidation) {
|
||||
t.Fatalf("prepare execution=(%+v, %v), want ErrValidation", prepared, err)
|
||||
}
|
||||
if reader.calls != 0 {
|
||||
t.Fatalf("unsupported validator allowed completion, artifact calls=%d", reader.calls)
|
||||
}
|
||||
}
|
||||
103
internal/usecase/profile_inspection.go
Normal file
103
internal/usecase/profile_inspection.go
Normal file
@@ -0,0 +1,103 @@
|
||||
package usecase
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
)
|
||||
|
||||
type resolvedProfileSelection struct {
|
||||
id string
|
||||
profile *domain.ExecutionProfile
|
||||
backend *domain.Backend
|
||||
}
|
||||
|
||||
func (r *Runner) resolveProfileSelection(
|
||||
ctx context.Context,
|
||||
profileID string,
|
||||
) (*resolvedProfileSelection, error) {
|
||||
normalizedID := strings.TrimSpace(profileID)
|
||||
if normalizedID == "" {
|
||||
return nil, fmt.Errorf("%w: profile id is required", ErrInvalidRequest)
|
||||
}
|
||||
if r == nil || r.profiles == nil {
|
||||
return nil, fmt.Errorf("%w: profile repository is not configured", ErrProfileLoad)
|
||||
}
|
||||
|
||||
selectedProfile, err := r.profiles.GetProfile(ctx, normalizedID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: %w", ErrProfileLoad, err)
|
||||
}
|
||||
if selectedProfile == nil {
|
||||
return nil, fmt.Errorf("%w: profile repository returned nil profile", ErrProfileLoad)
|
||||
}
|
||||
|
||||
profileValue := *selectedProfile
|
||||
profileValue.BackendID = strings.TrimSpace(profileValue.BackendID)
|
||||
|
||||
var selectedBackend *domain.Backend
|
||||
if profileValue.BackendID != "" {
|
||||
if r.backends == nil {
|
||||
return nil, fmt.Errorf("%w: backend %q cannot be resolved", ErrProfileLoad, profileValue.BackendID)
|
||||
}
|
||||
backendValue, err := r.backends.GetBackend(profileValue.BackendID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: backend %q: %w", ErrProfileLoad, profileValue.BackendID, err)
|
||||
}
|
||||
selectedBackend = &backendValue
|
||||
}
|
||||
|
||||
return &resolvedProfileSelection{
|
||||
id: normalizedID,
|
||||
profile: &profileValue,
|
||||
backend: selectedBackend,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func validateResolvedExecutionTarget(target domain.ExecutionTarget) error {
|
||||
if strings.TrimSpace(target.Endpoint) == "" {
|
||||
return errors.New("execution endpoint is required")
|
||||
}
|
||||
if strings.TrimSpace(target.Model) == "" {
|
||||
return errors.New("execution model is required")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// InspectProfile resolves one explicit profile without prompt or execution work.
|
||||
func (r *Runner) InspectProfile(
|
||||
ctx context.Context,
|
||||
profileID string,
|
||||
) (*domain.ProfileInspection, error) {
|
||||
normalizedID := strings.TrimSpace(profileID)
|
||||
if normalizedID == "" {
|
||||
return nil, fmt.Errorf("%w: profile id is required", ErrInvalidRequest)
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, fmt.Errorf("%w: %w", ErrProfileLoad, ctx.Err())
|
||||
default:
|
||||
}
|
||||
|
||||
selection, err := r.resolveProfileSelection(ctx, normalizedID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
target, _, err := resolveExecutionTarget(selection.backend, selection.profile, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: %w", ErrProfileLoad, err)
|
||||
}
|
||||
if err := validateResolvedExecutionTarget(target); err != nil {
|
||||
return nil, fmt.Errorf("%w: %w", ErrProfileLoad, err)
|
||||
}
|
||||
target.APIKey = ""
|
||||
|
||||
return &domain.ProfileInspection{
|
||||
ProfileID: selection.id,
|
||||
EffectiveModelParams: target,
|
||||
APIKeyRequired: target.APIKeyRequired,
|
||||
}, nil
|
||||
}
|
||||
213
internal/usecase/profile_inspection_test.go
Normal file
213
internal/usecase/profile_inspection_test.go
Normal file
@@ -0,0 +1,213 @@
|
||||
package usecase
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/defaults"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/profile"
|
||||
)
|
||||
|
||||
type inspectionProfileRepository struct {
|
||||
profile *domain.ExecutionProfile
|
||||
err error
|
||||
calls int
|
||||
id string
|
||||
}
|
||||
|
||||
func (r *inspectionProfileRepository) GetProfile(
|
||||
_ context.Context,
|
||||
id string,
|
||||
) (*domain.ExecutionProfile, error) {
|
||||
r.calls++
|
||||
r.id = id
|
||||
if r.err != nil {
|
||||
return nil, r.err
|
||||
}
|
||||
return r.profile, nil
|
||||
}
|
||||
|
||||
type inspectionBackendResolver struct {
|
||||
backend domain.Backend
|
||||
err error
|
||||
calls int
|
||||
id string
|
||||
}
|
||||
|
||||
func (r *inspectionBackendResolver) GetBackend(id string) (domain.Backend, error) {
|
||||
r.calls++
|
||||
r.id = id
|
||||
if r.err != nil {
|
||||
return domain.Backend{}, r.err
|
||||
}
|
||||
return r.backend, nil
|
||||
}
|
||||
|
||||
func TestRunnerInspectProfileResolvesProfileAndBackendOnce(t *testing.T) {
|
||||
profiles := &inspectionProfileRepository{profile: &domain.ExecutionProfile{
|
||||
ID: "profile",
|
||||
BackendID: " backend ",
|
||||
Model: "profile-model",
|
||||
Temperature: 0.4,
|
||||
MaxTokens: 32,
|
||||
TimeoutSeconds: 45,
|
||||
ServiceTier: "priority",
|
||||
ReasoningEffort: "high",
|
||||
ExtraParams: map[string]any{
|
||||
"profile": "value",
|
||||
},
|
||||
}}
|
||||
backends := &inspectionBackendResolver{backend: domain.Backend{
|
||||
ID: "backend",
|
||||
Endpoint: "https://backend.example/v1",
|
||||
APIKeyEnv: "BACKEND_KEY",
|
||||
ExtraParams: map[string]any{
|
||||
"backend": "value",
|
||||
},
|
||||
}}
|
||||
runner := &Runner{profiles: profiles, backends: backends}
|
||||
|
||||
inspection, err := runner.InspectProfile(context.Background(), " profile ")
|
||||
if err != nil {
|
||||
t.Fatalf("inspect profile: %v", err)
|
||||
}
|
||||
if profiles.calls != 1 || profiles.id != "profile" {
|
||||
t.Fatalf("profile lookup=(calls=%d id=%q), want one exact lookup", profiles.calls, profiles.id)
|
||||
}
|
||||
if backends.calls != 1 || backends.id != "backend" {
|
||||
t.Fatalf("backend lookup=(calls=%d id=%q), want one exact lookup", backends.calls, backends.id)
|
||||
}
|
||||
if profiles.profile.BackendID != " backend " {
|
||||
t.Fatalf("inspection mutated repository profile backend: %q", profiles.profile.BackendID)
|
||||
}
|
||||
|
||||
wantTarget := domain.ExecutionTarget{
|
||||
BackendID: "backend",
|
||||
Endpoint: "https://backend.example/v1",
|
||||
Model: "profile-model",
|
||||
Temperature: 0.4,
|
||||
MaxTokens: 32,
|
||||
TopP: defaults.ExecutionTargetDefault().TopP,
|
||||
TimeoutSeconds: 45,
|
||||
ServiceTier: "priority",
|
||||
ReasoningEffort: "high",
|
||||
APIKeyEnv: "BACKEND_KEY",
|
||||
ExtraParams: map[string]any{
|
||||
"profile": "value",
|
||||
},
|
||||
}
|
||||
if inspection.ProfileID != "profile" || inspection.APIKeyRequired ||
|
||||
!reflect.DeepEqual(inspection.EffectiveModelParams, wantTarget) {
|
||||
t.Fatalf("inspection=%#v, want profile=%q target=%#v", inspection, "profile", wantTarget)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerInspectProfileDoesNotNeedExecutionCollaboratorsOrCredentials(t *testing.T) {
|
||||
t.Setenv("PROMPTKIT_INSPECTION_TEST_KEY", "")
|
||||
profiles := &inspectionProfileRepository{profile: &domain.ExecutionProfile{
|
||||
ID: "endpoint-only",
|
||||
Endpoint: "https://profile.example/v1",
|
||||
Model: "profile-model",
|
||||
APIKeyEnv: "PROMPTKIT_INSPECTION_TEST_KEY",
|
||||
}}
|
||||
runner := &Runner{profiles: profiles}
|
||||
|
||||
inspection, err := runner.InspectProfile(context.Background(), "endpoint-only")
|
||||
if err != nil {
|
||||
t.Fatalf("inspect endpoint-only profile: %v", err)
|
||||
}
|
||||
if inspection.EffectiveModelParams.BackendID != "" ||
|
||||
inspection.EffectiveModelParams.APIKeyEnv != "PROMPTKIT_INSPECTION_TEST_KEY" ||
|
||||
inspection.APIKeyRequired {
|
||||
t.Fatalf("unexpected endpoint-only inspection: %#v", inspection)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerInspectProfileDirectCredentialRequirementClearsBackendEnvironment(t *testing.T) {
|
||||
profiles := &inspectionProfileRepository{profile: &domain.ExecutionProfile{
|
||||
ID: "direct-key",
|
||||
BackendID: "backend",
|
||||
Model: "profile-model",
|
||||
APIKeyRequired: true,
|
||||
}}
|
||||
backends := &inspectionBackendResolver{backend: domain.Backend{
|
||||
ID: "backend",
|
||||
Endpoint: "https://backend.example/v1",
|
||||
APIKeyEnv: "BACKEND_KEY",
|
||||
}}
|
||||
|
||||
inspection, err := (&Runner{profiles: profiles, backends: backends}).InspectProfile(
|
||||
context.Background(),
|
||||
"direct-key",
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("inspect direct-key profile: %v", err)
|
||||
}
|
||||
if !inspection.APIKeyRequired || inspection.EffectiveModelParams.APIKeyEnv != "" {
|
||||
t.Fatalf("credential requirement was not resolved exclusively: %#v", inspection)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerInspectProfileClassifiesFailuresWithoutRepositoryWorkAfterCancellation(t *testing.T) {
|
||||
t.Run("blank ID", func(t *testing.T) {
|
||||
profiles := &inspectionProfileRepository{}
|
||||
_, err := (&Runner{profiles: profiles}).InspectProfile(context.Background(), " \t ")
|
||||
if !errors.Is(err, ErrInvalidRequest) || profiles.calls != 0 {
|
||||
t.Fatalf("blank inspection=(%v, calls=%d), want invalid request without lookup", err, profiles.calls)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("canceled context", func(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
profiles := &inspectionProfileRepository{}
|
||||
_, err := (&Runner{profiles: profiles}).InspectProfile(ctx, "profile")
|
||||
if !errors.Is(err, ErrProfileLoad) || !errors.Is(err, context.Canceled) || profiles.calls != 0 {
|
||||
t.Fatalf("canceled inspection=(%v, calls=%d), want profile load and context identities without lookup", err, profiles.calls)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("missing profile", func(t *testing.T) {
|
||||
profiles := &inspectionProfileRepository{err: profile.ErrProfileNotFound}
|
||||
_, err := (&Runner{profiles: profiles}).InspectProfile(context.Background(), "missing")
|
||||
if !errors.Is(err, ErrProfileLoad) || !errors.Is(err, profile.ErrProfileNotFound) {
|
||||
t.Fatalf("missing profile error=%v, want profile load and not-found identities", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("unknown backend", func(t *testing.T) {
|
||||
backendErr := errors.New("unknown backend")
|
||||
profiles := &inspectionProfileRepository{profile: &domain.ExecutionProfile{
|
||||
ID: "profile", BackendID: "backend", Model: "profile-model",
|
||||
}}
|
||||
backends := &inspectionBackendResolver{err: backendErr}
|
||||
_, err := (&Runner{profiles: profiles, backends: backends}).InspectProfile(context.Background(), "profile")
|
||||
if !errors.Is(err, ErrProfileLoad) || !errors.Is(err, backendErr) {
|
||||
t.Fatalf("unknown backend error=%v, want profile load and backend identities", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("defensive invalid dependencies", func(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
runner *Runner
|
||||
}{
|
||||
{name: "nil repository", runner: &Runner{}},
|
||||
{name: "nil profile", runner: &Runner{profiles: &inspectionProfileRepository{}}},
|
||||
{name: "invalid target", runner: &Runner{profiles: &inspectionProfileRepository{
|
||||
profile: &domain.ExecutionProfile{ID: "profile", Endpoint: "https://profile.example/v1"},
|
||||
}}},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
_, err := tc.runner.InspectProfile(context.Background(), "profile")
|
||||
if !errors.Is(err, ErrProfileLoad) {
|
||||
t.Fatalf("inspection error=%v, want profile load", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
})
|
||||
}
|
||||
73
internal/usecase/prompt_inspection.go
Normal file
73
internal/usecase/prompt_inspection.go
Normal file
@@ -0,0 +1,73 @@
|
||||
package usecase
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
)
|
||||
|
||||
type resolvedPromptDefinition struct {
|
||||
definition *domain.PromptDefinition
|
||||
hash string
|
||||
}
|
||||
|
||||
func (r *Runner) resolvePromptDefinition(
|
||||
ctx context.Context,
|
||||
promptID string,
|
||||
promptVersion string,
|
||||
) (*resolvedPromptDefinition, error) {
|
||||
if strings.TrimSpace(promptID) == "" {
|
||||
return nil, fmt.Errorf("%w: prompt id is required", ErrInvalidRequest)
|
||||
}
|
||||
if r == nil || r.promptDefs == nil {
|
||||
return nil, fmt.Errorf("%w: prompt repository is not configured", ErrPromptLoad)
|
||||
}
|
||||
|
||||
definition, err := r.promptDefs.GetPromptDefinition(ctx, promptID, promptVersion)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: %w", ErrPromptLoad, err)
|
||||
}
|
||||
if definition == nil {
|
||||
return nil, fmt.Errorf("%w: prompt repository returned nil definition", ErrPromptLoad)
|
||||
}
|
||||
|
||||
hash, err := hashPromptDefinition(definition)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: failed to hash prompt definition: %v", ErrPromptLoad, err)
|
||||
}
|
||||
return &resolvedPromptDefinition{definition: definition, hash: hash}, nil
|
||||
}
|
||||
|
||||
// InspectPrompt resolves one explicit prompt without execution work.
|
||||
func (r *Runner) InspectPrompt(
|
||||
ctx context.Context,
|
||||
promptID string,
|
||||
promptVersion string,
|
||||
) (*domain.PromptInspection, error) {
|
||||
if strings.TrimSpace(promptID) == "" {
|
||||
return nil, fmt.Errorf("%w: prompt id is required", ErrInvalidRequest)
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, fmt.Errorf("%w: %w", ErrPromptLoad, ctx.Err())
|
||||
default:
|
||||
}
|
||||
|
||||
selection, err := r.resolvePromptDefinition(ctx, promptID, promptVersion)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
inputs := make([]domain.PromptInput, len(selection.definition.Inputs))
|
||||
copy(inputs, selection.definition.Inputs)
|
||||
|
||||
return &domain.PromptInspection{
|
||||
PromptID: selection.definition.ID,
|
||||
PromptVersion: selection.definition.Version,
|
||||
PromptHash: selection.hash,
|
||||
DefaultProfileID: selection.definition.DefaultProfile,
|
||||
Inputs: inputs,
|
||||
OutputContract: selection.definition.Validation,
|
||||
}, nil
|
||||
}
|
||||
157
internal/usecase/prompt_inspection_test.go
Normal file
157
internal/usecase/prompt_inspection_test.go
Normal file
@@ -0,0 +1,157 @@
|
||||
package usecase
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/promptdef"
|
||||
)
|
||||
|
||||
type inspectionPromptRepository struct {
|
||||
definition *domain.PromptDefinition
|
||||
err error
|
||||
calls int
|
||||
id string
|
||||
version string
|
||||
}
|
||||
|
||||
func (r *inspectionPromptRepository) GetPromptDefinition(
|
||||
_ context.Context,
|
||||
id string,
|
||||
version string,
|
||||
) (*domain.PromptDefinition, error) {
|
||||
r.calls++
|
||||
r.id = id
|
||||
r.version = version
|
||||
if r.err != nil {
|
||||
return nil, r.err
|
||||
}
|
||||
return r.definition, nil
|
||||
}
|
||||
|
||||
func TestRunnerInspectPromptResolvesOneDefinitionWithoutExecutionCollaborators(t *testing.T) {
|
||||
definition := &domain.PromptDefinition{
|
||||
ID: "normalized.prompt",
|
||||
Version: "1.2.3",
|
||||
DefaultProfile: "not-resolved",
|
||||
Inputs: []domain.PromptInput{
|
||||
{Name: "document", Required: true, ContentType: "text/plain", Description: "Source document."},
|
||||
{Name: "audience", ContentType: "text/plain", Description: "Intended reader."},
|
||||
},
|
||||
Validation: domain.OutputContract{
|
||||
Format: domain.FormatJSON,
|
||||
ValidationMode: domain.ValidationJSONSchema,
|
||||
SchemaPath: "schemas/result.json",
|
||||
RepairAttempts: 2,
|
||||
},
|
||||
}
|
||||
repository := &inspectionPromptRepository{definition: definition}
|
||||
runner := &Runner{promptDefs: repository}
|
||||
|
||||
inspection, err := runner.InspectPrompt(context.Background(), " prompt-id ", " version ")
|
||||
if err != nil {
|
||||
t.Fatalf("inspect prompt: %v", err)
|
||||
}
|
||||
wantHash, err := hashPromptDefinition(definition)
|
||||
if err != nil {
|
||||
t.Fatalf("hash prompt definition: %v", err)
|
||||
}
|
||||
if repository.calls != 1 || repository.id != " prompt-id " || repository.version != " version " {
|
||||
t.Fatalf("prompt lookup=(calls=%d id=%q version=%q), want one unchanged lookup", repository.calls, repository.id, repository.version)
|
||||
}
|
||||
if inspection.PromptID != definition.ID ||
|
||||
inspection.PromptVersion != definition.Version ||
|
||||
inspection.PromptHash != wantHash ||
|
||||
inspection.DefaultProfileID != definition.DefaultProfile ||
|
||||
!reflect.DeepEqual(inspection.Inputs, definition.Inputs) ||
|
||||
inspection.OutputContract != definition.Validation {
|
||||
t.Fatalf("inspection=%#v, want definition metadata", inspection)
|
||||
}
|
||||
|
||||
inspection.Inputs[0].Name = "changed"
|
||||
second, err := runner.InspectPrompt(context.Background(), " prompt-id ", " version ")
|
||||
if err != nil {
|
||||
t.Fatalf("inspect prompt again: %v", err)
|
||||
}
|
||||
if definition.Inputs[0].Name != "document" || second.Inputs[0].Name != "document" {
|
||||
t.Fatalf("inspection input mutation escaped caller result: definition=%#v next=%#v", definition.Inputs, second.Inputs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerInspectPromptClassifiesFailuresWithoutRepositoryWorkAfterCancellation(t *testing.T) {
|
||||
t.Run("blank ID", func(t *testing.T) {
|
||||
repository := &inspectionPromptRepository{}
|
||||
_, err := (&Runner{promptDefs: repository}).InspectPrompt(context.Background(), " \t ", "version")
|
||||
if !errors.Is(err, ErrInvalidRequest) || repository.calls != 0 {
|
||||
t.Fatalf("blank inspection=(%v, calls=%d), want invalid request without lookup", err, repository.calls)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("canceled context", func(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
repository := &inspectionPromptRepository{}
|
||||
_, err := (&Runner{promptDefs: repository}).InspectPrompt(ctx, "prompt", "version")
|
||||
if !errors.Is(err, ErrPromptLoad) || !errors.Is(err, context.Canceled) || repository.calls != 0 {
|
||||
t.Fatalf("canceled inspection=(%v, calls=%d), want prompt load and context identities without lookup", err, repository.calls)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("missing prompt", func(t *testing.T) {
|
||||
repository := &inspectionPromptRepository{err: promptdef.ErrPromptDefinitionNotFound}
|
||||
_, err := (&Runner{promptDefs: repository}).InspectPrompt(context.Background(), "missing", "version")
|
||||
if !errors.Is(err, ErrPromptLoad) || !errors.Is(err, promptdef.ErrPromptDefinitionNotFound) {
|
||||
t.Fatalf("missing prompt error=%v, want prompt load and not-found identities", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("defensive prompt dependencies", func(t *testing.T) {
|
||||
var nilRunner *Runner
|
||||
cases := []struct {
|
||||
name string
|
||||
runner *Runner
|
||||
}{
|
||||
{name: "nil runner", runner: nilRunner},
|
||||
{name: "nil repository", runner: &Runner{}},
|
||||
{name: "nil definition", runner: &Runner{promptDefs: &inspectionPromptRepository{}}},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
_, err := tc.runner.InspectPrompt(context.Background(), "prompt", "version")
|
||||
if !errors.Is(err, ErrPromptLoad) {
|
||||
t.Fatalf("inspection error=%v, want prompt load", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestRunnerPrepareUsesThePromptInspectionSelectionAndHash(t *testing.T) {
|
||||
definition := promptDef(domain.FormatMarkdown, domain.ValidationBasic, 0)
|
||||
repository := &fakePromptRepo{def: definition}
|
||||
runner := NewRunner(
|
||||
repository,
|
||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
||||
nil,
|
||||
&fakeArtifactReader{},
|
||||
&fakeRenderer{rendered: &domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hello"}}}},
|
||||
nil,
|
||||
nil,
|
||||
nil,
|
||||
)
|
||||
|
||||
inspection, err := runner.InspectPrompt(context.Background(), definition.ID, definition.Version)
|
||||
if err != nil {
|
||||
t.Fatalf("inspect prompt: %v", err)
|
||||
}
|
||||
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{PromptID: definition.ID, PromptVersion: definition.Version, ProfileID: "exec"})
|
||||
if err != nil {
|
||||
t.Fatalf("prepare prompt: %v", err)
|
||||
}
|
||||
if inspection.PromptHash != prepared.PromptHash {
|
||||
t.Fatalf("inspection hash=%q, preparation hash=%q", inspection.PromptHash, prepared.PromptHash)
|
||||
}
|
||||
}
|
||||
@@ -17,6 +17,7 @@ type OutputRepairer interface {
|
||||
type RepairRequest struct {
|
||||
PreviousOutput string
|
||||
ValidationErrors []string
|
||||
SessionID string
|
||||
Target domain.ExecutionTarget
|
||||
StructuredOutput *domain.StructuredOutputSpec
|
||||
Attempt int
|
||||
@@ -42,23 +43,26 @@ func (r *defaultOutputRepairer) Repair(ctx context.Context, req RepairRequest) (
|
||||
errs = strings.Join(req.ValidationErrors, "\n")
|
||||
}
|
||||
|
||||
prompt := domain.RenderedPrompt{Messages: []domain.RenderedMessage{
|
||||
{
|
||||
Role: "system",
|
||||
Content: "You repair invalid JSON output. Return only corrected JSON. Do not include explanations or markdown code fences.",
|
||||
prompt := domain.RenderedPrompt{
|
||||
SessionID: req.SessionID,
|
||||
Messages: []domain.RenderedMessage{
|
||||
{
|
||||
Role: "system",
|
||||
Content: "You repair invalid JSON output. Return only corrected JSON. Do not include explanations or markdown code fences.",
|
||||
},
|
||||
{
|
||||
Role: "user",
|
||||
Content: fmt.Sprintf(
|
||||
"Repair attempt %d of %d for validation mode %s.\n\nValidation errors:\n%s\n\nPrevious output:\n%s\n\nReturn only corrected JSON.",
|
||||
req.Attempt,
|
||||
req.MaxAttempts,
|
||||
req.Mode,
|
||||
errs,
|
||||
req.PreviousOutput,
|
||||
),
|
||||
},
|
||||
},
|
||||
{
|
||||
Role: "user",
|
||||
Content: fmt.Sprintf(
|
||||
"Repair attempt %d of %d for validation mode %s.\n\nValidation errors:\n%s\n\nPrevious output:\n%s\n\nReturn only corrected JSON.",
|
||||
req.Attempt,
|
||||
req.MaxAttempts,
|
||||
req.Mode,
|
||||
errs,
|
||||
req.PreviousOutput,
|
||||
),
|
||||
},
|
||||
}}
|
||||
}
|
||||
|
||||
resp, err := r.llm.Generate(ctx, domain.GenerateRequest{
|
||||
Prompt: prompt,
|
||||
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
"unicode"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/artifact"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/capacity"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/defaults"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/llm"
|
||||
@@ -40,41 +41,80 @@ var (
|
||||
type Runner struct {
|
||||
promptDefs promptdef.Repository
|
||||
profiles profile.Repository
|
||||
backends BackendResolver
|
||||
artifacts artifact.Reader
|
||||
renderer prompt.Renderer
|
||||
llm llm.Client
|
||||
validator validate.Validator
|
||||
repairer OutputRepairer
|
||||
admitter RunAdmitter
|
||||
}
|
||||
|
||||
// BackendResolver resolves one normalized backend ID.
|
||||
type BackendResolver interface {
|
||||
GetBackend(string) (domain.Backend, error)
|
||||
}
|
||||
|
||||
// RunAdmitter reserves capacity for one resolved backend run.
|
||||
type RunAdmitter interface {
|
||||
Admit(context.Context, string) (func(), error)
|
||||
}
|
||||
|
||||
type preparationState struct {
|
||||
definition *domain.PromptDefinition
|
||||
directSessionID string
|
||||
promptDefinitionHash string
|
||||
selectedProfileID string
|
||||
effectiveModel domain.ExecutionTarget
|
||||
targetPresence domain.ExecutionTargetPresence
|
||||
effectiveContract domain.OutputContract
|
||||
start time.Time
|
||||
}
|
||||
|
||||
func NewRunner(
|
||||
promptDefs promptdef.Repository,
|
||||
profiles profile.Repository,
|
||||
backends BackendResolver,
|
||||
artifacts artifact.Reader,
|
||||
renderer prompt.Renderer,
|
||||
llmClient llm.Client,
|
||||
validator validate.Validator,
|
||||
admitter RunAdmitter,
|
||||
) *Runner {
|
||||
return NewRunnerWithRepairer(promptDefs, profiles, artifacts, renderer, llmClient, validator, nil)
|
||||
return NewRunnerWithRepairer(
|
||||
promptDefs,
|
||||
profiles,
|
||||
backends,
|
||||
artifacts,
|
||||
renderer,
|
||||
llmClient,
|
||||
validator,
|
||||
nil,
|
||||
admitter,
|
||||
)
|
||||
}
|
||||
|
||||
func NewRunnerWithRepairer(
|
||||
promptDefs promptdef.Repository,
|
||||
profiles profile.Repository,
|
||||
backends BackendResolver,
|
||||
artifacts artifact.Reader,
|
||||
renderer prompt.Renderer,
|
||||
llmClient llm.Client,
|
||||
validator validate.Validator,
|
||||
repairer OutputRepairer,
|
||||
admitter RunAdmitter,
|
||||
) *Runner {
|
||||
return &Runner{
|
||||
promptDefs: promptDefs,
|
||||
profiles: profiles,
|
||||
backends: backends,
|
||||
artifacts: artifacts,
|
||||
renderer: renderer,
|
||||
llm: llmClient,
|
||||
validator: validator,
|
||||
repairer: repairer,
|
||||
admitter: admitter,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -86,14 +126,51 @@ func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunRes
|
||||
|
||||
start := time.Now().UTC()
|
||||
|
||||
prepared, err := r.Prepare(ctx, req)
|
||||
state, err := r.resolvePreparation(ctx, req, time.Now().UTC())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
release, err := r.admitRun(ctx, state.effectiveModel.BackendID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer release()
|
||||
|
||||
prepared, err := r.completePreparation(ctx, req, state)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
directAPIKey := state.effectiveModel.APIKey
|
||||
|
||||
return r.executePreparedRun(ctx, prepared, directAPIKey, runID, start, func(
|
||||
ctx context.Context,
|
||||
artifact *domain.Artifact,
|
||||
attemptsUsed int,
|
||||
) (domain.ValidationResult, error) {
|
||||
return r.validateOutput(ctx, artifact, prepared.OutputContract, attemptsUsed)
|
||||
})
|
||||
}
|
||||
|
||||
type preparedValidationFunc func(
|
||||
context.Context,
|
||||
*domain.Artifact,
|
||||
int,
|
||||
) (domain.ValidationResult, error)
|
||||
|
||||
func (r *Runner) executePreparedRun(
|
||||
ctx context.Context,
|
||||
prepared *domain.PreparedRun,
|
||||
directAPIKey string,
|
||||
runID string,
|
||||
start time.Time,
|
||||
validateArtifact preparedValidationFunc,
|
||||
) (*domain.RunResult, error) {
|
||||
executionTarget := prepared.EffectiveModelParams
|
||||
executionTarget.APIKey = directAPIKey
|
||||
genResp, err := r.llm.Generate(ctx, domain.GenerateRequest{
|
||||
Prompt: domain.RenderedPrompt{SessionID: prepared.SessionID, Messages: prepared.Messages},
|
||||
Target: prepared.EffectiveModelParams,
|
||||
Target: executionTarget,
|
||||
TargetPresence: prepared.TargetPresence,
|
||||
StructuredOutput: prepared.StructuredOutput,
|
||||
})
|
||||
@@ -105,7 +182,7 @@ func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunRes
|
||||
}
|
||||
|
||||
outputArtifact := buildOutputArtifact(genResp.Content, prepared.OutputContract.Format)
|
||||
validationResult, err := r.validateOutput(ctx, &outputArtifact, prepared.OutputContract, 0)
|
||||
validationResult, err := validateArtifact(ctx, &outputArtifact, 0)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: %w", ErrValidation, err)
|
||||
}
|
||||
@@ -118,7 +195,8 @@ func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunRes
|
||||
repairResp, repairErr := r.repairer.Repair(ctx, RepairRequest{
|
||||
PreviousOutput: genResp.Content,
|
||||
ValidationErrors: validationResult.Errors,
|
||||
Target: prepared.EffectiveModelParams,
|
||||
SessionID: prepared.SessionID,
|
||||
Target: executionTarget,
|
||||
StructuredOutput: prepared.StructuredOutput,
|
||||
Attempt: attemptsUsed,
|
||||
MaxAttempts: prepared.OutputContract.RepairAttempts,
|
||||
@@ -134,7 +212,7 @@ func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunRes
|
||||
genResp = repairResp
|
||||
outputArtifact = buildOutputArtifact(genResp.Content, prepared.OutputContract.Format)
|
||||
|
||||
validationResult, err = r.validateOutput(ctx, &outputArtifact, prepared.OutputContract, attemptsUsed)
|
||||
validationResult, err = validateArtifact(ctx, &outputArtifact, attemptsUsed)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: %w", ErrValidation, err)
|
||||
}
|
||||
@@ -142,6 +220,7 @@ func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunRes
|
||||
}
|
||||
|
||||
end := time.Now().UTC()
|
||||
executionTarget.APIKey = ""
|
||||
|
||||
return &domain.RunResult{
|
||||
RunID: runID,
|
||||
@@ -151,11 +230,13 @@ func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunRes
|
||||
PromptID: prepared.PromptID,
|
||||
PromptVersion: prepared.PromptVersion,
|
||||
PromptHash: prepared.PromptHash,
|
||||
SessionID: prepared.SessionID,
|
||||
RenderedPromptHash: prepared.RenderedPromptHash,
|
||||
SelectedProfileID: prepared.SelectedProfileID,
|
||||
SelectedBackendID: prepared.SelectedBackendID,
|
||||
ModelName: prepared.EffectiveModelParams.Model,
|
||||
Endpoint: prepared.EffectiveModelParams.Endpoint,
|
||||
EffectiveModelParams: prepared.EffectiveModelParams,
|
||||
EffectiveModelParams: executionTarget,
|
||||
InputHashes: prepared.InputHashes,
|
||||
Usage: genResp.Usage,
|
||||
StartTime: start,
|
||||
@@ -165,20 +246,32 @@ func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunRes
|
||||
}
|
||||
|
||||
func (r *Runner) Prepare(ctx context.Context, req domain.RunRequest) (*domain.PreparedRun, error) {
|
||||
state, err := r.resolvePreparation(ctx, req, time.Now().UTC())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return r.completePreparation(ctx, req, state)
|
||||
}
|
||||
|
||||
func (r *Runner) resolvePreparation(
|
||||
ctx context.Context,
|
||||
req domain.RunRequest,
|
||||
start time.Time,
|
||||
) (*preparationState, error) {
|
||||
if strings.TrimSpace(req.PromptID) == "" {
|
||||
return nil, fmt.Errorf("%w: prompt id is required", ErrInvalidRequest)
|
||||
}
|
||||
|
||||
start := time.Now().UTC()
|
||||
|
||||
def, err := r.promptDefs.GetPromptDefinition(ctx, req.PromptID, req.PromptVersion)
|
||||
directSessionID, err := domain.NormalizeSessionID(req.SessionID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: %w", ErrPromptLoad, err)
|
||||
return nil, fmt.Errorf("%w: session_id: %v", ErrInvalidRequest, err)
|
||||
}
|
||||
promptDefinitionHash, err := hashPromptDefinition(def)
|
||||
|
||||
promptSelection, err := r.resolvePromptDefinition(ctx, req.PromptID, req.PromptVersion)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: failed to hash prompt definition: %v", ErrPromptLoad, err)
|
||||
return nil, err
|
||||
}
|
||||
def := promptSelection.definition
|
||||
promptDefinitionHash := promptSelection.hash
|
||||
|
||||
selectedProfileID := strings.TrimSpace(req.ProfileID)
|
||||
if selectedProfileID == "" {
|
||||
@@ -188,32 +281,58 @@ func (r *Runner) Prepare(ctx context.Context, req domain.RunRequest) (*domain.Pr
|
||||
return nil, fmt.Errorf("%w: %w: profile id is required either in request or prompt default_profile", ErrInvalidRequest, ErrProfileRequired)
|
||||
}
|
||||
|
||||
execProfile, err := r.profiles.GetProfile(ctx, selectedProfileID)
|
||||
selection, err := r.resolveProfileSelection(ctx, selectedProfileID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: %w", ErrProfileLoad, err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
effectiveModel, targetPresence, err := resolveExecutionTarget(execProfile, req.Execution)
|
||||
effectiveModel, targetPresence, err := resolveExecutionTarget(selection.backend, selection.profile, req.Execution)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: %w", ErrInvalidRequest, err)
|
||||
}
|
||||
effectiveModel.APIKey = req.APIKey
|
||||
if strings.TrimSpace(effectiveModel.Endpoint) == "" {
|
||||
return nil, fmt.Errorf("%w: execution endpoint is required", ErrInvalidRequest)
|
||||
}
|
||||
if strings.TrimSpace(effectiveModel.Model) == "" {
|
||||
return nil, fmt.Errorf("%w: execution model is required", ErrInvalidRequest)
|
||||
if err := validateResolvedExecutionTarget(effectiveModel); err != nil {
|
||||
return nil, fmt.Errorf("%w: %w", ErrInvalidRequest, err)
|
||||
}
|
||||
if err := validateAPIKey(effectiveModel.APIKeyEnv, effectiveModel.APIKey, effectiveModel.APIKeyRequired); err != nil {
|
||||
return nil, fmt.Errorf("%w: %w", ErrInvalidRequest, err)
|
||||
}
|
||||
|
||||
effectiveContract := resolveOutputContract(def, req.Validation)
|
||||
structuredOutput, err := r.resolveStructuredOutput(ctx, def, effectiveContract)
|
||||
return &preparationState{
|
||||
definition: def,
|
||||
directSessionID: directSessionID,
|
||||
promptDefinitionHash: promptDefinitionHash,
|
||||
selectedProfileID: selection.id,
|
||||
effectiveModel: effectiveModel,
|
||||
targetPresence: targetPresence,
|
||||
effectiveContract: effectiveContract,
|
||||
start: start,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (r *Runner) completePreparation(
|
||||
ctx context.Context,
|
||||
req domain.RunRequest,
|
||||
state *preparationState,
|
||||
) (*domain.PreparedRun, error) {
|
||||
structuredOutput, err := r.resolveStructuredOutput(
|
||||
ctx,
|
||||
state.definition,
|
||||
state.effectiveContract,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return r.completePreparationWithStructuredOutput(ctx, req, state, structuredOutput)
|
||||
}
|
||||
|
||||
func (r *Runner) completePreparationWithStructuredOutput(
|
||||
ctx context.Context,
|
||||
req domain.RunRequest,
|
||||
state *preparationState,
|
||||
structuredOutput *domain.StructuredOutputSpec,
|
||||
) (*domain.PreparedRun, error) {
|
||||
resolvedInputs := make(map[string]*domain.Artifact, len(req.Inputs))
|
||||
inputHashes := make(map[string]string, len(req.Inputs))
|
||||
for name, ref := range req.Inputs {
|
||||
@@ -228,31 +347,57 @@ func (r *Runner) Prepare(ctx context.Context, req domain.RunRequest) (*domain.Pr
|
||||
inputHashes[name] = art.Hash
|
||||
}
|
||||
|
||||
renderedPrompt, err := r.renderer.Render(ctx, def, resolvedInputs, req.Vars)
|
||||
definitionToRender := state.definition
|
||||
if state.directSessionID != "" {
|
||||
definitionCopy := *state.definition
|
||||
definitionCopy.SessionID = ""
|
||||
definitionToRender = &definitionCopy
|
||||
}
|
||||
renderedPrompt, err := r.renderer.Render(ctx, definitionToRender, resolvedInputs, req.Vars)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: %w", ErrPromptRender, err)
|
||||
}
|
||||
if state.directSessionID != "" {
|
||||
renderedPrompt.SessionID = state.directSessionID
|
||||
}
|
||||
|
||||
end := time.Now().UTC()
|
||||
effectiveModel := state.effectiveModel
|
||||
effectiveModel.APIKey = ""
|
||||
return &domain.PreparedRun{
|
||||
PromptID: def.ID,
|
||||
PromptVersion: def.Version,
|
||||
PromptHash: promptDefinitionHash,
|
||||
SelectedProfileID: selectedProfileID,
|
||||
PromptID: state.definition.ID,
|
||||
PromptVersion: state.definition.Version,
|
||||
PromptHash: state.promptDefinitionHash,
|
||||
SelectedProfileID: state.selectedProfileID,
|
||||
SelectedBackendID: state.effectiveModel.BackendID,
|
||||
EffectiveModelParams: effectiveModel,
|
||||
TargetPresence: targetPresence,
|
||||
OutputContract: effectiveContract,
|
||||
TargetPresence: state.targetPresence,
|
||||
OutputContract: state.effectiveContract,
|
||||
StructuredOutput: structuredOutput,
|
||||
InputHashes: inputHashes,
|
||||
SessionID: renderedPrompt.SessionID,
|
||||
RenderedPromptHash: hashRenderedPrompt(*renderedPrompt),
|
||||
Messages: renderedPrompt.Messages,
|
||||
StartTime: start,
|
||||
StartTime: state.start,
|
||||
EndTime: end,
|
||||
DurationMS: end.Sub(start).Milliseconds(),
|
||||
DurationMS: end.Sub(state.start).Milliseconds(),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (r *Runner) admitRun(ctx context.Context, backendID string) (func(), error) {
|
||||
if r.admitter == nil {
|
||||
return func() {}, nil
|
||||
}
|
||||
release, err := r.admitter.Admit(ctx, backendID)
|
||||
if err != nil {
|
||||
if errors.Is(err, capacity.ErrCapacityExceeded) && strings.TrimSpace(backendID) != "" {
|
||||
return nil, &CapacityError{BackendID: backendID}
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
return release, nil
|
||||
}
|
||||
|
||||
func (r *Runner) resolveStructuredOutput(ctx context.Context, def *domain.PromptDefinition, contract domain.OutputContract) (*domain.StructuredOutputSpec, error) {
|
||||
if contract.ValidationMode != domain.ValidationJSONSchema {
|
||||
return nil, nil
|
||||
@@ -268,14 +413,18 @@ func (r *Runner) resolveStructuredOutput(ctx context.Context, def *domain.Prompt
|
||||
return nil, fmt.Errorf("%w: failed to load json schema for structured output: %v", ErrValidation, err)
|
||||
}
|
||||
|
||||
return structuredOutputSpec(def, schemaDoc), nil
|
||||
}
|
||||
|
||||
func structuredOutputSpec(def *domain.PromptDefinition, schemaDocument any) *domain.StructuredOutputSpec {
|
||||
return &domain.StructuredOutputSpec{
|
||||
Type: domain.StructuredOutputJSONSchema,
|
||||
JSONSchema: &domain.StructuredOutputJSONSpec{
|
||||
Name: deriveStructuredSchemaName(def.ID, def.Version),
|
||||
Strict: true,
|
||||
Schema: schemaDoc,
|
||||
Schema: schemaDocument,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
}
|
||||
|
||||
func deriveStructuredSchemaName(promptID string, promptVersion string) string {
|
||||
@@ -338,7 +487,10 @@ func (r *Runner) shouldAttemptRepair(contract domain.OutputContract, validationR
|
||||
|
||||
func mergeExecutionTarget(base domain.ExecutionTarget, override domain.ExecutionTarget) domain.ExecutionTarget {
|
||||
out := base
|
||||
if override.Endpoint != "" {
|
||||
if strings.TrimSpace(override.BackendID) != "" {
|
||||
out.BackendID = override.BackendID
|
||||
}
|
||||
if strings.TrimSpace(override.Endpoint) != "" {
|
||||
out.Endpoint = override.Endpoint
|
||||
}
|
||||
if override.Model != "" {
|
||||
@@ -367,6 +519,7 @@ func mergeExecutionTarget(base domain.ExecutionTarget, override domain.Execution
|
||||
}
|
||||
if override.APIKeyRequired {
|
||||
out.APIKeyRequired = true
|
||||
out.APIKeyEnv = ""
|
||||
}
|
||||
if len(override.ExtraParams) > 0 {
|
||||
out.ExtraParams = copyExtraParams(override.ExtraParams)
|
||||
@@ -414,8 +567,8 @@ func mergeExecutionTargetOverride(base domain.ExecutionTarget, override domain.E
|
||||
if strings.TrimSpace(override.ServiceTier) != "" {
|
||||
out.ServiceTier = override.ServiceTier
|
||||
}
|
||||
if strings.TrimSpace(override.ReasoningEffort) != "" {
|
||||
out.ReasoningEffort = override.ReasoningEffort
|
||||
if override.ReasoningEffort != nil {
|
||||
out.ReasoningEffort = strings.TrimSpace(*override.ReasoningEffort)
|
||||
}
|
||||
if strings.TrimSpace(override.APIKeyEnv) != "" {
|
||||
out.APIKeyEnv = override.APIKeyEnv
|
||||
@@ -426,8 +579,9 @@ func mergeExecutionTargetOverride(base domain.ExecutionTarget, override domain.E
|
||||
return out, presence, nil
|
||||
}
|
||||
|
||||
func resolveExecutionTarget(profileValue *domain.ExecutionProfile, override *domain.ExecutionTargetOverride) (domain.ExecutionTarget, domain.ExecutionTargetPresence, error) {
|
||||
func resolveExecutionTarget(backendValue *domain.Backend, profileValue *domain.ExecutionProfile, override *domain.ExecutionTargetOverride) (domain.ExecutionTarget, domain.ExecutionTargetPresence, error) {
|
||||
out := defaults.ExecutionTargetDefault()
|
||||
out = mergeExecutionTarget(out, backendToTarget(backendValue))
|
||||
out = mergeExecutionTarget(out, executionProfileToTarget(profileValue))
|
||||
var presence domain.ExecutionTargetPresence
|
||||
if override != nil {
|
||||
@@ -461,8 +615,13 @@ func executionProfileToTarget(p *domain.ExecutionProfile) domain.ExecutionTarget
|
||||
if p == nil {
|
||||
return domain.ExecutionTarget{}
|
||||
}
|
||||
endpoint := p.Endpoint
|
||||
if strings.TrimSpace(endpoint) == "" {
|
||||
endpoint = ""
|
||||
}
|
||||
return domain.ExecutionTarget{
|
||||
Endpoint: p.Endpoint,
|
||||
BackendID: p.BackendID,
|
||||
Endpoint: endpoint,
|
||||
Model: p.Model,
|
||||
Temperature: p.Temperature,
|
||||
MaxTokens: p.MaxTokens,
|
||||
@@ -476,6 +635,18 @@ func executionProfileToTarget(p *domain.ExecutionProfile) domain.ExecutionTarget
|
||||
}
|
||||
}
|
||||
|
||||
func backendToTarget(value *domain.Backend) domain.ExecutionTarget {
|
||||
if value == nil {
|
||||
return domain.ExecutionTarget{}
|
||||
}
|
||||
return domain.ExecutionTarget{
|
||||
BackendID: value.ID,
|
||||
Endpoint: value.Endpoint,
|
||||
APIKeyEnv: value.APIKeyEnv,
|
||||
ExtraParams: copyExtraParams(value.ExtraParams),
|
||||
}
|
||||
}
|
||||
|
||||
func copyExtraParams(src map[string]any) map[string]any {
|
||||
if len(src) == 0 {
|
||||
return nil
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -6,6 +6,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"net/url"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
@@ -16,6 +17,8 @@ import (
|
||||
"github.com/santhosh-tekuri/jsonschema/v6"
|
||||
)
|
||||
|
||||
const jsonSchemaDraft2020 = "https://json-schema.org/draft/2020-12/schema"
|
||||
|
||||
// StandardValidator provides basic, JSON, and JSON Schema output validation.
|
||||
type StandardValidator struct {
|
||||
schemaBaseDir string
|
||||
@@ -42,6 +45,101 @@ func (v *FSValidator) Validate(ctx context.Context, artifact *domain.Artifact, c
|
||||
return validateArtifact(ctx, artifact, contract, v.validateJSONSchema)
|
||||
}
|
||||
|
||||
type preparedValidation struct {
|
||||
contract domain.OutputContract
|
||||
schemaDocument any
|
||||
schema *jsonschema.Schema
|
||||
}
|
||||
|
||||
func (p *preparedValidation) Validate(ctx context.Context, artifact *domain.Artifact) (domain.ValidationResult, error) {
|
||||
return validateArtifact(ctx, artifact, p.contract, p.validateJSONSchema)
|
||||
}
|
||||
|
||||
func (p *preparedValidation) SchemaDocument() any {
|
||||
return p.schemaDocument
|
||||
}
|
||||
|
||||
func (p *preparedValidation) validateJSONSchema(instance any, _ string) ([]string, error) {
|
||||
if p.schema == nil {
|
||||
return nil, errors.New("prepared JSON schema is unavailable")
|
||||
}
|
||||
if err := p.schema.Validate(instance); err != nil {
|
||||
return []string{fmt.Sprintf("json schema validation failed: %v", err)}, nil
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (v *StandardValidator) PrepareValidation(ctx context.Context, contract domain.OutputContract) (PreparedValidation, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
prepared := &preparedValidation{contract: contract}
|
||||
if contract.ValidationMode != domain.ValidationJSONSchema {
|
||||
return prepared, nil
|
||||
}
|
||||
|
||||
resolvedSchemaPath, err := v.resolveSchemaPath(contract.SchemaPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
schemaDocument, err := loadJSONSchemaFile(resolvedSchemaPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to compile JSON schema %q: %w", resolvedSchemaPath, err)
|
||||
}
|
||||
schemaRoot, err := v.schemaRoot()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
compiler := newSchemaCompiler(standardSchemaLoader{root: schemaRoot})
|
||||
if err := compiler.AddResource(resolvedSchemaPath, schemaDocument); err != nil {
|
||||
return nil, fmt.Errorf("failed to register JSON schema %q: %w", resolvedSchemaPath, err)
|
||||
}
|
||||
schema, err := compiler.Compile(resolvedSchemaPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to compile JSON schema %q: %w", resolvedSchemaPath, err)
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
prepared.schemaDocument = schemaDocument
|
||||
prepared.schema = schema
|
||||
return prepared, nil
|
||||
}
|
||||
|
||||
func (v *FSValidator) PrepareValidation(ctx context.Context, contract domain.OutputContract) (PreparedValidation, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
prepared := &preparedValidation{contract: contract}
|
||||
if contract.ValidationMode != domain.ValidationJSONSchema {
|
||||
return prepared, nil
|
||||
}
|
||||
|
||||
schemaName, schemaDocument, err := v.loadSchemaDocument(contract.SchemaPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
resourceURL := fsSchemaResourceURL(schemaName)
|
||||
compiler := newSchemaCompiler(fsSchemaLoader{fsys: v.fsys, root: filecatalog.CleanFSRoot(v.root)})
|
||||
if err := compiler.AddResource(resourceURL, schemaDocument); err != nil {
|
||||
return nil, fmt.Errorf("failed to register JSON schema %q: %w", schemaName, err)
|
||||
}
|
||||
schema, err := compiler.Compile(resourceURL)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to compile JSON schema %q: %w", schemaName, err)
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
prepared.schemaDocument = schemaDocument
|
||||
prepared.schema = schema
|
||||
return prepared, nil
|
||||
}
|
||||
|
||||
type schemaValidatorFunc func(instance any, schemaPath string) ([]string, error)
|
||||
|
||||
func validateArtifact(ctx context.Context, artifact *domain.Artifact, contract domain.OutputContract, validateSchema schemaValidatorFunc) (domain.ValidationResult, error) {
|
||||
@@ -121,7 +219,11 @@ func (v *StandardValidator) validateJSONSchema(instance any, schemaPath string)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
compiler := jsonschema.NewCompiler()
|
||||
schemaRoot, err := v.schemaRoot()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
compiler := newSchemaCompiler(standardSchemaLoader{root: schemaRoot})
|
||||
schema, err := compiler.Compile(resolvedSchemaPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to compile JSON schema %q: %w", resolvedSchemaPath, err)
|
||||
@@ -140,7 +242,10 @@ func (v *FSValidator) validateJSONSchema(instance any, schemaPath string) ([]str
|
||||
}
|
||||
|
||||
resourceURL := fsSchemaResourceURL(schemaName)
|
||||
compiler := jsonschema.NewCompiler()
|
||||
if err := validateSchemaDialect(schemaDoc); err != nil {
|
||||
return nil, fmt.Errorf("failed to compile JSON schema %q: %w", schemaName, err)
|
||||
}
|
||||
compiler := newSchemaCompiler(fsSchemaLoader{fsys: v.fsys, root: filecatalog.CleanFSRoot(v.root)})
|
||||
if err := compiler.AddResource(resourceURL, schemaDoc); err != nil {
|
||||
return nil, fmt.Errorf("failed to register JSON schema %q: %w", schemaName, err)
|
||||
}
|
||||
@@ -184,6 +289,9 @@ func (v *StandardValidator) LoadSchemaDocument(ctx context.Context, schemaPath s
|
||||
if err := json.Unmarshal(raw, &doc); err != nil {
|
||||
return nil, fmt.Errorf("failed to decode JSON schema %q: %w", resolved, err)
|
||||
}
|
||||
if err := validateSchemaDialect(doc); err != nil {
|
||||
return nil, fmt.Errorf("failed to decode JSON schema %q: %w", resolved, err)
|
||||
}
|
||||
return doc, nil
|
||||
}
|
||||
|
||||
@@ -206,12 +314,14 @@ func (v *StandardValidator) resolveSchemaPath(schemaPath string) (string, error)
|
||||
return "", errors.New("schema path is required for json_schema validation")
|
||||
}
|
||||
|
||||
resolved := schemaPath
|
||||
if !filepath.IsAbs(schemaPath) {
|
||||
resolved = filepath.Join(v.schemaBaseDir, schemaPath)
|
||||
root, err := v.schemaRoot()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
resolved, err := containedFilesystemPath(root, schemaPath)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
resolved = filepath.Clean(resolved)
|
||||
if _, err := os.Stat(resolved); err != nil {
|
||||
return "", fmt.Errorf("failed to access schema file %q: %w", resolved, err)
|
||||
}
|
||||
@@ -219,6 +329,22 @@ func (v *StandardValidator) resolveSchemaPath(schemaPath string) (string, error)
|
||||
return resolved, nil
|
||||
}
|
||||
|
||||
func (v *StandardValidator) schemaRoot() (string, error) {
|
||||
root := v.schemaBaseDir
|
||||
if strings.TrimSpace(root) == "" {
|
||||
root = "."
|
||||
}
|
||||
absolute, err := filepath.Abs(root)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to resolve schema source %q: %w", root, err)
|
||||
}
|
||||
resolved, err := filepath.EvalSymlinks(absolute)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to access schema source %q: %w", root, err)
|
||||
}
|
||||
return resolved, nil
|
||||
}
|
||||
|
||||
func (v *FSValidator) loadSchemaDocument(schemaPath string) (string, any, error) {
|
||||
resolved, err := v.resolveSchemaPath(schemaPath)
|
||||
if err != nil {
|
||||
@@ -234,6 +360,9 @@ func (v *FSValidator) loadSchemaDocument(schemaPath string) (string, any, error)
|
||||
if err := json.Unmarshal(raw, &doc); err != nil {
|
||||
return "", nil, fmt.Errorf("failed to decode JSON schema %q: %w", resolved, err)
|
||||
}
|
||||
if err := validateSchemaDialect(doc); err != nil {
|
||||
return "", nil, fmt.Errorf("failed to decode JSON schema %q: %w", resolved, err)
|
||||
}
|
||||
return resolved, doc, nil
|
||||
}
|
||||
|
||||
@@ -290,3 +419,131 @@ func cleanSchemaFSPath(schemaPath string) (string, error) {
|
||||
func fsSchemaResourceURL(schemaName string) string {
|
||||
return "promptkit-schema:///" + strings.TrimPrefix(path.Clean(schemaName), "/")
|
||||
}
|
||||
|
||||
func newSchemaCompiler(loader jsonschema.URLLoader) *jsonschema.Compiler {
|
||||
compiler := jsonschema.NewCompiler()
|
||||
compiler.DefaultDraft(jsonschema.Draft2020)
|
||||
compiler.UseLoader(loader)
|
||||
return compiler
|
||||
}
|
||||
|
||||
func validateSchemaDialect(doc any) error {
|
||||
object, ok := doc.(map[string]any)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
value, ok := object["$schema"]
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
dialect, ok := value.(string)
|
||||
if !ok {
|
||||
return errors.New("$schema must be a string")
|
||||
}
|
||||
if dialect != jsonSchemaDraft2020 && dialect != jsonSchemaDraft2020+"#" {
|
||||
return fmt.Errorf("unsupported JSON Schema dialect %q; expected %q", dialect, jsonSchemaDraft2020)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type standardSchemaLoader struct {
|
||||
root string
|
||||
}
|
||||
|
||||
func (l standardSchemaLoader) Load(resourceURL string) (any, error) {
|
||||
fileName, err := (jsonschema.FileLoader{}).ToFile(resourceURL)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("schema reference %q is not a contained file reference: %w", resourceURL, err)
|
||||
}
|
||||
resolved, err := containedFilesystemPath(l.root, fileName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return loadJSONSchemaFile(resolved)
|
||||
}
|
||||
|
||||
func containedFilesystemPath(root, name string) (string, error) {
|
||||
candidate := name
|
||||
if !filepath.IsAbs(candidate) {
|
||||
candidate = filepath.Join(root, candidate)
|
||||
}
|
||||
candidate, err := filepath.Abs(candidate)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to resolve schema path %q: %w", name, err)
|
||||
}
|
||||
candidate, err = filepath.EvalSymlinks(candidate)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to access schema file %q: %w", candidate, err)
|
||||
}
|
||||
relative, err := filepath.Rel(root, candidate)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to compare schema path %q with source root: %w", candidate, err)
|
||||
}
|
||||
if relative == ".." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) {
|
||||
return "", fmt.Errorf("schema path %q escapes source root", name)
|
||||
}
|
||||
return candidate, nil
|
||||
}
|
||||
|
||||
func loadJSONSchemaFile(name string) (any, error) {
|
||||
raw, err := os.ReadFile(name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var doc any
|
||||
if err := json.Unmarshal(raw, &doc); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := validateSchemaDialect(doc); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return doc, nil
|
||||
}
|
||||
|
||||
type fsSchemaLoader struct {
|
||||
fsys fs.FS
|
||||
root string
|
||||
}
|
||||
|
||||
func (l fsSchemaLoader) Load(resourceURL string) (any, error) {
|
||||
parsed, err := url.Parse(resourceURL)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid schema reference %q: %w", resourceURL, err)
|
||||
}
|
||||
if parsed.Scheme != "promptkit-schema" || parsed.Host != "" {
|
||||
return nil, fmt.Errorf("schema reference %q is not allowed", resourceURL)
|
||||
}
|
||||
name, err := url.PathUnescape(strings.TrimPrefix(parsed.Path, "/"))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid schema reference %q: %w", resourceURL, err)
|
||||
}
|
||||
name = path.Clean(name)
|
||||
if l.root == "." {
|
||||
if strings.HasPrefix(name, "../") || name == ".." {
|
||||
return nil, fmt.Errorf("schema reference %q escapes source root", resourceURL)
|
||||
}
|
||||
} else if name != l.root && !strings.HasPrefix(name, l.root+"/") {
|
||||
return nil, fmt.Errorf("schema reference %q escapes source root", resourceURL)
|
||||
}
|
||||
|
||||
rootInfo, err := fs.Stat(l.fsys, l.root)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !rootInfo.IsDir() && name != l.root {
|
||||
return nil, fmt.Errorf("schema reference %q is outside the configured schema file", resourceURL)
|
||||
}
|
||||
|
||||
raw, err := fs.ReadFile(l.fsys, name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var doc any
|
||||
if err := json.Unmarshal(raw, &doc); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := validateSchemaDialect(doc); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return doc, nil
|
||||
}
|
||||
|
||||
@@ -2,8 +2,11 @@ package validate
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"testing/fstest"
|
||||
@@ -118,6 +121,68 @@ func TestStandardValidatorJSONSchemaSuccess(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestStandardValidatorPreparedSchemaSurvivesSourceRemoval(t *testing.T) {
|
||||
tmp := t.TempDir()
|
||||
rootSchema := []byte(`{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"title": "original root",
|
||||
"type": "object",
|
||||
"required": ["value"],
|
||||
"properties": {
|
||||
"value": {"$ref": "value.json"}
|
||||
}
|
||||
}`)
|
||||
rootPath := filepath.Join(tmp, "schema.json")
|
||||
referencePath := filepath.Join(tmp, "value.json")
|
||||
if err := os.WriteFile(rootPath, rootSchema, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(referencePath, []byte(`{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"type": "integer",
|
||||
"minimum": 2
|
||||
}`), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
validator := NewStandardValidator(tmp)
|
||||
preparer, ok := validator.(ValidationPreparer)
|
||||
if !ok {
|
||||
t.Fatal("standard validator does not support validation preparation")
|
||||
}
|
||||
prepared, err := preparer.PrepareValidation(context.Background(), domain.OutputContract{
|
||||
ValidationMode: domain.ValidationJSONSchema,
|
||||
SchemaPath: "schema.json",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("prepare validation: %v", err)
|
||||
}
|
||||
assertSchemaDocument(t, prepared.SchemaDocument(), rootSchema)
|
||||
|
||||
if err := os.Remove(rootPath); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Remove(referencePath); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
valid, err := prepared.Validate(context.Background(), &domain.Artifact{Body: []byte(`{"value":3}`)})
|
||||
if err != nil {
|
||||
t.Fatalf("validate prepared artifact: %v", err)
|
||||
}
|
||||
if valid.Status != domain.ValidationPassed || !valid.IsValid {
|
||||
t.Fatalf("expected passed/valid, got status=%q valid=%v errors=%v", valid.Status, valid.IsValid, valid.Errors)
|
||||
}
|
||||
|
||||
invalid, err := prepared.Validate(context.Background(), &domain.Artifact{Body: []byte(`{"value":"changed"}`)})
|
||||
if err != nil {
|
||||
t.Fatalf("validate prepared artifact: %v", err)
|
||||
}
|
||||
if invalid.Status != domain.ValidationFailed || invalid.IsValid {
|
||||
t.Fatalf("expected failed/invalid, got status=%q valid=%v", invalid.Status, invalid.IsValid)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStandardValidatorJSONSchemaNestedSchemaPathSuccess(t *testing.T) {
|
||||
tmp := t.TempDir()
|
||||
nestedDir := filepath.Join(tmp, "dnd")
|
||||
@@ -293,6 +358,64 @@ func TestFSValidatorJSONSchemaSuccess(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestFSValidatorPreparedSchemaSurvivesSourceMutation(t *testing.T) {
|
||||
rootSchema := []byte(`{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"title": "original root",
|
||||
"type": "object",
|
||||
"required": ["value"],
|
||||
"properties": {
|
||||
"value": {"$ref": "value.json"}
|
||||
}
|
||||
}`)
|
||||
fsys := fstest.MapFS{
|
||||
"schema.json": &fstest.MapFile{Data: rootSchema},
|
||||
"value.json": &fstest.MapFile{Data: []byte(`{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"type": "integer",
|
||||
"minimum": 2
|
||||
}`)},
|
||||
}
|
||||
validator := NewFSValidator(fsys, ".")
|
||||
preparer, ok := validator.(ValidationPreparer)
|
||||
if !ok {
|
||||
t.Fatal("filesystem validator does not support validation preparation")
|
||||
}
|
||||
prepared, err := preparer.PrepareValidation(context.Background(), domain.OutputContract{
|
||||
ValidationMode: domain.ValidationJSONSchema,
|
||||
SchemaPath: "schema.json",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("prepare validation: %v", err)
|
||||
}
|
||||
assertSchemaDocument(t, prepared.SchemaDocument(), rootSchema)
|
||||
|
||||
fsys["schema.json"] = &fstest.MapFile{Data: []byte(`{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"type": "string"
|
||||
}`)}
|
||||
fsys["value.json"] = &fstest.MapFile{Data: []byte(`{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"type": "string"
|
||||
}`)}
|
||||
|
||||
valid, err := prepared.Validate(context.Background(), &domain.Artifact{Body: []byte(`{"value":3}`)})
|
||||
if err != nil {
|
||||
t.Fatalf("validate prepared artifact: %v", err)
|
||||
}
|
||||
if valid.Status != domain.ValidationPassed || !valid.IsValid {
|
||||
t.Fatalf("expected passed/valid, got status=%q valid=%v errors=%v", valid.Status, valid.IsValid, valid.Errors)
|
||||
}
|
||||
|
||||
invalid, err := prepared.Validate(context.Background(), &domain.Artifact{Body: []byte(`{"value":"changed"}`)})
|
||||
if err != nil {
|
||||
t.Fatalf("validate prepared artifact: %v", err)
|
||||
}
|
||||
if invalid.Status != domain.ValidationFailed || invalid.IsValid {
|
||||
t.Fatalf("expected failed/invalid, got status=%q valid=%v", invalid.Status, invalid.IsValid)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFSValidatorJSONSchemaRegistrationError(t *testing.T) {
|
||||
v := NewFSValidator(fstest.MapFS{
|
||||
"schemas/%zz.json": &fstest.MapFile{Data: []byte(`{"type":"object"}`)},
|
||||
@@ -411,3 +534,150 @@ func TestFSValidatorLoadSchemaDocument(t *testing.T) {
|
||||
t.Fatalf("unexpected schema document: %#v", doc)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStandardValidatorJSONSchemaReferenceBoundaries(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
if err := os.WriteFile(filepath.Join(root, "child.json"), []byte(`{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"type": "string"
|
||||
}`), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
reference string
|
||||
wantError string
|
||||
writeOuter bool
|
||||
}{
|
||||
{name: "contained relative reference", reference: "child.json"},
|
||||
{name: "remote reference", reference: "https://example.test/schema.json", wantError: "not a contained file reference"},
|
||||
{name: "escaping reference", reference: "../outside.json", wantError: "escapes source root", writeOuter: true},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if tc.writeOuter {
|
||||
if err := os.WriteFile(filepath.Join(filepath.Dir(root), "outside.json"), []byte(`{"type":"string"}`), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
schema := `{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"$ref": ` + strconv.Quote(tc.reference) + `
|
||||
}`
|
||||
if err := os.WriteFile(filepath.Join(root, "root.json"), []byte(schema), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
v := NewStandardValidator(root)
|
||||
result, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte(`"value"`)}, domain.OutputContract{
|
||||
ValidationMode: domain.ValidationJSONSchema,
|
||||
SchemaPath: "root.json",
|
||||
})
|
||||
if tc.wantError == "" {
|
||||
if err != nil || !result.IsValid {
|
||||
t.Fatalf("expected contained reference to validate, got result=%#v error=%v", result, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err == nil || !strings.Contains(err.Error(), tc.wantError) {
|
||||
t.Fatalf("expected error containing %q, got %v", tc.wantError, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFSValidatorJSONSchemaReferenceBoundaries(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
reference string
|
||||
wantError string
|
||||
}{
|
||||
{name: "same document fragment", reference: "#/$defs/value"},
|
||||
{name: "contained relative reference", reference: "child.json"},
|
||||
{name: "remote reference", reference: "https://example.test/schema.json", wantError: "is not allowed"},
|
||||
{name: "escaping reference", reference: "../outside.json", wantError: "escapes source root"},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
rootSchema := `{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"$defs": {"value": {"type": "string"}},
|
||||
"$ref": ` + strconv.Quote(tc.reference) + `
|
||||
}`
|
||||
v := NewFSValidator(fstest.MapFS{
|
||||
"schemas/root.json": &fstest.MapFile{Data: []byte(rootSchema)},
|
||||
"schemas/child.json": &fstest.MapFile{Data: []byte(`{"type":"string"}`)},
|
||||
"outside.json": &fstest.MapFile{Data: []byte(`{"type":"string"}`)},
|
||||
}, "schemas")
|
||||
result, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte(`"value"`)}, domain.OutputContract{
|
||||
ValidationMode: domain.ValidationJSONSchema,
|
||||
SchemaPath: "root.json",
|
||||
})
|
||||
if tc.wantError == "" {
|
||||
if err != nil || !result.IsValid {
|
||||
t.Fatalf("expected supported reference to validate, got result=%#v error=%v", result, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err == nil || !strings.Contains(err.Error(), tc.wantError) {
|
||||
t.Fatalf("expected error containing %q, got %v", tc.wantError, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestJSONSchemaDialectIsDraft2020(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
dialect string
|
||||
wantError bool
|
||||
}{
|
||||
{name: "omitted uses supported default"},
|
||||
{name: "draft 2020-12", dialect: "https://json-schema.org/draft/2020-12/schema"},
|
||||
{name: "draft 7 rejected", dialect: "http://json-schema.org/draft-07/schema#", wantError: true},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
schema := map[string]any{"type": "object"}
|
||||
if tc.dialect != "" {
|
||||
schema["$schema"] = tc.dialect
|
||||
}
|
||||
data, err := json.Marshal(schema)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
v := NewFSValidator(fstest.MapFS{
|
||||
"schema.json": &fstest.MapFile{Data: data},
|
||||
}, ".")
|
||||
_, err = v.Validate(context.Background(), &domain.Artifact{Body: []byte(`{}`)}, domain.OutputContract{
|
||||
ValidationMode: domain.ValidationJSONSchema,
|
||||
SchemaPath: "schema.json",
|
||||
})
|
||||
if tc.wantError {
|
||||
if err == nil || !strings.Contains(err.Error(), "unsupported JSON Schema dialect") {
|
||||
t.Fatalf("expected unsupported-dialect error, got %v", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("expected supported dialect, got %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func assertSchemaDocument(t *testing.T, got any, expectedJSON []byte) {
|
||||
t.Helper()
|
||||
|
||||
var expected any
|
||||
if err := json.Unmarshal(expectedJSON, &expected); err != nil {
|
||||
t.Fatalf("decode expected schema document: %v", err)
|
||||
}
|
||||
if !reflect.DeepEqual(got, expected) {
|
||||
t.Fatalf("schema document mismatch:\n got: %#v\nwant: %#v", got, expected)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ package validate
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
)
|
||||
|
||||
@@ -10,6 +11,20 @@ type Validator interface {
|
||||
Validate(ctx context.Context, artifact *domain.Artifact, contract domain.OutputContract) (domain.ValidationResult, error)
|
||||
}
|
||||
|
||||
// PreparedValidation validates artifacts against one frozen output contract.
|
||||
type PreparedValidation interface {
|
||||
Validate(ctx context.Context, artifact *domain.Artifact) (domain.ValidationResult, error)
|
||||
// SchemaDocument returns the root JSON Schema document used for provider
|
||||
// structured output, or nil for non-schema modes. Returned internal
|
||||
// immutable state must not be mutated.
|
||||
SchemaDocument() any
|
||||
}
|
||||
|
||||
// ValidationPreparer freezes validation resources for one output contract.
|
||||
type ValidationPreparer interface {
|
||||
PrepareValidation(ctx context.Context, contract domain.OutputContract) (PreparedValidation, error)
|
||||
}
|
||||
|
||||
// SchemaDocumentLoader loads JSON schema documents using validator path semantics.
|
||||
type SchemaDocumentLoader interface {
|
||||
LoadSchemaDocument(ctx context.Context, schemaPath string) (any, error)
|
||||
|
||||
162
json.go
Normal file
162
json.go
Normal file
@@ -0,0 +1,162 @@
|
||||
package promptkit
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"time"
|
||||
)
|
||||
|
||||
// MarshalJSON implements json.Marshaler for PreparedRun. It uses RFC 3339
|
||||
// timestamps, integer duration_ms, and omits zero timing values.
|
||||
func (r PreparedRun) MarshalJSON() ([]byte, error) {
|
||||
var startTime, endTime *time.Time
|
||||
if !r.StartTime.IsZero() {
|
||||
startTime = &r.StartTime
|
||||
}
|
||||
if !r.EndTime.IsZero() {
|
||||
endTime = &r.EndTime
|
||||
}
|
||||
|
||||
var durationMS *int64
|
||||
if r.DurationMS != 0 {
|
||||
durationMS = &r.DurationMS
|
||||
}
|
||||
|
||||
return json.Marshal(struct {
|
||||
PromptID string `json:"prompt_id"`
|
||||
PromptVersion string `json:"prompt_version,omitempty"`
|
||||
PromptHash string `json:"prompt_hash,omitempty"`
|
||||
SelectedProfileID string `json:"selected_profile_id"`
|
||||
SelectedBackendID string `json:"selected_backend_id,omitempty"`
|
||||
EffectiveModelParams ExecutionTarget `json:"effective_model_params"`
|
||||
OutputContract OutputContract `json:"output_contract"`
|
||||
StructuredOutput *StructuredOutputSpec `json:"structured_output,omitempty"`
|
||||
InputHashes map[string]string `json:"input_hashes,omitempty"`
|
||||
SessionID string `json:"session_id,omitempty"`
|
||||
RenderedPromptHash string `json:"rendered_prompt_hash"`
|
||||
Messages []RenderedMessage `json:"messages"`
|
||||
StartTime *time.Time `json:"start_time,omitempty"`
|
||||
EndTime *time.Time `json:"end_time,omitempty"`
|
||||
DurationMS *int64 `json:"duration_ms,omitempty"`
|
||||
}{
|
||||
PromptID: r.PromptID,
|
||||
PromptVersion: r.PromptVersion,
|
||||
PromptHash: r.PromptHash,
|
||||
SelectedProfileID: r.SelectedProfileID,
|
||||
SelectedBackendID: r.SelectedBackendID,
|
||||
EffectiveModelParams: r.EffectiveModelParams,
|
||||
OutputContract: r.OutputContract,
|
||||
StructuredOutput: r.StructuredOutput,
|
||||
InputHashes: r.InputHashes,
|
||||
SessionID: r.SessionID,
|
||||
RenderedPromptHash: r.RenderedPromptHash,
|
||||
Messages: r.Messages,
|
||||
StartTime: startTime,
|
||||
EndTime: endTime,
|
||||
DurationMS: durationMS,
|
||||
})
|
||||
}
|
||||
|
||||
// MarshalJSON implements json.Marshaler for RunResult. It encodes Duration as
|
||||
// integer milliseconds in duration_ms and omits zero timing values.
|
||||
func (r RunResult) MarshalJSON() ([]byte, error) {
|
||||
var startTime, endTime *time.Time
|
||||
if !r.StartTime.IsZero() {
|
||||
startTime = &r.StartTime
|
||||
}
|
||||
if !r.EndTime.IsZero() {
|
||||
endTime = &r.EndTime
|
||||
}
|
||||
|
||||
var durationMS *int64
|
||||
if r.Duration != 0 {
|
||||
value := r.Duration.Milliseconds()
|
||||
durationMS = &value
|
||||
}
|
||||
|
||||
return json.Marshal(runResultJSON{
|
||||
RunID: r.RunID,
|
||||
Artifact: r.Artifact,
|
||||
RawOutput: r.RawOutput,
|
||||
Validation: r.Validation,
|
||||
PromptID: r.PromptID,
|
||||
PromptVersion: r.PromptVersion,
|
||||
PromptHash: r.PromptHash,
|
||||
SessionID: r.SessionID,
|
||||
RenderedPromptHash: r.RenderedPromptHash,
|
||||
SelectedProfileID: r.SelectedProfileID,
|
||||
SelectedBackendID: r.SelectedBackendID,
|
||||
ModelName: r.ModelName,
|
||||
Endpoint: r.Endpoint,
|
||||
EffectiveModelParams: r.EffectiveModelParams,
|
||||
InputHashes: r.InputHashes,
|
||||
Usage: r.Usage,
|
||||
StartTime: startTime,
|
||||
EndTime: endTime,
|
||||
DurationMS: durationMS,
|
||||
})
|
||||
}
|
||||
|
||||
// UnmarshalJSON implements json.Unmarshaler for RunResult. It decodes
|
||||
// duration_ms into Duration with millisecond precision.
|
||||
func (r *RunResult) UnmarshalJSON(data []byte) error {
|
||||
var wire runResultJSON
|
||||
if err := json.Unmarshal(data, &wire); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
*r = RunResult{
|
||||
RunID: wire.RunID,
|
||||
Artifact: wire.Artifact,
|
||||
RawOutput: wire.RawOutput,
|
||||
Validation: wire.Validation,
|
||||
PromptID: wire.PromptID,
|
||||
PromptVersion: wire.PromptVersion,
|
||||
PromptHash: wire.PromptHash,
|
||||
SessionID: wire.SessionID,
|
||||
RenderedPromptHash: wire.RenderedPromptHash,
|
||||
SelectedProfileID: wire.SelectedProfileID,
|
||||
SelectedBackendID: wire.SelectedBackendID,
|
||||
ModelName: wire.ModelName,
|
||||
Endpoint: wire.Endpoint,
|
||||
EffectiveModelParams: wire.EffectiveModelParams,
|
||||
InputHashes: wire.InputHashes,
|
||||
Usage: wire.Usage,
|
||||
Duration: time.Duration(valueOrZero(wire.DurationMS)) * time.Millisecond,
|
||||
}
|
||||
if wire.StartTime != nil {
|
||||
r.StartTime = *wire.StartTime
|
||||
}
|
||||
if wire.EndTime != nil {
|
||||
r.EndTime = *wire.EndTime
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type runResultJSON struct {
|
||||
RunID string `json:"run_id"`
|
||||
Artifact Artifact `json:"artifact"`
|
||||
RawOutput string `json:"raw_output"`
|
||||
Validation ValidationResult `json:"validation"`
|
||||
PromptID string `json:"prompt_id"`
|
||||
PromptVersion string `json:"prompt_version,omitempty"`
|
||||
PromptHash string `json:"prompt_hash,omitempty"`
|
||||
SessionID string `json:"session_id,omitempty"`
|
||||
RenderedPromptHash string `json:"rendered_prompt_hash"`
|
||||
SelectedProfileID string `json:"selected_profile_id"`
|
||||
SelectedBackendID string `json:"selected_backend_id,omitempty"`
|
||||
ModelName string `json:"model_name"`
|
||||
Endpoint string `json:"endpoint"`
|
||||
EffectiveModelParams ExecutionTarget `json:"effective_model_params"`
|
||||
InputHashes map[string]string `json:"input_hashes,omitempty"`
|
||||
Usage TokenUsage `json:"usage"`
|
||||
StartTime *time.Time `json:"start_time,omitempty"`
|
||||
EndTime *time.Time `json:"end_time,omitempty"`
|
||||
DurationMS *int64 `json:"duration_ms,omitempty"`
|
||||
}
|
||||
|
||||
func valueOrZero(value *int64) int64 {
|
||||
if value == nil {
|
||||
return 0
|
||||
}
|
||||
return *value
|
||||
}
|
||||
55
prepared_execution.go
Normal file
55
prepared_execution.go
Normal file
@@ -0,0 +1,55 @@
|
||||
package promptkit
|
||||
|
||||
import "gitea.maximumdirect.net/eric/promptkit/internal/usecase"
|
||||
|
||||
const preparedExecutionString = "promptkit.PreparedExecution{opaque}"
|
||||
|
||||
// PreparedExecution is an opaque, in-process handle for one completely
|
||||
// prepared execution. A handle is bound to the [Engine] that created it and
|
||||
// permits one [Engine.RunPrepared] invocation.
|
||||
//
|
||||
// PreparedExecution contains no supported serializable state and cannot be
|
||||
// used as a restartable job. Copying the value preserves the same shared
|
||||
// lifecycle; it does not create another execution attempt.
|
||||
type PreparedExecution struct {
|
||||
internal *usecase.PreparedExecution
|
||||
}
|
||||
|
||||
// Details returns a fresh caller-owned, credential-redacted copy of the
|
||||
// prepared request details. Mutating the result cannot affect execution or a
|
||||
// later Details call. Details remains available after execution or discard.
|
||||
//
|
||||
// A nil receiver or zero-value PreparedExecution returns a zero [PreparedRun].
|
||||
func (p *PreparedExecution) Details() PreparedRun {
|
||||
if p == nil || p.internal == nil {
|
||||
return PreparedRun{}
|
||||
}
|
||||
details := fromDomainPreparedRun(p.internal.Details())
|
||||
if details == nil {
|
||||
return PreparedRun{}
|
||||
}
|
||||
return *details
|
||||
}
|
||||
|
||||
// Discard invalidates an unclaimed handle and drops Promptkit's references to
|
||||
// its execution-only state. Discard is nil-safe and idempotent. It does not
|
||||
// cancel an execution that has already claimed the handle; use the
|
||||
// [Engine.RunPrepared] context for cancellation.
|
||||
func (p *PreparedExecution) Discard() {
|
||||
if p == nil || p.internal == nil {
|
||||
return
|
||||
}
|
||||
p.internal.Discard()
|
||||
}
|
||||
|
||||
// String returns a constant representation that exposes no retained request,
|
||||
// rendered content, or credential data.
|
||||
func (p *PreparedExecution) String() string {
|
||||
return preparedExecutionString
|
||||
}
|
||||
|
||||
// GoString returns a constant Go-syntax representation that exposes no
|
||||
// retained request, rendered content, or credential data.
|
||||
func (p *PreparedExecution) GoString() string {
|
||||
return preparedExecutionString
|
||||
}
|
||||
773
prepared_execution_contract_test.go
Normal file
773
prepared_execution_contract_test.go
Normal file
@@ -0,0 +1,773 @@
|
||||
package promptkit_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"reflect"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"testing/fstest"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
func TestPreparedExecutionFreezesSourcesAndReturnsIndependentDetails(t *testing.T) {
|
||||
promptSource := preparedPromptSource("original")
|
||||
profileSource := preparedProfileSource("original-model")
|
||||
schemaSource := preparedSchemaSource()
|
||||
reader := &mutablePreparedArtifactReader{
|
||||
body: "original artifact",
|
||||
hash: "original-input-hash",
|
||||
}
|
||||
client := &preparedRecordingClient{
|
||||
response: &promptkit.GenerateResponse{Content: `{"value":3}`},
|
||||
}
|
||||
engine, err := promptkit.NewEngine(
|
||||
promptkit.Config{},
|
||||
promptkit.WithPromptFS(promptSource, "."),
|
||||
promptkit.WithProfileFS(profileSource, "."),
|
||||
promptkit.WithSchemaFS(schemaSource, "."),
|
||||
promptkit.WithArtifactReader(reader),
|
||||
promptkit.WithLLMClient(client),
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("construct engine: %v", err)
|
||||
}
|
||||
|
||||
temperature := 0.25
|
||||
extraParams := map[string]any{
|
||||
"nested": map[string]any{"source": "original"},
|
||||
}
|
||||
request := promptkit.RunRequest{
|
||||
PromptID: "prepared",
|
||||
Inputs: map[string]promptkit.ArtifactRef{
|
||||
"input": promptkit.Inline("original request input"),
|
||||
},
|
||||
Vars: map[string]string{"label": "original variable"},
|
||||
Execution: &promptkit.ExecutionTargetOverride{
|
||||
Temperature: &temperature,
|
||||
ExtraParams: extraParams,
|
||||
},
|
||||
}
|
||||
preparationContext, cancelPreparation := context.WithCancel(context.Background())
|
||||
prepared, err := engine.PrepareExecution(preparationContext, request)
|
||||
if err != nil {
|
||||
t.Fatalf("prepare execution: %v", err)
|
||||
}
|
||||
cancelPreparation()
|
||||
|
||||
request.PromptID = "changed"
|
||||
request.Inputs["input"] = promptkit.Inline("changed request input")
|
||||
request.Vars["label"] = "changed variable"
|
||||
temperature = 1.5
|
||||
extraParams["nested"].(map[string]any)["source"] = "changed"
|
||||
promptSource["prompt.yaml"] = &fstest.MapFile{Data: []byte(`id: changed`)}
|
||||
profileSource["profile.yaml"] = &fstest.MapFile{Data: []byte(`id: changed`)}
|
||||
schemaSource["schema.json"] = &fstest.MapFile{Data: []byte(`{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"title": "changed root",
|
||||
"type": "string"
|
||||
}`)}
|
||||
schemaSource["value.json"] = &fstest.MapFile{Data: []byte(`{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"type": "string"
|
||||
}`)}
|
||||
reader.set("changed artifact", "changed-input-hash")
|
||||
|
||||
first := prepared.Details()
|
||||
first.Messages[0].Content = "changed details"
|
||||
first.InputHashes["input"] = "changed-details-hash"
|
||||
first.EffectiveModelParams.ExtraParams["nested"].(map[string]any)["source"] = "changed details"
|
||||
first.StructuredOutput.JSONSchema.Schema.(map[string]any)["title"] = "changed details"
|
||||
|
||||
second := prepared.Details()
|
||||
if second.Messages[0].Content != "Input=original artifact Label=original variable" {
|
||||
t.Fatalf("details message changed: %q", second.Messages[0].Content)
|
||||
}
|
||||
if second.InputHashes["input"] != "original-input-hash" {
|
||||
t.Fatalf("details input hash changed: %q", second.InputHashes["input"])
|
||||
}
|
||||
if second.EffectiveModelParams.Model != "original-model" ||
|
||||
second.EffectiveModelParams.Temperature != 0.25 ||
|
||||
second.EffectiveModelParams.ExtraParams["nested"].(map[string]any)["source"] != "original" {
|
||||
t.Fatalf("details target changed: %+v", second.EffectiveModelParams)
|
||||
}
|
||||
schema := second.StructuredOutput.JSONSchema.Schema.(map[string]any)
|
||||
if schema["title"] != "original root" {
|
||||
t.Fatalf("details schema changed: %#v", schema)
|
||||
}
|
||||
|
||||
result, err := engine.RunPrepared(context.Background(), prepared)
|
||||
if err != nil {
|
||||
t.Fatalf("run prepared after preparation-context cancellation: %v", err)
|
||||
}
|
||||
if result.Validation.Status != promptkit.ValidationPassed || !result.Validation.IsValid {
|
||||
t.Fatalf("frozen schema did not validate original output: %+v", result.Validation)
|
||||
}
|
||||
if reader.callCount() != 1 {
|
||||
t.Fatalf("execution reopened artifact source: calls=%d", reader.callCount())
|
||||
}
|
||||
|
||||
requests := client.snapshot()
|
||||
if len(requests) != 1 {
|
||||
t.Fatalf("generation calls=%d, want 1", len(requests))
|
||||
}
|
||||
generated := requests[0]
|
||||
if generated.Prompt.Messages[0].Content != second.Messages[0].Content ||
|
||||
generated.Target.Model != second.EffectiveModelParams.Model ||
|
||||
!reflect.DeepEqual(generated.Target.ExtraParams, second.EffectiveModelParams.ExtraParams) ||
|
||||
!reflect.DeepEqual(generated.StructuredOutput, second.StructuredOutput) {
|
||||
t.Fatalf("generation did not use frozen details:\nrequest=%+v\ndetails=%+v", generated, second)
|
||||
}
|
||||
if result.PromptID != second.PromptID ||
|
||||
result.PromptVersion != second.PromptVersion ||
|
||||
result.PromptHash != second.PromptHash ||
|
||||
result.SessionID != second.SessionID ||
|
||||
result.RenderedPromptHash != second.RenderedPromptHash ||
|
||||
result.SelectedProfileID != second.SelectedProfileID ||
|
||||
result.SelectedBackendID != second.SelectedBackendID ||
|
||||
!reflect.DeepEqual(result.EffectiveModelParams, second.EffectiveModelParams) ||
|
||||
!reflect.DeepEqual(result.InputHashes, second.InputHashes) {
|
||||
t.Fatalf("result provenance does not match details:\nresult=%+v\ndetails=%+v", result, second)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreparedExecutionLifecycleAndEngineBinding(t *testing.T) {
|
||||
ownerClient := &preparedRecordingClient{
|
||||
response: &promptkit.GenerateResponse{Content: "ok"},
|
||||
}
|
||||
owner := newPreparedContractEngine(t, ownerClient, "owner content")
|
||||
foreign := newPreparedContractEngine(t, &preparedRecordingClient{
|
||||
response: &promptkit.GenerateResponse{Content: "unexpected"},
|
||||
}, "foreign content")
|
||||
|
||||
prepared, err := owner.PrepareExecution(context.Background(), promptkit.RunRequest{PromptID: "prepared"})
|
||||
if err != nil {
|
||||
t.Fatalf("prepare execution: %v", err)
|
||||
}
|
||||
copied := *prepared
|
||||
|
||||
var nilEngine *promptkit.Engine
|
||||
if result, err := nilEngine.RunPrepared(context.Background(), prepared); result != nil ||
|
||||
!errors.Is(err, promptkit.ErrInvalidConfig) {
|
||||
t.Fatalf("nil engine result=(%+v, %v), want ErrInvalidConfig", result, err)
|
||||
}
|
||||
if result, err := foreign.RunPrepared(context.Background(), prepared); result != nil ||
|
||||
!errors.Is(err, promptkit.ErrInvalidRequest) {
|
||||
t.Fatalf("foreign engine result=(%+v, %v), want ErrInvalidRequest", result, err)
|
||||
}
|
||||
if result, err := owner.RunPrepared(context.Background(), nil); result != nil ||
|
||||
!errors.Is(err, promptkit.ErrInvalidRequest) {
|
||||
t.Fatalf("nil handle result=(%+v, %v), want ErrInvalidRequest", result, err)
|
||||
}
|
||||
if result, err := owner.RunPrepared(context.Background(), &promptkit.PreparedExecution{}); result != nil ||
|
||||
!errors.Is(err, promptkit.ErrInvalidRequest) {
|
||||
t.Fatalf("zero handle result=(%+v, %v), want ErrInvalidRequest", result, err)
|
||||
}
|
||||
|
||||
result, err := owner.RunPrepared(context.Background(), &copied)
|
||||
if err != nil || result == nil {
|
||||
t.Fatalf("owner run prepared=(%+v, %v), want success", result, err)
|
||||
}
|
||||
for name, handle := range map[string]*promptkit.PreparedExecution{
|
||||
"original": prepared,
|
||||
"copy": &copied,
|
||||
} {
|
||||
if result, err := owner.RunPrepared(context.Background(), handle); result != nil ||
|
||||
!errors.Is(err, promptkit.ErrInvalidRequest) {
|
||||
t.Fatalf("%s reused handle result=(%+v, %v), want ErrInvalidRequest", name, result, err)
|
||||
}
|
||||
if handle.Details().PromptID != "prepared" {
|
||||
t.Fatalf("%s details unavailable after execution", name)
|
||||
}
|
||||
}
|
||||
if len(ownerClient.snapshot()) != 1 {
|
||||
t.Fatalf("owner generation calls=%d, want 1", len(ownerClient.snapshot()))
|
||||
}
|
||||
|
||||
collaboratorFailure := errors.New("prepared collaborator failure")
|
||||
failingClient := &preparedRecordingClient{err: collaboratorFailure}
|
||||
failingEngine := newPreparedContractEngine(t, failingClient, "failure content")
|
||||
failing, err := failingEngine.PrepareExecution(context.Background(), promptkit.RunRequest{PromptID: "prepared"})
|
||||
if err != nil {
|
||||
t.Fatalf("prepare failing execution: %v", err)
|
||||
}
|
||||
if result, err := failingEngine.RunPrepared(context.Background(), failing); result != nil ||
|
||||
!errors.Is(err, promptkit.ErrLLMGenerate) ||
|
||||
!errors.Is(err, collaboratorFailure) {
|
||||
t.Fatalf("generation failure result=(%+v, %v), want public and collaborator identities", result, err)
|
||||
}
|
||||
if result, err := failingEngine.RunPrepared(context.Background(), failing); result != nil ||
|
||||
!errors.Is(err, promptkit.ErrInvalidRequest) {
|
||||
t.Fatalf("failed execution was reusable: result=(%+v, %v)", result, err)
|
||||
}
|
||||
|
||||
cancellationRelease := make(chan struct{})
|
||||
cancellationStarted := make(chan struct{}, 1)
|
||||
cancelingEngine := newPreparedContractEngine(t, &preparedRecordingClient{
|
||||
response: &promptkit.GenerateResponse{Content: "unexpected"},
|
||||
started: cancellationStarted,
|
||||
release: cancellationRelease,
|
||||
}, "cancellation content")
|
||||
canceling, err := cancelingEngine.PrepareExecution(
|
||||
context.Background(),
|
||||
promptkit.RunRequest{PromptID: "prepared"},
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("prepare canceled execution: %v", err)
|
||||
}
|
||||
executionContext, cancelExecution := context.WithCancel(context.Background())
|
||||
type canceledOutcome struct {
|
||||
result *promptkit.RunResult
|
||||
err error
|
||||
}
|
||||
canceledResult := make(chan canceledOutcome, 1)
|
||||
go func() {
|
||||
result, runErr := cancelingEngine.RunPrepared(executionContext, canceling)
|
||||
canceledResult <- canceledOutcome{result: result, err: runErr}
|
||||
}()
|
||||
select {
|
||||
case <-cancellationStarted:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("timed out waiting for cancelable generation")
|
||||
}
|
||||
cancelExecution()
|
||||
select {
|
||||
case outcome := <-canceledResult:
|
||||
if outcome.result != nil ||
|
||||
!errors.Is(outcome.err, promptkit.ErrLLMGenerate) ||
|
||||
!errors.Is(outcome.err, context.Canceled) {
|
||||
t.Fatalf(
|
||||
"canceled execution=(%+v, %v), want generation and context identities",
|
||||
outcome.result,
|
||||
outcome.err,
|
||||
)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("timed out waiting for canceled execution")
|
||||
}
|
||||
if result, err := cancelingEngine.RunPrepared(context.Background(), canceling); result != nil ||
|
||||
!errors.Is(err, promptkit.ErrInvalidRequest) {
|
||||
t.Fatalf("canceled execution was reusable: result=(%+v, %v)", result, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreparedExecutionConcurrentClaimAllowsOneGeneration(t *testing.T) {
|
||||
release := make(chan struct{})
|
||||
client := &preparedRecordingClient{
|
||||
response: &promptkit.GenerateResponse{Content: "ok"},
|
||||
started: make(chan struct{}, 1),
|
||||
release: release,
|
||||
}
|
||||
engine := newPreparedContractEngine(t, client, "concurrent content")
|
||||
prepared, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{PromptID: "prepared"})
|
||||
if err != nil {
|
||||
t.Fatalf("prepare execution: %v", err)
|
||||
}
|
||||
|
||||
type outcome struct {
|
||||
result *promptkit.RunResult
|
||||
err error
|
||||
}
|
||||
outcomes := make(chan outcome, 2)
|
||||
for i := 0; i < 2; i++ {
|
||||
go func() {
|
||||
result, runErr := engine.RunPrepared(context.Background(), prepared)
|
||||
outcomes <- outcome{result: result, err: runErr}
|
||||
}()
|
||||
}
|
||||
|
||||
select {
|
||||
case <-client.started:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("timed out waiting for generation")
|
||||
}
|
||||
select {
|
||||
case loser := <-outcomes:
|
||||
if loser.result != nil || !errors.Is(loser.err, promptkit.ErrInvalidRequest) {
|
||||
t.Fatalf("concurrent loser=(%+v, %v), want ErrInvalidRequest", loser.result, loser.err)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("timed out waiting for rejected concurrent claim")
|
||||
}
|
||||
|
||||
close(release)
|
||||
select {
|
||||
case winner := <-outcomes:
|
||||
if winner.err != nil || winner.result == nil {
|
||||
t.Fatalf("concurrent winner=(%+v, %v), want success", winner.result, winner.err)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("timed out waiting for successful concurrent claim")
|
||||
}
|
||||
if len(client.snapshot()) != 1 {
|
||||
t.Fatalf("generation calls=%d, want 1", len(client.snapshot()))
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreparedExecutionRunAndDiscardRaceHasOneWinner(t *testing.T) {
|
||||
const attempts = 32
|
||||
|
||||
for i := 0; i < attempts; i++ {
|
||||
client := &preparedRecordingClient{
|
||||
response: &promptkit.GenerateResponse{Content: "ok"},
|
||||
}
|
||||
engine := newPreparedContractEngine(t, client, "race content")
|
||||
prepared, err := engine.PrepareExecution(
|
||||
context.Background(),
|
||||
promptkit.RunRequest{PromptID: "prepared"},
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("attempt %d prepare execution: %v", i, err)
|
||||
}
|
||||
|
||||
start := make(chan struct{})
|
||||
type outcome struct {
|
||||
result *promptkit.RunResult
|
||||
err error
|
||||
}
|
||||
runOutcome := make(chan outcome, 1)
|
||||
discardDone := make(chan struct{})
|
||||
|
||||
go func() {
|
||||
<-start
|
||||
result, runErr := engine.RunPrepared(context.Background(), prepared)
|
||||
runOutcome <- outcome{result: result, err: runErr}
|
||||
}()
|
||||
go func() {
|
||||
<-start
|
||||
prepared.Discard()
|
||||
close(discardDone)
|
||||
}()
|
||||
|
||||
close(start)
|
||||
runResult := <-runOutcome
|
||||
<-discardDone
|
||||
calls := len(client.snapshot())
|
||||
switch {
|
||||
case runResult.err == nil:
|
||||
if runResult.result == nil || calls != 1 {
|
||||
t.Fatalf(
|
||||
"attempt %d run won with outcome=(%+v, %v), generation calls=%d",
|
||||
i,
|
||||
runResult.result,
|
||||
runResult.err,
|
||||
calls,
|
||||
)
|
||||
}
|
||||
case errors.Is(runResult.err, promptkit.ErrInvalidRequest):
|
||||
if runResult.result != nil || calls != 0 {
|
||||
t.Fatalf(
|
||||
"attempt %d discard won with outcome=(%+v, %v), generation calls=%d",
|
||||
i,
|
||||
runResult.result,
|
||||
runResult.err,
|
||||
calls,
|
||||
)
|
||||
}
|
||||
default:
|
||||
t.Fatalf("attempt %d unexpected run outcome=(%+v, %v)", i, runResult.result, runResult.err)
|
||||
}
|
||||
if prepared.Details().PromptID != "prepared" {
|
||||
t.Fatalf("attempt %d details unavailable after race", i)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreparedExecutionDiscardAndFormattingDoNotExposePrivateState(t *testing.T) {
|
||||
const (
|
||||
directCredential = "pk-test-direct-credential-41f7"
|
||||
renderedContent = "rendered-content-sentinel-98d2"
|
||||
)
|
||||
client := &preparedRecordingClient{
|
||||
response: &promptkit.GenerateResponse{Content: "generated output"},
|
||||
}
|
||||
engine, err := promptkit.NewEngine(
|
||||
promptkit.Config{},
|
||||
promptkit.WithPromptFS(contractPromptFS("prepared", "profile", renderedContent), "."),
|
||||
promptkit.WithProfiles(promptkit.Profile{
|
||||
ID: "profile",
|
||||
Endpoint: "http://example.test/v1",
|
||||
Model: "model",
|
||||
APIKeyRequired: true,
|
||||
}),
|
||||
promptkit.WithLLMClient(client),
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("construct engine: %v", err)
|
||||
}
|
||||
prepared, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{
|
||||
PromptID: "prepared",
|
||||
APIKey: directCredential,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("prepare execution: %v", err)
|
||||
}
|
||||
|
||||
formattedValues := []string{
|
||||
fmt.Sprint(prepared),
|
||||
fmt.Sprintf("%+v", prepared),
|
||||
fmt.Sprintf("%#v", prepared),
|
||||
}
|
||||
for _, formatted := range formattedValues {
|
||||
if formatted != "promptkit.PreparedExecution{opaque}" {
|
||||
t.Fatalf("unexpected opaque formatting: %q", formatted)
|
||||
}
|
||||
assertPreparedPrivateValuesAbsent(t, formatted, directCredential, renderedContent)
|
||||
}
|
||||
payload, err := json.Marshal(prepared)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal opaque handle: %v", err)
|
||||
}
|
||||
assertPreparedPrivateValuesAbsent(t, string(payload), directCredential, renderedContent)
|
||||
|
||||
detailsBefore := prepared.Details()
|
||||
detailsJSON, err := json.Marshal(detailsBefore)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal prepared details: %v", err)
|
||||
}
|
||||
assertPreparedPrivateValuesAbsent(t, string(detailsJSON), directCredential)
|
||||
|
||||
prepared.Discard()
|
||||
prepared.Discard()
|
||||
result, lifecycleErr := engine.RunPrepared(context.Background(), prepared)
|
||||
if result != nil || !errors.Is(lifecycleErr, promptkit.ErrInvalidRequest) {
|
||||
t.Fatalf("discarded execution result=(%+v, %v), want ErrInvalidRequest", result, lifecycleErr)
|
||||
}
|
||||
assertPreparedPrivateValuesAbsent(t, lifecycleErr.Error(), directCredential, renderedContent)
|
||||
if !reflect.DeepEqual(prepared.Details(), detailsBefore) {
|
||||
t.Fatal("details changed after discard")
|
||||
}
|
||||
|
||||
executed, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{
|
||||
PromptID: "prepared",
|
||||
APIKey: directCredential,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("prepare execution for request inspection: %v", err)
|
||||
}
|
||||
executionResult, err := engine.RunPrepared(context.Background(), executed)
|
||||
if err != nil {
|
||||
t.Fatalf("run execution for request inspection: %v", err)
|
||||
}
|
||||
requests := client.snapshot()
|
||||
if len(requests) != 1 || requests[0].APIKey != directCredential {
|
||||
t.Fatalf("direct credential did not reach only the client credential field: %#v", requests)
|
||||
}
|
||||
requestJSON, err := json.Marshal(requests[0])
|
||||
if err != nil {
|
||||
t.Fatalf("marshal captured generate request: %v", err)
|
||||
}
|
||||
for _, value := range []string{
|
||||
fmt.Sprint(requests[0]),
|
||||
fmt.Sprintf("%+v", requests[0]),
|
||||
fmt.Sprintf("%#v", requests[0]),
|
||||
string(requestJSON),
|
||||
fmt.Sprint(executionResult),
|
||||
} {
|
||||
assertPreparedPrivateValuesAbsent(t, value, directCredential)
|
||||
}
|
||||
resultJSON, err := json.Marshal(executionResult)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal execution result: %v", err)
|
||||
}
|
||||
assertPreparedPrivateValuesAbsent(t, string(resultJSON), directCredential)
|
||||
|
||||
var nilHandle *promptkit.PreparedExecution
|
||||
nilHandle.Discard()
|
||||
if !reflect.DeepEqual(nilHandle.Details(), promptkit.PreparedRun{}) {
|
||||
t.Fatalf("nil handle details=%+v, want zero value", nilHandle.Details())
|
||||
}
|
||||
zeroHandle := &promptkit.PreparedExecution{}
|
||||
zeroHandle.Discard()
|
||||
if !reflect.DeepEqual(zeroHandle.Details(), promptkit.PreparedRun{}) {
|
||||
t.Fatalf("zero handle details=%+v, want zero value", zeroHandle.Details())
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreparedExecutionCredentialCapacityAndTimingBoundaries(t *testing.T) {
|
||||
t.Run("credential is rechecked before generation", func(t *testing.T) {
|
||||
const (
|
||||
environmentName = "PROMPTKIT_PREPARED_CONTRACT_KEY"
|
||||
environmentKey = "environment-credential-sentinel"
|
||||
)
|
||||
t.Setenv(environmentName, environmentKey)
|
||||
client := &preparedRecordingClient{
|
||||
response: &promptkit.GenerateResponse{Content: "unexpected"},
|
||||
}
|
||||
engine, err := promptkit.NewEngine(
|
||||
promptkit.Config{},
|
||||
promptkit.WithPromptFS(contractPromptFS("prepared", "profile", "content"), "."),
|
||||
promptkit.WithProfileFS(preparedCredentialProfileSource(environmentName), "."),
|
||||
promptkit.WithLLMClient(client),
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("construct credential engine: %v", err)
|
||||
}
|
||||
prepared, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{PromptID: "prepared"})
|
||||
if err != nil {
|
||||
t.Fatalf("prepare credential execution: %v", err)
|
||||
}
|
||||
if err := os.Unsetenv(environmentName); err != nil {
|
||||
t.Fatalf("unset credential environment: %v", err)
|
||||
}
|
||||
|
||||
result, err := engine.RunPrepared(context.Background(), prepared)
|
||||
if result != nil ||
|
||||
!errors.Is(err, promptkit.ErrInvalidRequest) ||
|
||||
!errors.Is(err, promptkit.ErrAPIKeyEnvMissing) {
|
||||
t.Fatalf("credential execution=(%+v, %v), want credential identities", result, err)
|
||||
}
|
||||
if len(client.snapshot()) != 0 {
|
||||
t.Fatalf("credential failure reached generation: %d calls", len(client.snapshot()))
|
||||
}
|
||||
assertPreparedPrivateValuesAbsent(t, err.Error(), environmentKey)
|
||||
if result, err := engine.RunPrepared(context.Background(), prepared); result != nil ||
|
||||
!errors.Is(err, promptkit.ErrInvalidRequest) {
|
||||
t.Fatalf("credential failure did not consume handle: result=(%+v, %v)", result, err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("preparation does not admit and execution timing starts after retention", func(t *testing.T) {
|
||||
release := make(chan struct{})
|
||||
client := newCapacityGateClient(release, 4)
|
||||
engine := newBackendCapacityEngine(t, client, 1, capacityInt(0), nil)
|
||||
|
||||
activeRun := make(chan capacityRunResult, 1)
|
||||
go runCapacityRequest(
|
||||
engine,
|
||||
context.Background(),
|
||||
promptkit.RunRequest{PromptID: "prompt"},
|
||||
activeRun,
|
||||
)
|
||||
awaitCapacityRequest(t, client.started)
|
||||
|
||||
prepared, err := engine.PrepareExecution(
|
||||
context.Background(),
|
||||
promptkit.RunRequest{PromptID: "prompt"},
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("prepare while capacity is full: %v", err)
|
||||
}
|
||||
if _, _, calls := client.snapshot(); calls != 1 {
|
||||
t.Fatalf("preparation invoked generation: calls=%d", calls)
|
||||
}
|
||||
if result, err := engine.RunPrepared(context.Background(), prepared); result != nil ||
|
||||
!errors.Is(err, promptkit.ErrCapacityExceeded) {
|
||||
t.Fatalf("capacity execution=(%+v, %v), want ErrCapacityExceeded", result, err)
|
||||
} else {
|
||||
var capacityErr *promptkit.CapacityError
|
||||
if !errors.As(err, &capacityErr) || capacityErr == nil || capacityErr.BackendID != "limited" {
|
||||
t.Fatalf("capacity execution=%v, want limited CapacityError", err)
|
||||
}
|
||||
}
|
||||
if result, err := engine.RunPrepared(context.Background(), prepared); result != nil ||
|
||||
!errors.Is(err, promptkit.ErrInvalidRequest) {
|
||||
t.Fatalf("capacity rejection did not consume handle: result=(%+v, %v)", result, err)
|
||||
}
|
||||
if prepared.Details().PromptID != "prompt" {
|
||||
t.Fatal("details unavailable after capacity rejection")
|
||||
}
|
||||
|
||||
close(release)
|
||||
activeOutcome := awaitCapacityRun(t, activeRun)
|
||||
if activeOutcome.err != nil || activeOutcome.result == nil {
|
||||
t.Fatalf("active run outcome=(%+v, %v), want success", activeOutcome.result, activeOutcome.err)
|
||||
}
|
||||
|
||||
timed, err := engine.PrepareExecution(
|
||||
context.Background(),
|
||||
promptkit.RunRequest{PromptID: "prompt"},
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("prepare timed execution: %v", err)
|
||||
}
|
||||
details := timed.Details()
|
||||
time.Sleep(25 * time.Millisecond)
|
||||
executionFloor := time.Now().UTC()
|
||||
result, err := engine.RunPrepared(context.Background(), timed)
|
||||
if err != nil {
|
||||
t.Fatalf("run timed execution: %v", err)
|
||||
}
|
||||
if result.StartTime.Before(executionFloor) ||
|
||||
!result.StartTime.After(details.EndTime) ||
|
||||
result.EndTime.Before(result.StartTime) ||
|
||||
result.Duration != result.EndTime.Sub(result.StartTime) {
|
||||
t.Fatalf(
|
||||
"execution timing includes preparation or retention: details_end=%s floor=%s result=%+v",
|
||||
details.EndTime,
|
||||
executionFloor,
|
||||
result,
|
||||
)
|
||||
}
|
||||
if _, _, calls := client.snapshot(); calls != 2 {
|
||||
t.Fatalf("generation calls=%d, want active and timed executions only", calls)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
type mutablePreparedArtifactReader struct {
|
||||
mu sync.Mutex
|
||||
body string
|
||||
hash string
|
||||
calls int
|
||||
}
|
||||
|
||||
func (r *mutablePreparedArtifactReader) Read(
|
||||
_ context.Context,
|
||||
_ promptkit.ArtifactRef,
|
||||
) (*promptkit.Artifact, error) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
r.calls++
|
||||
return &promptkit.Artifact{
|
||||
Body: []byte(r.body),
|
||||
Hash: r.hash,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (r *mutablePreparedArtifactReader) set(body, hash string) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
r.body = body
|
||||
r.hash = hash
|
||||
}
|
||||
|
||||
func (r *mutablePreparedArtifactReader) callCount() int {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
return r.calls
|
||||
}
|
||||
|
||||
type preparedRecordingClient struct {
|
||||
mu sync.Mutex
|
||||
response *promptkit.GenerateResponse
|
||||
err error
|
||||
requests []promptkit.GenerateRequest
|
||||
started chan struct{}
|
||||
release <-chan struct{}
|
||||
}
|
||||
|
||||
func (c *preparedRecordingClient) Generate(
|
||||
ctx context.Context,
|
||||
request promptkit.GenerateRequest,
|
||||
) (*promptkit.GenerateResponse, error) {
|
||||
c.mu.Lock()
|
||||
c.requests = append(c.requests, request)
|
||||
c.mu.Unlock()
|
||||
|
||||
if c.started != nil {
|
||||
c.started <- struct{}{}
|
||||
}
|
||||
if c.release != nil {
|
||||
select {
|
||||
case <-c.release:
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
}
|
||||
if c.err != nil {
|
||||
return nil, c.err
|
||||
}
|
||||
return c.response, nil
|
||||
}
|
||||
|
||||
func (c *preparedRecordingClient) snapshot() []promptkit.GenerateRequest {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
return append([]promptkit.GenerateRequest(nil), c.requests...)
|
||||
}
|
||||
|
||||
func newPreparedContractEngine(
|
||||
t *testing.T,
|
||||
client promptkit.LLMClient,
|
||||
message string,
|
||||
) *promptkit.Engine {
|
||||
t.Helper()
|
||||
engine, err := promptkit.NewEngine(
|
||||
promptkit.Config{},
|
||||
promptkit.WithPromptFS(contractPromptFS("prepared", "profile", message), "."),
|
||||
promptkit.WithProfiles(promptkit.Profile{
|
||||
ID: "profile",
|
||||
Endpoint: "http://example.test/v1",
|
||||
Model: "model",
|
||||
}),
|
||||
promptkit.WithLLMClient(client),
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("construct prepared execution engine: %v", err)
|
||||
}
|
||||
return engine
|
||||
}
|
||||
|
||||
func preparedPromptSource(label string) fstest.MapFS {
|
||||
return fstest.MapFS{
|
||||
"prompt.yaml": &fstest.MapFile{Data: []byte(`id: prepared
|
||||
version: "1"
|
||||
default_profile: profile
|
||||
inputs:
|
||||
- name: input
|
||||
required: true
|
||||
messages:
|
||||
- role: user
|
||||
content: 'Input={{input "input"}} Label={{.label}}'
|
||||
description: ` + label + `
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: schema.json
|
||||
`)},
|
||||
}
|
||||
}
|
||||
|
||||
func preparedProfileSource(model string) fstest.MapFS {
|
||||
return fstest.MapFS{
|
||||
"profile.yaml": &fstest.MapFile{Data: []byte(`id: profile
|
||||
endpoint: http://example.test/v1
|
||||
model: ` + model + `
|
||||
`)},
|
||||
}
|
||||
}
|
||||
|
||||
func preparedCredentialProfileSource(environmentName string) fstest.MapFS {
|
||||
return fstest.MapFS{
|
||||
"profile.yaml": &fstest.MapFile{Data: []byte(`id: profile
|
||||
endpoint: http://example.test/v1
|
||||
model: model
|
||||
api_key_env: ` + environmentName + `
|
||||
`)},
|
||||
}
|
||||
}
|
||||
|
||||
func preparedSchemaSource() fstest.MapFS {
|
||||
return fstest.MapFS{
|
||||
"schema.json": &fstest.MapFile{Data: []byte(`{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"title": "original root",
|
||||
"type": "object",
|
||||
"required": ["value"],
|
||||
"properties": {
|
||||
"value": {"$ref": "value.json"}
|
||||
}
|
||||
}`)},
|
||||
"value.json": &fstest.MapFile{Data: []byte(`{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"type": "integer",
|
||||
"minimum": 2
|
||||
}`)},
|
||||
}
|
||||
}
|
||||
|
||||
func assertPreparedPrivateValuesAbsent(t *testing.T, value string, privateValues ...string) {
|
||||
t.Helper()
|
||||
for _, privateValue := range privateValues {
|
||||
if strings.Contains(value, privateValue) {
|
||||
t.Fatalf("value exposed private data %q: %s", privateValue, value)
|
||||
}
|
||||
}
|
||||
}
|
||||
16
profiles.go
16
profiles.go
@@ -7,6 +7,7 @@ import (
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/jsonvalue"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/profile"
|
||||
)
|
||||
|
||||
@@ -15,10 +16,16 @@ import (
|
||||
//
|
||||
// It does not register global state, maintain a model catalog, or resolve
|
||||
// credentials. If APIKeyRequired is true, callers satisfy it with
|
||||
// RunRequest.APIKey. Raw API keys do not belong in profiles.
|
||||
// RunRequest.APIKey or an explicit request ExecutionTargetOverride.APIKeyEnv.
|
||||
// Raw API keys do not belong in profiles.
|
||||
//
|
||||
// The function copies the ExtraParams map itself but does not recursively copy
|
||||
// nested values. Validation and a deep copy occur when NewEngine applies a
|
||||
// WithProfiles option containing the returned Profile.
|
||||
func OpenAICompatibleProfile(cfg OpenAICompatibleProfileConfig) Profile {
|
||||
return Profile{
|
||||
ID: cfg.ID,
|
||||
BackendID: cfg.BackendID,
|
||||
Endpoint: cfg.Endpoint,
|
||||
Model: cfg.Model,
|
||||
Temperature: cfg.Temperature,
|
||||
@@ -75,12 +82,13 @@ func (r *memoryProfileRepository) GetProfile(_ context.Context, id string) (*dom
|
||||
}
|
||||
|
||||
func toDomainProfile(publicProfile Profile) (domain.ExecutionProfile, error) {
|
||||
extraParams, err := copyPublicJSONMap(publicProfile.ExtraParams)
|
||||
extraParams, err := jsonvalue.CopyMap(publicProfile.ExtraParams)
|
||||
if err != nil {
|
||||
return domain.ExecutionProfile{}, err
|
||||
}
|
||||
prof := domain.ExecutionProfile{
|
||||
ID: strings.TrimSpace(publicProfile.ID),
|
||||
BackendID: strings.TrimSpace(publicProfile.BackendID),
|
||||
Endpoint: publicProfile.Endpoint,
|
||||
Model: publicProfile.Model,
|
||||
Temperature: publicProfile.Temperature,
|
||||
@@ -102,8 +110,8 @@ func validatePublicProfile(prof domain.ExecutionProfile) error {
|
||||
if strings.TrimSpace(prof.ID) == "" {
|
||||
return errors.New("id is required")
|
||||
}
|
||||
if strings.TrimSpace(prof.Endpoint) == "" {
|
||||
return errors.New("endpoint is required")
|
||||
if strings.TrimSpace(prof.BackendID) == "" && strings.TrimSpace(prof.Endpoint) == "" {
|
||||
return errors.New("backend or endpoint is required")
|
||||
}
|
||||
if strings.TrimSpace(prof.Model) == "" {
|
||||
return errors.New("model is required")
|
||||
|
||||
1411
public_contract_test.go
Normal file
1411
public_contract_test.go
Normal file
File diff suppressed because it is too large
Load Diff
734
types.go
734
types.go
@@ -5,302 +5,714 @@ import (
|
||||
"time"
|
||||
)
|
||||
|
||||
// ArtifactRefType defines how an artifact is referenced.
|
||||
// ArtifactRefType identifies how an [ArtifactRef] supplies content.
|
||||
type ArtifactRefType string
|
||||
|
||||
const (
|
||||
// ArtifactRefInline selects ArtifactRef.Body as the content.
|
||||
ArtifactRefInline ArtifactRefType = "inline"
|
||||
ArtifactRefFile ArtifactRefType = "file"
|
||||
// ArtifactRefFile selects the filesystem path in ArtifactRef.URI.
|
||||
ArtifactRefFile ArtifactRefType = "file"
|
||||
)
|
||||
|
||||
// OutputFormat defines the desired output format.
|
||||
// OutputFormat identifies the media format of generated output.
|
||||
// OutputFormat has a stable JSON string representation.
|
||||
type OutputFormat string
|
||||
|
||||
const (
|
||||
FormatText OutputFormat = "text"
|
||||
// FormatText identifies plain-text output.
|
||||
FormatText OutputFormat = "text"
|
||||
// FormatMarkdown identifies Markdown output.
|
||||
FormatMarkdown OutputFormat = "markdown"
|
||||
FormatJSON OutputFormat = "json"
|
||||
// FormatJSON identifies JSON output.
|
||||
FormatJSON OutputFormat = "json"
|
||||
)
|
||||
|
||||
// ValidationMode defines the output validation strategy.
|
||||
// ValidationMode identifies how generated output is checked.
|
||||
// ValidationMode has a stable JSON string representation.
|
||||
type ValidationMode string
|
||||
|
||||
const (
|
||||
ValidationNone ValidationMode = "none"
|
||||
ValidationBasic ValidationMode = "basic"
|
||||
ValidationJSON ValidationMode = "json"
|
||||
// ValidationNone skips content validation.
|
||||
ValidationNone ValidationMode = "none"
|
||||
// ValidationBasic requires non-empty output.
|
||||
ValidationBasic ValidationMode = "basic"
|
||||
// ValidationJSON requires syntactically valid JSON.
|
||||
ValidationJSON ValidationMode = "json"
|
||||
// ValidationJSONSchema requires JSON that satisfies OutputContract.SchemaPath.
|
||||
ValidationJSONSchema ValidationMode = "json_schema"
|
||||
)
|
||||
|
||||
// ValidationStatus defines the result of a validation check.
|
||||
// ValidationStatus identifies the completed state of an output check.
|
||||
// ValidationStatus has a stable JSON string representation.
|
||||
type ValidationStatus string
|
||||
|
||||
const (
|
||||
ValidationPassed ValidationStatus = "passed"
|
||||
ValidationFailed ValidationStatus = "failed"
|
||||
// ValidationPassed means the generated output satisfied its contract.
|
||||
ValidationPassed ValidationStatus = "passed"
|
||||
// ValidationFailed means validation completed and rejected the generated
|
||||
// output. Engine.Run and Engine.RunPrepared return this status in a result,
|
||||
// not as an error.
|
||||
ValidationFailed ValidationStatus = "failed"
|
||||
// ValidationSkipped means ValidationNone selected no content check.
|
||||
ValidationSkipped ValidationStatus = "skipped"
|
||||
)
|
||||
|
||||
// CacheControlType defines provider cache behavior for prompt content.
|
||||
// CacheControlType identifies provider cache behavior for prompt content.
|
||||
// CacheControlType has a stable JSON string representation.
|
||||
type CacheControlType string
|
||||
|
||||
const (
|
||||
// CacheControlEphemeral requests provider-defined ephemeral caching.
|
||||
CacheControlEphemeral CacheControlType = "ephemeral"
|
||||
)
|
||||
|
||||
// StructuredOutputType identifies provider-level structured output modes.
|
||||
// StructuredOutputType has a stable JSON string representation.
|
||||
type StructuredOutputType string
|
||||
|
||||
const (
|
||||
// StructuredOutputJSONSchema supplies JSON Schema response constraints.
|
||||
StructuredOutputJSONSchema StructuredOutputType = "json_schema"
|
||||
)
|
||||
|
||||
// RunRequest represents a request to prepare or run a single prompt.
|
||||
// RunRequest selects one prompt execution. It has no stable JSON
|
||||
// representation.
|
||||
//
|
||||
// Prepare, PrepareExecution, and Run copy the request's maps, pointers, and
|
||||
// nested JSON-compatible values before using them. The caller may mutate the
|
||||
// request after any method returns. A successful PrepareExecution retains its
|
||||
// own private execution snapshot for RunPrepared.
|
||||
type RunRequest struct {
|
||||
PromptID string
|
||||
// PromptID is the required non-empty prompt identifier.
|
||||
PromptID string
|
||||
// PromptVersion optionally selects one version of PromptID. When empty, the
|
||||
// prompt source must contain exactly one matching version.
|
||||
PromptVersion string
|
||||
ProfileID string
|
||||
APIKey string `json:"-"`
|
||||
Inputs map[string]ArtifactRef
|
||||
Vars map[string]string
|
||||
Execution *ExecutionTargetOverride
|
||||
Validation *OutputContract
|
||||
Metadata map[string]string
|
||||
// ProfileID selects an execution profile. When empty, the prompt's default
|
||||
// profile is used; if both are empty, the error matches ErrProfileRequired
|
||||
// and ErrInvalidRequest.
|
||||
ProfileID string
|
||||
// SessionID optionally supplies a direct per-run session identifier. A
|
||||
// nonblank value is trimmed and overrides the prompt definition's
|
||||
// session_id template. A blank value supplies no direct override. The
|
||||
// maximum is 256 Unicode code points after trimming. A direct value is
|
||||
// opaque consumer metadata, not a credential, and may be exposed in
|
||||
// prepared values, results, collaborator requests, provider requests, and
|
||||
// provider observability. Callers should use stable, non-sensitive
|
||||
// identifiers. An overlong direct value makes Prepare, PrepareExecution, or
|
||||
// Run return an error matching ErrInvalidRequest.
|
||||
SessionID string
|
||||
// APIKey is a request-scoped direct credential. It takes precedence over
|
||||
// APIKeyEnv, is passed to the selected LLMClient, and is never included in
|
||||
// prepared values, results, hashes, JSON, String, or GoString output. A
|
||||
// successful PrepareExecution retains it only in the opaque handle until
|
||||
// RunPrepared claims the handle or Discard invalidates it.
|
||||
APIKey string `json:"-"`
|
||||
// Inputs maps prompt input names to references. A nil or empty map is valid
|
||||
// only when the selected prompt and its templates require no inputs.
|
||||
Inputs map[string]ArtifactRef
|
||||
// Vars supplies Go-template data for messages and the session ID. Nil and
|
||||
// empty maps are equivalent.
|
||||
Vars map[string]string
|
||||
// Execution optionally overrides individual execution settings. Nil uses
|
||||
// the selected profile over its backend, when any, and the framework
|
||||
// baseline.
|
||||
Execution *ExecutionTargetOverride
|
||||
// Validation optionally replaces the prompt's complete output contract. It
|
||||
// does not merge individual fields. Nil uses the prompt contract.
|
||||
Validation *OutputContract
|
||||
}
|
||||
|
||||
// PreparedRun contains prepared prompt execution state. It does not include
|
||||
// resolved API key values, model output, validation results, or internal target
|
||||
// presence metadata.
|
||||
// PreparedRun contains prepared prompt execution state returned by
|
||||
// [Engine.Prepare] or [PreparedExecution.Details]. It does not include resolved
|
||||
// API key values, model output, validation results, or internal target presence
|
||||
// metadata. PreparedRun has a stable JSON representation.
|
||||
//
|
||||
// All maps, slices, pointers, and schema values are caller-owned copies. JSON
|
||||
// timestamps use RFC 3339 and zero timing values are omitted. Hash formats are
|
||||
// opaque.
|
||||
type PreparedRun struct {
|
||||
PromptID string `json:"prompt_id"`
|
||||
PromptVersion string `json:"prompt_version,omitempty"`
|
||||
PromptHash string `json:"prompt_hash,omitempty"`
|
||||
SelectedProfileID string `json:"selected_profile_id"`
|
||||
EffectiveModelParams ExecutionTarget `json:"effective_model_params"`
|
||||
OutputContract OutputContract `json:"output_contract"`
|
||||
StructuredOutput *StructuredOutputSpec `json:"structured_output,omitempty"`
|
||||
InputHashes map[string]string `json:"input_hashes,omitempty"`
|
||||
SessionID string `json:"session_id,omitempty"`
|
||||
RenderedPromptHash string `json:"rendered_prompt_hash"`
|
||||
Messages []RenderedMessage `json:"messages"`
|
||||
StartTime time.Time `json:"start_time,omitempty"`
|
||||
EndTime time.Time `json:"end_time,omitempty"`
|
||||
DurationMS int64 `json:"duration_ms,omitempty"`
|
||||
// PromptID is the selected prompt identifier.
|
||||
PromptID string `json:"prompt_id"`
|
||||
// PromptVersion is the selected prompt version.
|
||||
PromptVersion string `json:"prompt_version,omitempty"`
|
||||
// PromptHash is an opaque equality value for the selected definition.
|
||||
PromptHash string `json:"prompt_hash,omitempty"`
|
||||
// SelectedProfileID is the explicit request profile or prompt default that
|
||||
// supplied execution settings.
|
||||
SelectedProfileID string `json:"selected_profile_id"`
|
||||
// SelectedBackendID equals EffectiveModelParams.BackendID. It is empty for
|
||||
// an endpoint-only profile.
|
||||
SelectedBackendID string `json:"selected_backend_id,omitempty"`
|
||||
// EffectiveModelParams contains settings resolved from the framework timeout
|
||||
// baseline, selected backend, profile, and then request overrides. Unset
|
||||
// optional provider controls remain zero rather than reporting a provider
|
||||
// default. It excludes resolved API-key values.
|
||||
EffectiveModelParams ExecutionTarget `json:"effective_model_params"`
|
||||
// OutputContract is the complete effective output contract.
|
||||
OutputContract OutputContract `json:"output_contract"`
|
||||
// StructuredOutput is non-nil for JSON Schema validation and contains the
|
||||
// provider-facing response constraint passed to an LLM client.
|
||||
StructuredOutput *StructuredOutputSpec `json:"structured_output,omitempty"`
|
||||
// InputHashes maps every supplied input name to its opaque artifact hash.
|
||||
InputHashes map[string]string `json:"input_hashes,omitempty"`
|
||||
// SessionID is the effective direct or rendered session identifier, if any.
|
||||
SessionID string `json:"session_id,omitempty"`
|
||||
// RenderedPromptHash is an opaque equality value for SessionID and Messages.
|
||||
RenderedPromptHash string `json:"rendered_prompt_hash"`
|
||||
// Messages are the rendered messages that Run or RunPrepared passes to the
|
||||
// LLM client.
|
||||
Messages []RenderedMessage `json:"messages"`
|
||||
// StartTime is the UTC time at which preparation began.
|
||||
StartTime time.Time `json:"start_time,omitempty"`
|
||||
// EndTime is the UTC time at which preparation completed.
|
||||
EndTime time.Time `json:"end_time,omitempty"`
|
||||
// DurationMS is preparation elapsed time in integer milliseconds. JSON uses
|
||||
// duration_ms and omits a zero value.
|
||||
DurationMS int64 `json:"duration_ms,omitempty"`
|
||||
}
|
||||
|
||||
// RunResult contains generated output, validation state, and run metadata.
|
||||
// RunResult has a stable JSON representation and round-trips its Duration
|
||||
// through the duration_ms JSON field.
|
||||
//
|
||||
// All maps, slices, and nested values are caller-owned copies. JSON timestamps
|
||||
// use RFC 3339 and zero timing values are omitted. Run IDs and hash formats are
|
||||
// opaque.
|
||||
type RunResult struct {
|
||||
RunID string `json:"run_id"`
|
||||
Artifact Artifact `json:"artifact"`
|
||||
RawOutput string `json:"raw_output"`
|
||||
Validation ValidationResult `json:"validation"`
|
||||
PromptID string `json:"prompt_id"`
|
||||
PromptVersion string `json:"prompt_version,omitempty"`
|
||||
PromptHash string `json:"prompt_hash,omitempty"`
|
||||
RenderedPromptHash string `json:"rendered_prompt_hash"`
|
||||
SelectedProfileID string `json:"selected_profile_id"`
|
||||
ModelName string `json:"model_name"`
|
||||
Endpoint string `json:"endpoint"`
|
||||
EffectiveModelParams ExecutionTarget `json:"effective_model_params"`
|
||||
InputHashes map[string]string `json:"input_hashes,omitempty"`
|
||||
Usage TokenUsage `json:"usage"`
|
||||
StartTime time.Time `json:"start_time,omitempty"`
|
||||
EndTime time.Time `json:"end_time,omitempty"`
|
||||
Duration time.Duration `json:"duration,omitempty"`
|
||||
// RunID is an opaque identifier for this invocation.
|
||||
RunID string `json:"run_id"`
|
||||
// Artifact contains the generated output and derived metadata.
|
||||
Artifact Artifact `json:"artifact"`
|
||||
// RawOutput is the exact generated content before artifact classification
|
||||
// and validation.
|
||||
RawOutput string `json:"raw_output"`
|
||||
// Validation records the completed content check.
|
||||
Validation ValidationResult `json:"validation"`
|
||||
// PromptID is the selected prompt identifier.
|
||||
PromptID string `json:"prompt_id"`
|
||||
// PromptVersion is the selected prompt version.
|
||||
PromptVersion string `json:"prompt_version,omitempty"`
|
||||
// PromptHash is the same opaque definition equality value exposed by
|
||||
// PreparedRun.
|
||||
PromptHash string `json:"prompt_hash,omitempty"`
|
||||
// SessionID is the effective direct or rendered session identifier, if any.
|
||||
// JSON omits an empty value.
|
||||
SessionID string `json:"session_id,omitempty"`
|
||||
// RenderedPromptHash is the same opaque rendered-prompt equality value
|
||||
// computed during preparation.
|
||||
RenderedPromptHash string `json:"rendered_prompt_hash"`
|
||||
// SelectedProfileID identifies the profile used for execution.
|
||||
SelectedProfileID string `json:"selected_profile_id"`
|
||||
// SelectedBackendID equals EffectiveModelParams.BackendID. It is empty for
|
||||
// an endpoint-only profile.
|
||||
SelectedBackendID string `json:"selected_backend_id,omitempty"`
|
||||
// ModelName is the effective model name and equals
|
||||
// EffectiveModelParams.Model.
|
||||
ModelName string `json:"model_name"`
|
||||
// Endpoint is the effective base endpoint and equals
|
||||
// EffectiveModelParams.Endpoint.
|
||||
Endpoint string `json:"endpoint"`
|
||||
// EffectiveModelParams contains the settings supplied to the LLM client,
|
||||
// excluding resolved API-key values.
|
||||
EffectiveModelParams ExecutionTarget `json:"effective_model_params"`
|
||||
// InputHashes are the opaque input equality values computed during
|
||||
// preparation.
|
||||
InputHashes map[string]string `json:"input_hashes,omitempty"`
|
||||
// Usage is the token accounting reported by the LLM client.
|
||||
Usage TokenUsage `json:"usage"`
|
||||
// StartTime is the UTC time immediately before ordinary Run preparation or
|
||||
// after RunPrepared claims its handle.
|
||||
StartTime time.Time `json:"start_time,omitempty"`
|
||||
// EndTime is the UTC time after generation and validation complete.
|
||||
EndTime time.Time `json:"end_time,omitempty"`
|
||||
// Duration covers preparation, generation, and validation for Run. For
|
||||
// RunPrepared it covers only the execution attempt after claim and excludes
|
||||
// preparation and consumer-held delay. JSON represents it as integer
|
||||
// milliseconds in duration_ms and omits a zero value.
|
||||
Duration time.Duration `json:"-"`
|
||||
}
|
||||
|
||||
// ArtifactRef represents a reference to prompt input content.
|
||||
// ArtifactRef identifies prompt input content. It has no stable JSON
|
||||
// representation. Prefer [File], [Inline], or [InlineWithURI] to construct one.
|
||||
type ArtifactRef struct {
|
||||
// Type must be ArtifactRefInline or ArtifactRefFile.
|
||||
Type ArtifactRefType
|
||||
URI string
|
||||
// URI is the file path for ArtifactRefFile and optional provenance metadata
|
||||
// for ArtifactRefInline.
|
||||
URI string
|
||||
// Body is the content for ArtifactRefInline and is ignored for
|
||||
// ArtifactRefFile.
|
||||
Body string
|
||||
}
|
||||
|
||||
// Artifact represents loaded artifact content.
|
||||
// Artifact represents loaded or generated content and has a stable JSON
|
||||
// representation. Body uses encoding/json's base64 representation for []byte.
|
||||
type Artifact struct {
|
||||
Name string
|
||||
ContentType string
|
||||
Body []byte
|
||||
URI string
|
||||
Size int64
|
||||
Hash string
|
||||
// Name is artifact metadata. During input preparation the engine fills an
|
||||
// empty reader-supplied name with the request input-map key.
|
||||
Name string `json:"name"`
|
||||
// ContentType is the media type reported by the reader or derived for
|
||||
// generated output.
|
||||
ContentType string `json:"content_type"`
|
||||
// Body is the artifact content. Engine boundaries copy this slice.
|
||||
Body []byte `json:"body"`
|
||||
// URI is optional source or result provenance metadata.
|
||||
URI string `json:"uri"`
|
||||
// Size is content-size metadata in bytes.
|
||||
Size int64 `json:"size"`
|
||||
// Hash is an opaque content equality value when the producing reader
|
||||
// supplies one. Its format and algorithm are not API contracts.
|
||||
Hash string `json:"hash"`
|
||||
}
|
||||
|
||||
// ArtifactReader resolves a prompt input reference into its content.
|
||||
//
|
||||
// Readers are responsible for supplying artifact metadata. The engine assigns
|
||||
// an input-map name only when the returned artifact name is empty.
|
||||
// Read may be called concurrently. It must honor ctx cancellation to make
|
||||
// Prepare, PrepareExecution, and Run responsive to cancellation. The engine
|
||||
// passes a copied ref and immediately copies the returned Artifact.Body; it
|
||||
// does not retain either value. Readers supply artifact metadata, and the
|
||||
// engine assigns an input-map name only when the returned artifact name is
|
||||
// empty.
|
||||
//
|
||||
// An injected reader owns any application-specific path containment,
|
||||
// authorization, content-size, and content-type policy. It must protect
|
||||
// sensitive references and bodies in its logging and in any copies it retains.
|
||||
// It may reuse or mutate the returned artifact and body after Read returns.
|
||||
//
|
||||
// Returning a non-nil error makes the engine return an error matching
|
||||
// ErrArtifactLoad while preserving the reader error through errors.Is.
|
||||
// Returning a nil artifact with a nil error also produces ErrArtifactLoad.
|
||||
type ArtifactReader interface {
|
||||
Read(context.Context, ArtifactRef) (*Artifact, error)
|
||||
}
|
||||
|
||||
// ExecutionTarget represents effective model runtime settings.
|
||||
// ExecutionTarget represents effective model runtime settings and has a stable
|
||||
// JSON representation. It never exposes a resolved API-key value.
|
||||
type ExecutionTarget struct {
|
||||
Endpoint string `json:"endpoint"`
|
||||
Model string `json:"model"`
|
||||
Temperature float64 `json:"temperature"`
|
||||
MaxTokens int `json:"max_tokens"`
|
||||
TopP float64 `json:"top_p"`
|
||||
TimeoutSeconds int `json:"timeout_seconds"`
|
||||
ServiceTier string `json:"service_tier"`
|
||||
ReasoningEffort string `json:"reasoning_effort"`
|
||||
APIKeyEnv string `json:"api_key_env"`
|
||||
ExtraParams map[string]any `json:"extra_params"`
|
||||
// BackendID is the effective routing identity selected by the profile. It
|
||||
// remains unchanged when a profile or request overrides Endpoint and is
|
||||
// empty for endpoint-only profiles. It is supplied to injected LLMClient
|
||||
// implementations as part of the effective target.
|
||||
BackendID string `json:"backend_id,omitempty"`
|
||||
// Endpoint is the model-provider base URL.
|
||||
Endpoint string `json:"endpoint"`
|
||||
// Model is the provider model identifier.
|
||||
Model string `json:"model"`
|
||||
// Temperature is the resolved sampling temperature from 0 through 2. Zero
|
||||
// leaves the field unspecified to compatible providers unless the
|
||||
// corresponding ExecutionTargetPresence bit is true.
|
||||
Temperature float64 `json:"temperature"`
|
||||
// MaxTokens is the non-negative resolved output-token limit. Zero leaves
|
||||
// the limit unspecified to compatible providers unless the corresponding
|
||||
// ExecutionTargetPresence bit is true.
|
||||
MaxTokens int `json:"max_tokens"`
|
||||
// TopP is the resolved nucleus-sampling value from 0 through 1. Zero leaves
|
||||
// the field unspecified to compatible providers unless the corresponding
|
||||
// ExecutionTargetPresence bit is true.
|
||||
TopP float64 `json:"top_p"`
|
||||
// TimeoutSeconds is the non-negative per-generation deadline. Zero disables
|
||||
// this deadline without disabling caller cancellation or the transport cap.
|
||||
TimeoutSeconds int `json:"timeout_seconds"`
|
||||
// ServiceTier is an optional provider-specific request tier.
|
||||
ServiceTier string `json:"service_tier"`
|
||||
// ReasoningEffort is the effective opaque provider-specific reasoning
|
||||
// setting. An empty value instructs model clients to omit reasoning.
|
||||
ReasoningEffort string `json:"reasoning_effort"`
|
||||
// APIKeyEnv is an environment-variable name, not its credential value.
|
||||
APIKeyEnv string `json:"api_key_env"`
|
||||
// ExtraParams contains copied JSON-compatible provider parameters.
|
||||
ExtraParams map[string]any `json:"extra_params"`
|
||||
}
|
||||
|
||||
// ExecutionTargetOverride represents per-request runtime setting overrides.
|
||||
// ProfileInspection is the caller-owned result of [Engine.InspectProfile].
|
||||
// It has no stable JSON representation.
|
||||
//
|
||||
// EffectiveModelParams contains a copied effective target. APIKeyRequired is
|
||||
// separate from that target to preserve ExecutionTarget's general execution
|
||||
// and stable JSON contracts.
|
||||
type ProfileInspection struct {
|
||||
// ProfileID is the trimmed, exact profile ID inspected by the engine.
|
||||
ProfileID string
|
||||
// EffectiveModelParams contains settings resolved from the framework timeout
|
||||
// baseline, selected backend, and then profile, without a request override.
|
||||
// Unset optional provider controls remain zero rather than reporting a
|
||||
// provider default. APIKeyEnv is an environment-variable name, never its
|
||||
// credential value.
|
||||
EffectiveModelParams ExecutionTarget
|
||||
// APIKeyRequired reports that a later execution must supply a direct API
|
||||
// key or an explicit request environment override. It is mutually exclusive
|
||||
// with a nonblank EffectiveModelParams.APIKeyEnv.
|
||||
APIKeyRequired bool
|
||||
}
|
||||
|
||||
// PromptInputDefinition describes one declared prompt input.
|
||||
// It has no stable JSON representation.
|
||||
type PromptInputDefinition struct {
|
||||
// Name is the normalized prompt input name.
|
||||
Name string
|
||||
// Required is the prompt definition's declared required flag. When true,
|
||||
// preparation fails if the input is omitted. A false value does not account
|
||||
// for input references in message or session-ID templates.
|
||||
Required bool
|
||||
// ContentType is the declared input media-type metadata.
|
||||
ContentType string
|
||||
// Description is the declared human-readable input description.
|
||||
Description string
|
||||
}
|
||||
|
||||
// PromptInspection is the caller-owned result of [Engine.InspectPrompt].
|
||||
// It has no stable JSON representation.
|
||||
//
|
||||
// Inputs contains copied declared input metadata in definition order.
|
||||
// OutputContract is the normalized contract declared by the prompt definition,
|
||||
// rather than a request-level effective override. PromptHash is opaque.
|
||||
type PromptInspection struct {
|
||||
// PromptID is the normalized ID of the selected prompt definition.
|
||||
PromptID string
|
||||
// PromptVersion is the normalized version of the selected prompt definition.
|
||||
PromptVersion string
|
||||
// PromptHash is the opaque equality value for the selected definition.
|
||||
PromptHash string
|
||||
// DefaultProfileID is declared metadata and is not resolved by inspection.
|
||||
DefaultProfileID string
|
||||
// Inputs contains caller-owned declared input metadata in definition order.
|
||||
Inputs []PromptInputDefinition
|
||||
// OutputContract is the normalized contract declared by the definition.
|
||||
OutputContract OutputContract
|
||||
}
|
||||
|
||||
// ExecutionTargetOverride represents per-request runtime setting overrides and
|
||||
// has no stable JSON representation.
|
||||
//
|
||||
// Non-empty string fields replace profile and backend values. Non-nil pointer
|
||||
// fields replace profile values and preserve explicit zero or empty values. A
|
||||
// non-empty ExtraParams map replaces the complete profile or backend map
|
||||
// rather than merging keys. Empty string fields, nil pointers, and a nil or
|
||||
// empty ExtraParams map inherit lower-precedence values. An optional provider
|
||||
// control that remains zero is unspecified; TimeoutSeconds retains its
|
||||
// framework deadline when no higher-precedence value is present.
|
||||
type ExecutionTargetOverride struct {
|
||||
Endpoint string
|
||||
Model string
|
||||
Temperature *float64
|
||||
MaxTokens *int
|
||||
TopP *float64
|
||||
TimeoutSeconds *int
|
||||
ServiceTier string
|
||||
ReasoningEffort string
|
||||
APIKeyEnv string
|
||||
ExtraParams map[string]any
|
||||
// Endpoint replaces the profile or backend endpoint when non-empty without
|
||||
// changing the effective BackendID.
|
||||
Endpoint string
|
||||
// Model replaces the profile model when non-empty.
|
||||
Model string
|
||||
// Temperature, when non-nil, must point to a value from 0 through 2. A
|
||||
// pointed-to zero is explicitly present; nil inherits a lower-precedence
|
||||
// value and otherwise leaves the provider control unspecified.
|
||||
Temperature *float64
|
||||
// MaxTokens, when non-nil, must point to a non-negative value. A pointed-to
|
||||
// zero is explicitly present; nil inherits a lower-precedence value and
|
||||
// otherwise leaves the provider control unspecified.
|
||||
MaxTokens *int
|
||||
// TopP, when non-nil, must point to a value from 0 through 1. A pointed-to
|
||||
// zero is explicitly present; nil inherits a lower-precedence value and
|
||||
// otherwise leaves the provider control unspecified.
|
||||
TopP *float64
|
||||
// TimeoutSeconds, when non-nil, must point to a non-negative value. A
|
||||
// pointed-to zero disables the per-generation deadline.
|
||||
TimeoutSeconds *int
|
||||
// ServiceTier replaces the profile value when non-blank.
|
||||
ServiceTier string
|
||||
// ReasoningEffort controls the per-run reasoning setting. Nil inherits the
|
||||
// profile value. A pointer to a non-blank string trims and replaces the
|
||||
// profile value. A pointer to an empty or whitespace-only string clears the
|
||||
// inherited value and disables reasoning for this run. Non-blank values
|
||||
// are opaque and are not validated against a fixed vocabulary.
|
||||
ReasoningEffort *string
|
||||
// APIKeyEnv replaces the profile or backend environment-variable name when
|
||||
// non-blank. A direct RunRequest.APIKey still takes precedence over
|
||||
// environment lookup.
|
||||
APIKeyEnv string
|
||||
// ExtraParams, when non-empty, replaces the complete profile or backend map.
|
||||
// Values must be JSON-compatible: nil, booleans, finite numbers, strings,
|
||||
// arrays or slices, and maps with non-empty string keys. Cycles are invalid.
|
||||
ExtraParams map[string]any
|
||||
}
|
||||
|
||||
// Profile is an in-memory execution profile for library consumers.
|
||||
//
|
||||
// It is equivalent to a loaded profile file after validation. Raw API keys do
|
||||
// not belong in profiles; use APIKeyRequired to require callers to provide
|
||||
// RunRequest.APIKey for each request, or use profile YAML api_key_env with file
|
||||
// and FS profile sources.
|
||||
// not belong in profiles; use APIKeyRequired to require callers to provide a
|
||||
// RunRequest.APIKey or explicit request ExecutionTargetOverride.APIKeyEnv, or
|
||||
// use profile YAML api_key_env with file and FS profile sources. Profile has no
|
||||
// stable JSON representation.
|
||||
//
|
||||
// WithProfiles validates and copies Profile values during NewEngine. Zero
|
||||
// Temperature, MaxTokens, and TopP values and blank ServiceTier and
|
||||
// ReasoningEffort values leave those provider controls unspecified. A zero
|
||||
// TimeoutSeconds retains the framework deadline, while an empty ExtraParams map
|
||||
// inherits backend request defaults. Use ExecutionTargetOverride pointer fields
|
||||
// to request an explicit numeric zero.
|
||||
type Profile struct {
|
||||
ID string
|
||||
Endpoint string
|
||||
Model string
|
||||
Temperature float64
|
||||
MaxTokens int
|
||||
TopP float64
|
||||
TimeoutSeconds int
|
||||
ServiceTier string
|
||||
// ID is the required non-blank profile identifier. WithProfiles trims it.
|
||||
ID string
|
||||
// BackendID optionally selects an engine backend. WithProfiles trims it.
|
||||
// Backend membership is checked when a request selects the profile; an
|
||||
// unknown ID makes preparation fail with ErrProfileLoad.
|
||||
BackendID string
|
||||
// Endpoint is the model-provider base URL. It is required only when
|
||||
// BackendID is blank and otherwise overrides the backend endpoint when
|
||||
// non-blank.
|
||||
Endpoint string
|
||||
// Model is the required non-blank provider model identifier.
|
||||
Model string
|
||||
// Temperature is from 0 through 2. Zero leaves the provider control
|
||||
// unspecified.
|
||||
Temperature float64
|
||||
// MaxTokens is non-negative. Zero leaves the provider control unspecified.
|
||||
MaxTokens int
|
||||
// TopP is from 0 through 1. Zero leaves the provider control unspecified
|
||||
// rather than selecting an explicit zero.
|
||||
TopP float64
|
||||
// TimeoutSeconds is non-negative. Zero retains the framework deadline.
|
||||
TimeoutSeconds int
|
||||
// ServiceTier is optional; a blank value leaves it unspecified.
|
||||
ServiceTier string
|
||||
// ReasoningEffort is optional; a blank value leaves it unspecified.
|
||||
ReasoningEffort string
|
||||
APIKeyRequired bool
|
||||
ExtraParams map[string]any
|
||||
// APIKeyRequired clears a backend's inherited API-key environment name and
|
||||
// requires a non-blank RunRequest.APIKey unless the request explicitly
|
||||
// supplies ExecutionTargetOverride.APIKeyEnv. It does not store a credential.
|
||||
APIKeyRequired bool
|
||||
// ExtraParams contains provider-specific JSON-compatible values. An empty
|
||||
// map inherits backend request defaults, when any. WithProfiles validates
|
||||
// and deeply copies it during NewEngine.
|
||||
ExtraParams map[string]any
|
||||
}
|
||||
|
||||
// OpenAICompatibleProfileConfig configures an OpenAI-compatible in-memory
|
||||
// profile.
|
||||
//
|
||||
// It contains ordinary profile fields for OpenAI-compatible chat-completions
|
||||
// endpoints. APIKeyRequired is satisfied by RunRequest.APIKey. Raw API keys do
|
||||
// not belong in this config.
|
||||
// endpoints. APIKeyRequired follows Profile.APIKeyRequired. Raw API keys do not
|
||||
// belong in this config. OpenAICompatibleProfileConfig has no stable JSON
|
||||
// representation and is not validated until its resulting Profile is supplied
|
||||
// through WithProfiles to NewEngine.
|
||||
type OpenAICompatibleProfileConfig struct {
|
||||
ID string
|
||||
Endpoint string
|
||||
Model string
|
||||
APIKeyRequired bool
|
||||
Temperature float64
|
||||
MaxTokens int
|
||||
TopP float64
|
||||
TimeoutSeconds int
|
||||
ServiceTier string
|
||||
// ID becomes Profile.ID.
|
||||
ID string
|
||||
// BackendID becomes Profile.BackendID.
|
||||
BackendID string
|
||||
// Endpoint becomes Profile.Endpoint.
|
||||
Endpoint string
|
||||
// Model becomes Profile.Model.
|
||||
Model string
|
||||
// APIKeyRequired becomes Profile.APIKeyRequired.
|
||||
APIKeyRequired bool
|
||||
// Temperature becomes Profile.Temperature.
|
||||
Temperature float64
|
||||
// MaxTokens becomes Profile.MaxTokens.
|
||||
MaxTokens int
|
||||
// TopP becomes Profile.TopP.
|
||||
TopP float64
|
||||
// TimeoutSeconds becomes Profile.TimeoutSeconds.
|
||||
TimeoutSeconds int
|
||||
// ServiceTier becomes Profile.ServiceTier.
|
||||
ServiceTier string
|
||||
// ReasoningEffort becomes Profile.ReasoningEffort.
|
||||
ReasoningEffort string
|
||||
ExtraParams map[string]any
|
||||
// ExtraParams becomes a shallow-copied Profile.ExtraParams map. NewEngine
|
||||
// performs validation and a deep copy when WithProfiles applies the result.
|
||||
ExtraParams map[string]any
|
||||
}
|
||||
|
||||
// ExecutionTargetPresence tracks which numeric runtime settings were explicit
|
||||
// request overrides.
|
||||
// request overrides, including explicit zero values. It has a stable JSON
|
||||
// representation and is supplied to injected LLM clients so they can preserve
|
||||
// omission semantics.
|
||||
type ExecutionTargetPresence struct {
|
||||
Temperature bool
|
||||
MaxTokens bool
|
||||
TopP bool
|
||||
TimeoutSeconds bool
|
||||
// Temperature reports a non-nil ExecutionTargetOverride.Temperature.
|
||||
Temperature bool `json:"temperature"`
|
||||
// MaxTokens reports a non-nil ExecutionTargetOverride.MaxTokens.
|
||||
MaxTokens bool `json:"max_tokens"`
|
||||
// TopP reports a non-nil ExecutionTargetOverride.TopP.
|
||||
TopP bool `json:"top_p"`
|
||||
// TimeoutSeconds reports a non-nil ExecutionTargetOverride.TimeoutSeconds.
|
||||
TimeoutSeconds bool `json:"timeout_seconds"`
|
||||
}
|
||||
|
||||
// OutputContract defines output and validation requirements.
|
||||
// OutputContract defines output and validation requirements and has a stable
|
||||
// JSON representation.
|
||||
//
|
||||
// A non-nil RunRequest.Validation replaces the complete prompt contract. It
|
||||
// does not merge fields. The public Engine validates generated output once and
|
||||
// does not install an output repairer.
|
||||
type OutputContract struct {
|
||||
Format OutputFormat `json:"format"`
|
||||
// Format selects generated artifact metadata. An empty effective value
|
||||
// defaults to FormatText.
|
||||
Format OutputFormat `json:"format"`
|
||||
// ValidationMode selects the content check. Use one of the declared
|
||||
// ValidationMode constants.
|
||||
ValidationMode ValidationMode `json:"validation_mode"`
|
||||
SchemaPath string `json:"schema_path"`
|
||||
RepairAttempts int `json:"repair_attempts"`
|
||||
// SchemaPath is required when ValidationMode is ValidationJSONSchema and is
|
||||
// ignored by other modes.
|
||||
SchemaPath string `json:"schema_path"`
|
||||
// RepairAttempts is a requested repair limit. A non-positive value requests
|
||||
// no repairs. The public Engine performs no repairs even when this value is
|
||||
// positive, so its runs report zero attempts used.
|
||||
RepairAttempts int `json:"repair_attempts"`
|
||||
}
|
||||
|
||||
// ValidationResult represents output validation state.
|
||||
// ValidationResult represents a completed output check and has a stable JSON
|
||||
// representation. An operational inability to perform validation is returned
|
||||
// as ErrValidation instead of a ValidationResult.
|
||||
type ValidationResult struct {
|
||||
Status ValidationStatus `json:"status"`
|
||||
Mode ValidationMode `json:"mode"`
|
||||
Errors []string `json:"errors,omitempty"`
|
||||
SchemaPath string `json:"schema_path,omitempty"`
|
||||
RepairAttempts int `json:"repair_attempts"`
|
||||
IsValid bool `json:"is_valid"`
|
||||
// Status is Passed, Failed, or Skipped.
|
||||
Status ValidationStatus `json:"status"`
|
||||
// Mode is the effective validation mode.
|
||||
Mode ValidationMode `json:"mode"`
|
||||
// Errors contains validation diagnostics when Status is ValidationFailed.
|
||||
Errors []string `json:"errors,omitempty"`
|
||||
// SchemaPath is the effective schema path for JSON Schema validation.
|
||||
SchemaPath string `json:"schema_path,omitempty"`
|
||||
// RepairAttempts is the number of repairs actually attempted. It is always
|
||||
// zero for the public Engine.
|
||||
RepairAttempts int `json:"repair_attempts"`
|
||||
// IsValid is true for ValidationPassed and ValidationSkipped and false for
|
||||
// ValidationFailed.
|
||||
IsValid bool `json:"is_valid"`
|
||||
}
|
||||
|
||||
// TokenUsage tracks token consumption.
|
||||
// TokenUsage contains model-client token accounting and has a stable JSON
|
||||
// representation. Promptkit preserves values reported by the client and does
|
||||
// not derive or reconcile them.
|
||||
type TokenUsage struct {
|
||||
PromptTokens int `json:"prompt_tokens"`
|
||||
// PromptTokens is the reported input-token count.
|
||||
PromptTokens int `json:"prompt_tokens"`
|
||||
// CompletionTokens is the reported generated-token count.
|
||||
CompletionTokens int `json:"completion_tokens"`
|
||||
TotalTokens int `json:"total_tokens"`
|
||||
CachedTokens int `json:"cached_tokens"`
|
||||
// TotalTokens is the reported total-token count.
|
||||
TotalTokens int `json:"total_tokens"`
|
||||
// CachedTokens is the reported cached-input-token count.
|
||||
CachedTokens int `json:"cached_tokens"`
|
||||
// CacheWriteTokens is the reported cache-write-token count.
|
||||
CacheWriteTokens int `json:"cache_write_tokens"`
|
||||
}
|
||||
|
||||
// RenderedPrompt is the fully rendered prompt passed to an LLM client.
|
||||
// RenderedPrompt is the fully rendered prompt passed to an LLM client and has
|
||||
// a stable JSON representation.
|
||||
type RenderedPrompt struct {
|
||||
SessionID string `json:"session_id,omitempty"`
|
||||
Messages []RenderedMessage `json:"messages"`
|
||||
// SessionID is the optional effective direct or rendered session
|
||||
// identifier supplied to the model client.
|
||||
SessionID string `json:"session_id,omitempty"`
|
||||
// Messages contains rendered messages in definition order.
|
||||
Messages []RenderedMessage `json:"messages"`
|
||||
}
|
||||
|
||||
// RenderedMessage is a rendered chat message.
|
||||
// RenderedMessage is a rendered chat message and has a stable JSON
|
||||
// representation.
|
||||
type RenderedMessage struct {
|
||||
Role string `json:"role"`
|
||||
Content string `json:"content"`
|
||||
// Role is the definition-supplied chat role.
|
||||
Role string `json:"role"`
|
||||
// Content is the rendered message text.
|
||||
Content string `json:"content"`
|
||||
// CacheControl is optional provider cache metadata.
|
||||
CacheControl *CacheControl `json:"cache_control,omitempty"`
|
||||
}
|
||||
|
||||
// CacheControl describes provider cache metadata attached to prompt content.
|
||||
// CacheControl describes provider cache metadata attached to prompt content
|
||||
// and has a stable JSON representation.
|
||||
type CacheControl struct {
|
||||
// Type identifies the cache behavior.
|
||||
Type CacheControlType `json:"type"`
|
||||
TTL string `json:"ttl,omitempty"`
|
||||
// TTL is an optional provider cache lifetime.
|
||||
TTL string `json:"ttl,omitempty"`
|
||||
}
|
||||
|
||||
// StructuredOutputSpec describes provider-level structured output.
|
||||
// StructuredOutputSpec describes provider-level structured output and has a
|
||||
// stable JSON representation.
|
||||
type StructuredOutputSpec struct {
|
||||
Type StructuredOutputType `json:"type"`
|
||||
// Type identifies the structured-output mechanism.
|
||||
Type StructuredOutputType `json:"type"`
|
||||
// JSONSchema contains constraints when Type is StructuredOutputJSONSchema.
|
||||
JSONSchema *StructuredOutputJSONSpec `json:"json_schema,omitempty"`
|
||||
}
|
||||
|
||||
// StructuredOutputJSONSpec contains JSON Schema output constraints.
|
||||
// StructuredOutputJSONSpec contains provider-facing JSON Schema output
|
||||
// constraints and has a stable JSON representation.
|
||||
type StructuredOutputJSONSpec struct {
|
||||
Name string `json:"name"`
|
||||
Strict bool `json:"strict"`
|
||||
Schema any `json:"schema"`
|
||||
// Name is the provider-facing schema name.
|
||||
Name string `json:"name"`
|
||||
// Strict requests strict provider enforcement of Schema.
|
||||
Strict bool `json:"strict"`
|
||||
// Schema is a caller-owned copy of the loaded JSON Schema document.
|
||||
Schema any `json:"schema"`
|
||||
}
|
||||
|
||||
// LLMClient executes rendered prompts for Engine.Run.
|
||||
// LLMClient executes rendered prompts for [Engine.Run] and
|
||||
// [Engine.RunPrepared].
|
||||
//
|
||||
// Generate is scheduled according to the resolved backend's capacity policy.
|
||||
// It may still be called concurrently for different backend pools or unlimited
|
||||
// backends. Cancellation while waiting for capacity can prevent Generate from
|
||||
// being called. Once invoked, it must honor context cancellation to make Run
|
||||
// and RunPrepared responsive to cancellation. The request and all nested maps,
|
||||
// slices, and pointers are client-owned copies and may be mutated or retained
|
||||
// without affecting engine state.
|
||||
//
|
||||
// Generate receives rendered messages and may receive a direct API key. A
|
||||
// client must protect those values and any raw output in its logging, storage,
|
||||
// and retained copies. It is responsible for the cancellation behavior of any
|
||||
// work it starts and for synchronizing access to retained or shared data.
|
||||
//
|
||||
// A returned error makes Run or RunPrepared return ErrLLMGenerate while
|
||||
// preserving the client error through errors.Is. A nil response with a nil
|
||||
// error also produces ErrLLMGenerate. Promptkit copies the non-nil response
|
||||
// before returning from either method.
|
||||
type LLMClient interface {
|
||||
Generate(context.Context, GenerateRequest) (*GenerateResponse, error)
|
||||
}
|
||||
|
||||
// GenerateRequest is passed to an injected LLM client.
|
||||
// GenerateRequest is passed to an injected LLM client and has a stable JSON
|
||||
// representation. Its String and GoString methods omit rendered content and
|
||||
// direct credentials.
|
||||
type GenerateRequest struct {
|
||||
Prompt RenderedPrompt `json:"prompt"`
|
||||
Target ExecutionTarget `json:"target"`
|
||||
TargetPresence ExecutionTargetPresence `json:"target_presence"`
|
||||
StructuredOutput *StructuredOutputSpec `json:"structured_output,omitempty"`
|
||||
APIKey string `json:"-"`
|
||||
// Prompt contains the rendered session ID and messages.
|
||||
Prompt RenderedPrompt `json:"prompt"`
|
||||
// Target contains effective model settings without the direct API key.
|
||||
Target ExecutionTarget `json:"target"`
|
||||
// TargetPresence distinguishes inherited numeric zeros from explicit
|
||||
// request overrides.
|
||||
TargetPresence ExecutionTargetPresence `json:"target_presence"`
|
||||
// StructuredOutput contains provider response constraints when requested.
|
||||
StructuredOutput *StructuredOutputSpec `json:"structured_output,omitempty"`
|
||||
// APIKey is the direct request-scoped credential, if any. It is excluded
|
||||
// from JSON, String, and GoString output.
|
||||
APIKey string `json:"-"`
|
||||
}
|
||||
|
||||
// GenerateResponse is returned by an injected LLM client.
|
||||
// GenerateResponse is returned by an injected LLM client and has a stable JSON
|
||||
// representation.
|
||||
type GenerateResponse struct {
|
||||
Content string `json:"content"`
|
||||
Usage TokenUsage `json:"usage"`
|
||||
// Content is the generated output. It must be non-empty when using the
|
||||
// built-in client; injected clients may return empty content for Promptkit
|
||||
// validation to classify.
|
||||
Content string `json:"content"`
|
||||
// Usage is the client's token accounting.
|
||||
Usage TokenUsage `json:"usage"`
|
||||
}
|
||||
|
||||
// File returns a file-backed artifact reference.
|
||||
// File returns a file-backed artifact reference whose URI is path.
|
||||
//
|
||||
// The default artifact reader opens path as a caller-selected operating-system
|
||||
// path without restricting it to an application root or imposing a size limit.
|
||||
// Applications accepting untrusted paths must validate them before calling
|
||||
// Promptkit or use [WithArtifactReader] to enforce application policy.
|
||||
func File(path string) ArtifactRef {
|
||||
return ArtifactRef{Type: ArtifactRefFile, URI: path}
|
||||
}
|
||||
|
||||
// Inline returns an inline artifact reference.
|
||||
// Inline returns an inline artifact reference whose Body is body and whose URI
|
||||
// is empty.
|
||||
func Inline(body string) ArtifactRef {
|
||||
return ArtifactRef{Type: ArtifactRefInline, Body: body}
|
||||
}
|
||||
|
||||
// InlineWithURI returns an inline artifact reference with URI metadata.
|
||||
// InlineWithURI returns an inline artifact reference with body content and uri
|
||||
// provenance metadata.
|
||||
func InlineWithURI(uri string, body string) ArtifactRef {
|
||||
return ArtifactRef{Type: ArtifactRefInline, URI: uri, Body: body}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user