Compare commits
95 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 4ca3be2c14 | |||
| 227fb35f99 | |||
| e291b8bfe9 | |||
| 2b6a7f83c4 | |||
| 3a43550f70 | |||
| c281f721bc | |||
| 350b0e76d9 | |||
| e43350fd0d | |||
| 20d3e3b5ee | |||
| a93b799236 | |||
| e83a3ce179 | |||
| a04a3bbc5f | |||
| 731b66cff5 | |||
| 70e0ea0cf0 | |||
| d45c474c1e | |||
| 25f1ba0b30 | |||
| a718762da1 | |||
| 58ac3ce298 | |||
| 57f2ce1ce4 | |||
| c8b6d5c490 | |||
| abeb50b525 | |||
| 1cb07c7d91 | |||
| 8cfc71c351 | |||
| 5ccfa4a345 | |||
| 14e03f19d0 | |||
| 7c562a9374 | |||
| ef97d85ac9 | |||
| 805e48c873 | |||
| e9e126dcba | |||
| 32e7a3557c | |||
| 1d1b04e2e0 | |||
| 9748897751 | |||
| 9d020039d5 | |||
| 5247ce0b73 | |||
| c434aa1dae | |||
| ac9b3f3d80 | |||
| 4f12a89a1b | |||
| df31e7f58e | |||
| 0678d242b9 | |||
| 3b4ea21208 | |||
| 1430e85147 | |||
| 34d7a19da5 | |||
| ebf1602635 | |||
| 31f2ce3a09 | |||
| fd06e4ca6b | |||
| e63b8de1e9 | |||
| 9354d2b373 | |||
| 01ca5430bd | |||
| ae2179d103 | |||
| a248433d0f | |||
| bd6cffc9d0 | |||
| e40c4f182b | |||
| 7428e50c2c | |||
| 63c67a4520 | |||
| 25a7052a3d | |||
| fc3255967e | |||
| e920168b30 | |||
| 272b6a4bc1 | |||
| dde48a31fc | |||
| 242eace4a7 | |||
| 0bf5f88136 | |||
| 369ab5392d | |||
| 2ba0146e5d | |||
| 6112c2af0c | |||
| f5e12c00f5 | |||
| 49fe402dd2 | |||
| c301eb8d55 | |||
| c13e9710d9 | |||
| 87b5ec3d75 | |||
| cb4028a637 | |||
| 5a1bff4529 | |||
| 805a7f965d | |||
| 147f5e5ff5 | |||
| e361c97bb5 | |||
| be67707582 | |||
| e61ab700c7 | |||
| d2c4051dd0 | |||
| 861da355d8 | |||
| a752f88166 | |||
| 238fa90bfa | |||
| ffe6d261a9 | |||
| dc39562ff7 | |||
| 0a839aa16d | |||
| f6ee18f6b3 | |||
| eb8ab215e8 | |||
| f89cb94ed2 | |||
| 359b7313f4 | |||
| ae210b3c26 | |||
| 810f80e7c9 | |||
| 8d00354c59 | |||
| d0010689f3 | |||
| b462153483 | |||
| 086cf0fc86 | |||
| c1cecb1ee8 | |||
| bcb327f643 |
22
README.md
22
README.md
@@ -31,4 +31,26 @@ Contributors should start with the [development guide](docs/development.md).
|
|||||||
The [architecture policy](docs/policy/architecture.md) defines the library
|
The [architecture policy](docs/policy/architecture.md) defines the library
|
||||||
boundary and constraints that framework work must preserve.
|
boundary and constraints that framework work must preserve.
|
||||||
|
|
||||||
|
## Release Guidance
|
||||||
|
|
||||||
|
Consumers upgrading from `v0.5.0` to `v0.6.0` should read the
|
||||||
|
[v0.6.0 changelog and migration guide](docs/releases/v0.6.0.md).
|
||||||
|
|
||||||
|
Earlier adopters can consult the
|
||||||
|
[v0.5.0 changelog and migration guide](docs/releases/v0.5.0.md).
|
||||||
|
|
||||||
|
Consumers upgrading from `v0.3.0` to `v0.4.0` should read the
|
||||||
|
[v0.4.0 changelog and adoption guide](docs/releases/v0.4.0.md).
|
||||||
|
|
||||||
|
Consumers upgrading from `v0.2.0` to `v0.3.0` should read the
|
||||||
|
[v0.3.0 changelog](docs/releases/v0.3.0.md).
|
||||||
|
|
||||||
|
Consumers moving from `v0.1.0` to `v0.2.0` should read the
|
||||||
|
[v0.2.0 changelog and migration guide](docs/releases/v0.2.0.md).
|
||||||
|
|
||||||
|
## Related Project
|
||||||
|
|
||||||
|
[Scriptorium](https://gitea.maximumdirect.net/eric/scriptorium) is the CLI and
|
||||||
|
HTTP application built on Promptkit.
|
||||||
|
|
||||||
Promptkit is licensed under the [GNU General Public License version 3](LICENSE).
|
Promptkit is licensed under the [GNU General Public License version 3](LICENSE).
|
||||||
|
|||||||
100
backends.go
Normal file
100
backends.go
Normal file
@@ -0,0 +1,100 @@
|
|||||||
|
package promptkit
|
||||||
|
|
||||||
|
import (
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/backend"
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||||
|
)
|
||||||
|
|
||||||
|
// BackendOpenRouter is the reserved ID of Promptkit's built-in OpenRouter
|
||||||
|
// backend.
|
||||||
|
const BackendOpenRouter = backend.OpenRouterID
|
||||||
|
|
||||||
|
// BackendLocal is the case-sensitive conventional ID used by [LocalBackend].
|
||||||
|
// It is not a built-in or reserved backend and must be registered with
|
||||||
|
// [WithBackend].
|
||||||
|
const BackendLocal = "local"
|
||||||
|
|
||||||
|
// Backend configures one engine-scoped OpenAI-compatible backend.
|
||||||
|
//
|
||||||
|
// Backend has no stable JSON representation. Use keyed literals so additions
|
||||||
|
// to this configuration value do not break source compatibility.
|
||||||
|
type Backend struct {
|
||||||
|
// ID is the stable, case-sensitive registry key. NewEngine trims it and
|
||||||
|
// requires a non-blank value. BackendOpenRouter is reserved.
|
||||||
|
ID string
|
||||||
|
// Endpoint is the OpenAI-compatible base endpoint. NewEngine trims it and
|
||||||
|
// requires an absolute HTTP or HTTPS URL with a host and without user
|
||||||
|
// information, a query string, or a fragment. Paths are allowed.
|
||||||
|
Endpoint string
|
||||||
|
// APIKeyEnv optionally names the environment variable containing the API
|
||||||
|
// key. NewEngine trims it and requires the portable form
|
||||||
|
// [A-Za-z_][A-Za-z0-9_]*. Store only the name, never a credential value.
|
||||||
|
APIKeyEnv string
|
||||||
|
// ExtraParams contains backend-wide request defaults. Values must be
|
||||||
|
// JSON-compatible, finite, acyclic, and keyed by non-empty strings. Keys
|
||||||
|
// must not be model, session_id, messages, temperature, max_tokens, top_p,
|
||||||
|
// service_tier, reasoning_effort, or response_format. An empty map supplies
|
||||||
|
// no defaults. NewEngine deeply copies the map and rejects excessively deep
|
||||||
|
// or large values for safety.
|
||||||
|
ExtraParams map[string]any
|
||||||
|
// ConcurrencyLimit is the maximum number of simultaneous model-generation
|
||||||
|
// calls allowed for this backend within one Engine. Zero leaves the backend
|
||||||
|
// unlimited. A negative value makes NewEngine fail with ErrInvalidConfig.
|
||||||
|
ConcurrencyLimit int
|
||||||
|
// QueueCapacity controls how many additional Run or RunPrepared calls may
|
||||||
|
// be admitted beyond ConcurrencyLimit. Nil uses 1024 when ConcurrencyLimit
|
||||||
|
// is positive; a pointer uses its exact value, including zero. The pointed-to
|
||||||
|
// value must be non-negative, and QueueCapacity must be nil when
|
||||||
|
// ConcurrencyLimit is zero. Their sum must fit in an int. WithBackend copies
|
||||||
|
// the value and does not retain the pointer.
|
||||||
|
QueueCapacity *int
|
||||||
|
}
|
||||||
|
|
||||||
|
// LocalBackend returns a caller-owned Backend for a conventional local
|
||||||
|
// OpenAI-compatible endpoint. It sets ID to BackendLocal and copies endpoint
|
||||||
|
// and concurrencyLimit into Endpoint and ConcurrencyLimit without
|
||||||
|
// normalization or validation. APIKeyEnv, ExtraParams, and QueueCapacity keep
|
||||||
|
// their zero values.
|
||||||
|
//
|
||||||
|
// LocalBackend does not read environment variables, register the value, or
|
||||||
|
// mutate engine or package state. Supply the returned value through
|
||||||
|
// [WithBackend]; [NewEngine] then applies the ordinary backend validation and
|
||||||
|
// concurrency semantics, including default queue capacity for a positive
|
||||||
|
// limit, unlimited behavior for zero, and ErrInvalidConfig for a negative
|
||||||
|
// limit.
|
||||||
|
func LocalBackend(endpoint string, concurrencyLimit int) Backend {
|
||||||
|
return Backend{
|
||||||
|
ID: BackendLocal,
|
||||||
|
Endpoint: endpoint,
|
||||||
|
ConcurrencyLimit: concurrencyLimit,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// WithBackend adds one Backend registration to the constructed Engine.
|
||||||
|
//
|
||||||
|
// Registrations accumulate in option order. Every normalized ID must be unique
|
||||||
|
// across consumer registrations and built-ins; a duplicate or invalid
|
||||||
|
// definition makes NewEngine fail with ErrInvalidConfig. In particular,
|
||||||
|
// BackendOpenRouter cannot be replaced. The immutable registration is scoped
|
||||||
|
// to the resulting Engine and cannot be enumerated, replaced, removed, or
|
||||||
|
// mutated after construction. WithBackend does not install package-global
|
||||||
|
// state.
|
||||||
|
func WithBackend(backend Backend) Option {
|
||||||
|
queueCapacity := 0
|
||||||
|
queueCapacitySet := backend.QueueCapacity != nil
|
||||||
|
if queueCapacitySet {
|
||||||
|
queueCapacity = *backend.QueueCapacity
|
||||||
|
}
|
||||||
|
return optionFunc(func(options *engineOptions) error {
|
||||||
|
options.backends = append(options.backends, domain.Backend{
|
||||||
|
ID: backend.ID,
|
||||||
|
Endpoint: backend.Endpoint,
|
||||||
|
APIKeyEnv: backend.APIKeyEnv,
|
||||||
|
ExtraParams: backend.ExtraParams,
|
||||||
|
ConcurrencyLimit: backend.ConcurrencyLimit,
|
||||||
|
QueueCapacity: queueCapacity,
|
||||||
|
QueueCapacitySet: queueCapacitySet,
|
||||||
|
})
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
}
|
||||||
409
capacity_contract_test.go
Normal file
409
capacity_contract_test.go
Normal file
@@ -0,0 +1,409 @@
|
|||||||
|
package promptkit_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestEngineLimitsInjectedClientConcurrency(t *testing.T) {
|
||||||
|
release := make(chan struct{})
|
||||||
|
client := newCapacityGateClient(release, 8)
|
||||||
|
engine := newBackendCapacityEngine(t, client, 2, capacityInt(4), nil)
|
||||||
|
|
||||||
|
results := make(chan capacityRunResult, 6)
|
||||||
|
for i := 0; i < 6; i++ {
|
||||||
|
go runCapacityRequest(engine, context.Background(), promptkit.RunRequest{
|
||||||
|
PromptID: "prompt",
|
||||||
|
Execution: &promptkit.ExecutionTargetOverride{
|
||||||
|
Endpoint: "http://request.example/v1",
|
||||||
|
},
|
||||||
|
}, results)
|
||||||
|
}
|
||||||
|
|
||||||
|
first := awaitCapacityRequest(t, client.started)
|
||||||
|
second := awaitCapacityRequest(t, client.started)
|
||||||
|
if first.Target.BackendID != "limited" || second.Target.BackendID != "limited" {
|
||||||
|
t.Fatalf("endpoint override changed backend pool: first=%q second=%q",
|
||||||
|
first.Target.BackendID, second.Target.BackendID)
|
||||||
|
}
|
||||||
|
if active, peak, _ := client.snapshot(); active != 2 || peak != 2 {
|
||||||
|
t.Fatalf("client concurrency before release=(active=%d peak=%d), want 2", active, peak)
|
||||||
|
}
|
||||||
|
|
||||||
|
close(release)
|
||||||
|
for i := 0; i < 6; i++ {
|
||||||
|
outcome := awaitCapacityRun(t, results)
|
||||||
|
if outcome.err != nil || outcome.result == nil {
|
||||||
|
t.Fatalf("run outcome=(%+v, %v), want success", outcome.result, outcome.err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if _, peak, calls := client.snapshot(); peak > 2 || calls != 6 {
|
||||||
|
t.Fatalf("client observations=(peak=%d calls=%d), want peak <= 2 and 6 calls", peak, calls)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEngineRejectsRunBeforeCompletionWhenAdmissionIsFull(t *testing.T) {
|
||||||
|
artifactRelease := make(chan struct{})
|
||||||
|
reader := &capacityArtifactReader{
|
||||||
|
entered: make(chan struct{}, 2),
|
||||||
|
release: artifactRelease,
|
||||||
|
}
|
||||||
|
client := newCapacityGateClient(closedCapacityChannel(), 2)
|
||||||
|
engine := newBackendCapacityEngine(t, client, 1, capacityInt(0), reader)
|
||||||
|
firstResult := make(chan capacityRunResult, 1)
|
||||||
|
go runCapacityRequest(engine, context.Background(), capacityInputRequest("http://first.example/v1"), firstResult)
|
||||||
|
|
||||||
|
awaitCapacitySignal(t, reader.entered, "first artifact read")
|
||||||
|
|
||||||
|
canceledContext, cancel := context.WithCancel(context.Background())
|
||||||
|
cancel()
|
||||||
|
result, err := engine.Run(canceledContext, capacityInputRequest("http://canceled.example/v1"))
|
||||||
|
if result != nil || !errors.Is(err, context.Canceled) {
|
||||||
|
t.Fatalf("canceled capacity admission=(%+v, %v), want context cancellation", result, err)
|
||||||
|
}
|
||||||
|
var canceledCapacityErr *promptkit.CapacityError
|
||||||
|
if errors.Is(err, promptkit.ErrCapacityExceeded) || errors.As(err, &canceledCapacityErr) {
|
||||||
|
t.Fatalf("canceled admission exposed capacity rejection: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
result, err = engine.Run(context.Background(), capacityInputRequest("http://second.example/v1"))
|
||||||
|
if result != nil {
|
||||||
|
t.Fatalf("capacity rejection returned partial result: %+v", result)
|
||||||
|
}
|
||||||
|
if !errors.Is(err, promptkit.ErrCapacityExceeded) {
|
||||||
|
t.Fatalf("capacity rejection=%v, want ErrCapacityExceeded", err)
|
||||||
|
}
|
||||||
|
if errors.Is(err, promptkit.ErrInvalidRequest) || errors.Is(err, promptkit.ErrLLMGenerate) {
|
||||||
|
t.Fatalf("capacity rejection had an unrelated category: %v", err)
|
||||||
|
}
|
||||||
|
var capacityErr *promptkit.CapacityError
|
||||||
|
if !errors.As(err, &capacityErr) || capacityErr == nil {
|
||||||
|
t.Fatalf("capacity rejection=%v, want CapacityError", err)
|
||||||
|
}
|
||||||
|
if capacityErr.BackendID != "limited" {
|
||||||
|
t.Fatalf("capacity backend ID=%q, want limited", capacityErr.BackendID)
|
||||||
|
}
|
||||||
|
capacityErr.BackendID = "changed"
|
||||||
|
|
||||||
|
result, err = engine.Run(context.Background(), capacityInputRequest("http://third.example/v1"))
|
||||||
|
var subsequentCapacityErr *promptkit.CapacityError
|
||||||
|
if result != nil || !errors.As(err, &subsequentCapacityErr) ||
|
||||||
|
subsequentCapacityErr == nil || subsequentCapacityErr.BackendID != "limited" {
|
||||||
|
t.Fatalf("subsequent capacity rejection=(%+v, %v), want independent limited CapacityError", result, err)
|
||||||
|
}
|
||||||
|
if calls := reader.callCount(); calls != 1 {
|
||||||
|
t.Fatalf("artifact calls=%d, want only the admitted run", calls)
|
||||||
|
}
|
||||||
|
if _, _, calls := client.snapshot(); calls != 0 {
|
||||||
|
t.Fatalf("client calls=%d before admitted run was released, want 0", calls)
|
||||||
|
}
|
||||||
|
|
||||||
|
close(artifactRelease)
|
||||||
|
outcome := awaitCapacityRun(t, firstResult)
|
||||||
|
if outcome.err != nil || outcome.result == nil {
|
||||||
|
t.Fatalf("first run outcome=(%+v, %v), want success", outcome.result, outcome.err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBackendCapacityIsIndependentBetweenEngines(t *testing.T) {
|
||||||
|
firstRelease := make(chan struct{})
|
||||||
|
firstClient := newCapacityGateClient(firstRelease, 1)
|
||||||
|
firstEngine := newBackendCapacityEngine(t, firstClient, 1, capacityInt(0), nil)
|
||||||
|
secondClient := newCapacityGateClient(closedCapacityChannel(), 1)
|
||||||
|
secondEngine := newBackendCapacityEngine(t, secondClient, 1, capacityInt(0), nil)
|
||||||
|
|
||||||
|
firstResult := make(chan capacityRunResult, 1)
|
||||||
|
go runCapacityRequest(firstEngine, context.Background(), promptkit.RunRequest{PromptID: "prompt"}, firstResult)
|
||||||
|
awaitCapacityRequest(t, firstClient.started)
|
||||||
|
|
||||||
|
result, err := secondEngine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
|
||||||
|
if err != nil || result == nil {
|
||||||
|
t.Fatalf("second engine run=(%+v, %v), want independent success", result, err)
|
||||||
|
}
|
||||||
|
if _, _, calls := secondClient.snapshot(); calls != 1 {
|
||||||
|
t.Fatalf("second engine client calls=%d, want 1", calls)
|
||||||
|
}
|
||||||
|
|
||||||
|
close(firstRelease)
|
||||||
|
outcome := awaitCapacityRun(t, firstResult)
|
||||||
|
if outcome.err != nil || outcome.result == nil {
|
||||||
|
t.Fatalf("first engine run=(%+v, %v), want success", outcome.result, outcome.err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestUnlimitedBackendsRetainInjectedClientConcurrency(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
configure func(*testing.T, promptkit.LLMClient) *promptkit.Engine
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "custom backend",
|
||||||
|
configure: func(t *testing.T, client promptkit.LLMClient) *promptkit.Engine {
|
||||||
|
return newBackendCapacityEngine(t, client, 0, nil, nil)
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "endpoint-only profile",
|
||||||
|
configure: func(t *testing.T, client promptkit.LLMClient) *promptkit.Engine {
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{},
|
||||||
|
promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."),
|
||||||
|
promptkit.WithProfiles(promptkit.Profile{
|
||||||
|
ID: "profile", Endpoint: "http://endpoint.example/v1", Model: "model",
|
||||||
|
}),
|
||||||
|
promptkit.WithLLMClient(client),
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("construct endpoint-only engine: %v", err)
|
||||||
|
}
|
||||||
|
return engine
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
release := make(chan struct{})
|
||||||
|
client := newCapacityGateClient(release, 2)
|
||||||
|
engine := tc.configure(t, client)
|
||||||
|
results := make(chan capacityRunResult, 2)
|
||||||
|
for i := 0; i < 2; i++ {
|
||||||
|
go runCapacityRequest(
|
||||||
|
engine,
|
||||||
|
context.Background(),
|
||||||
|
promptkit.RunRequest{PromptID: "prompt"},
|
||||||
|
results,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
awaitCapacityRequest(t, client.started)
|
||||||
|
awaitCapacityRequest(t, client.started)
|
||||||
|
if active, peak, _ := client.snapshot(); active != 2 || peak != 2 {
|
||||||
|
t.Fatalf("unlimited concurrency=(active=%d peak=%d), want 2", active, peak)
|
||||||
|
}
|
||||||
|
close(release)
|
||||||
|
for i := 0; i < 2; i++ {
|
||||||
|
outcome := awaitCapacityRun(t, results)
|
||||||
|
if outcome.err != nil || outcome.result == nil {
|
||||||
|
t.Fatalf("run outcome=(%+v, %v), want success", outcome.result, outcome.err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCapacityExceededSentinelContract(t *testing.T) {
|
||||||
|
if promptkit.ErrCapacityExceeded == nil {
|
||||||
|
t.Fatal("ErrCapacityExceeded is nil")
|
||||||
|
}
|
||||||
|
var nilCapacityErr *promptkit.CapacityError
|
||||||
|
zeroCapacityErr := &promptkit.CapacityError{}
|
||||||
|
populatedCapacityErr := &promptkit.CapacityError{BackendID: "limited"}
|
||||||
|
for _, capacityErr := range []error{nilCapacityErr, zeroCapacityErr} {
|
||||||
|
if !errors.Is(capacityErr, promptkit.ErrCapacityExceeded) {
|
||||||
|
t.Fatalf("capacity error=%v, want ErrCapacityExceeded", capacityErr)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
var discoveredCapacityErr *promptkit.CapacityError
|
||||||
|
if !errors.As(populatedCapacityErr, &discoveredCapacityErr) || discoveredCapacityErr != populatedCapacityErr {
|
||||||
|
t.Fatalf("populated capacity error is not discoverable: %v", populatedCapacityErr)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, unrelated := range []error{
|
||||||
|
promptkit.ErrInvalidConfig,
|
||||||
|
promptkit.ErrInvalidRequest,
|
||||||
|
promptkit.ErrLLMGenerate,
|
||||||
|
promptkit.ErrValidation,
|
||||||
|
} {
|
||||||
|
if errors.Is(promptkit.ErrCapacityExceeded, unrelated) ||
|
||||||
|
errors.Is(unrelated, promptkit.ErrCapacityExceeded) ||
|
||||||
|
errors.Is(populatedCapacityErr, unrelated) {
|
||||||
|
t.Fatalf("ErrCapacityExceeded aliases unrelated sentinel %v", unrelated)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type capacityRunResult struct {
|
||||||
|
result *promptkit.RunResult
|
||||||
|
err error
|
||||||
|
}
|
||||||
|
|
||||||
|
func runCapacityRequest(
|
||||||
|
engine *promptkit.Engine,
|
||||||
|
ctx context.Context,
|
||||||
|
request promptkit.RunRequest,
|
||||||
|
results chan<- capacityRunResult,
|
||||||
|
) {
|
||||||
|
result, err := engine.Run(ctx, request)
|
||||||
|
results <- capacityRunResult{result: result, err: err}
|
||||||
|
}
|
||||||
|
|
||||||
|
func newBackendCapacityEngine(
|
||||||
|
t *testing.T,
|
||||||
|
client promptkit.LLMClient,
|
||||||
|
limit int,
|
||||||
|
queueCapacity *int,
|
||||||
|
reader promptkit.ArtifactReader,
|
||||||
|
) *promptkit.Engine {
|
||||||
|
t.Helper()
|
||||||
|
promptFS := contractPromptFS("prompt", "profile", "message")
|
||||||
|
if reader != nil {
|
||||||
|
promptFS = contractInputPromptFS()
|
||||||
|
}
|
||||||
|
options := []promptkit.Option{
|
||||||
|
promptkit.WithPromptFS(promptFS, "."),
|
||||||
|
promptkit.WithBackend(promptkit.Backend{
|
||||||
|
ID: "limited",
|
||||||
|
Endpoint: "http://backend.example/v1",
|
||||||
|
ConcurrencyLimit: limit,
|
||||||
|
QueueCapacity: queueCapacity,
|
||||||
|
}),
|
||||||
|
promptkit.WithProfiles(promptkit.Profile{
|
||||||
|
ID: "profile", BackendID: "limited", Model: "model",
|
||||||
|
}),
|
||||||
|
promptkit.WithLLMClient(client),
|
||||||
|
}
|
||||||
|
if reader != nil {
|
||||||
|
options = append(options, promptkit.WithArtifactReader(reader))
|
||||||
|
}
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{}, options...)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("construct capacity engine: %v", err)
|
||||||
|
}
|
||||||
|
return engine
|
||||||
|
}
|
||||||
|
|
||||||
|
func capacityInputRequest(endpoint string) promptkit.RunRequest {
|
||||||
|
return promptkit.RunRequest{
|
||||||
|
PromptID: "input-prompt",
|
||||||
|
Inputs: map[string]promptkit.ArtifactRef{
|
||||||
|
"input": promptkit.Inline("input"),
|
||||||
|
},
|
||||||
|
Execution: &promptkit.ExecutionTargetOverride{Endpoint: endpoint},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type capacityGateClient struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
active int
|
||||||
|
peak int
|
||||||
|
calls int
|
||||||
|
started chan promptkit.GenerateRequest
|
||||||
|
release <-chan struct{}
|
||||||
|
}
|
||||||
|
|
||||||
|
func newCapacityGateClient(release <-chan struct{}, buffer int) *capacityGateClient {
|
||||||
|
return &capacityGateClient{
|
||||||
|
started: make(chan promptkit.GenerateRequest, buffer),
|
||||||
|
release: release,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *capacityGateClient) Generate(
|
||||||
|
ctx context.Context,
|
||||||
|
request promptkit.GenerateRequest,
|
||||||
|
) (*promptkit.GenerateResponse, error) {
|
||||||
|
c.mu.Lock()
|
||||||
|
c.calls++
|
||||||
|
c.active++
|
||||||
|
if c.active > c.peak {
|
||||||
|
c.peak = c.active
|
||||||
|
}
|
||||||
|
c.mu.Unlock()
|
||||||
|
defer func() {
|
||||||
|
c.mu.Lock()
|
||||||
|
c.active--
|
||||||
|
c.mu.Unlock()
|
||||||
|
}()
|
||||||
|
|
||||||
|
c.started <- request
|
||||||
|
select {
|
||||||
|
case <-c.release:
|
||||||
|
return &promptkit.GenerateResponse{Content: "ok"}, nil
|
||||||
|
case <-ctx.Done():
|
||||||
|
return nil, ctx.Err()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *capacityGateClient) snapshot() (active, peak, calls int) {
|
||||||
|
c.mu.Lock()
|
||||||
|
defer c.mu.Unlock()
|
||||||
|
return c.active, c.peak, c.calls
|
||||||
|
}
|
||||||
|
|
||||||
|
type capacityArtifactReader struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
calls int
|
||||||
|
entered chan struct{}
|
||||||
|
release <-chan struct{}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *capacityArtifactReader) Read(
|
||||||
|
ctx context.Context,
|
||||||
|
_ promptkit.ArtifactRef,
|
||||||
|
) (*promptkit.Artifact, error) {
|
||||||
|
r.mu.Lock()
|
||||||
|
r.calls++
|
||||||
|
r.mu.Unlock()
|
||||||
|
r.entered <- struct{}{}
|
||||||
|
select {
|
||||||
|
case <-r.release:
|
||||||
|
return &promptkit.Artifact{Body: []byte("input")}, nil
|
||||||
|
case <-ctx.Done():
|
||||||
|
return nil, ctx.Err()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *capacityArtifactReader) callCount() int {
|
||||||
|
r.mu.Lock()
|
||||||
|
defer r.mu.Unlock()
|
||||||
|
return r.calls
|
||||||
|
}
|
||||||
|
|
||||||
|
func awaitCapacityRequest(
|
||||||
|
t *testing.T,
|
||||||
|
requests <-chan promptkit.GenerateRequest,
|
||||||
|
) promptkit.GenerateRequest {
|
||||||
|
t.Helper()
|
||||||
|
select {
|
||||||
|
case request := <-requests:
|
||||||
|
return request
|
||||||
|
case <-time.After(5 * time.Second):
|
||||||
|
t.Fatal("timed out waiting for client invocation")
|
||||||
|
return promptkit.GenerateRequest{}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func awaitCapacityRun(t *testing.T, results <-chan capacityRunResult) capacityRunResult {
|
||||||
|
t.Helper()
|
||||||
|
select {
|
||||||
|
case result := <-results:
|
||||||
|
return result
|
||||||
|
case <-time.After(5 * time.Second):
|
||||||
|
t.Fatal("timed out waiting for Run")
|
||||||
|
return capacityRunResult{}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func awaitCapacitySignal(t *testing.T, signal <-chan struct{}, name string) {
|
||||||
|
t.Helper()
|
||||||
|
select {
|
||||||
|
case <-signal:
|
||||||
|
case <-time.After(5 * time.Second):
|
||||||
|
t.Fatalf("timed out waiting for %s", name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func capacityInt(value int) *int {
|
||||||
|
return &value
|
||||||
|
}
|
||||||
|
|
||||||
|
func closedCapacityChannel() <-chan struct{} {
|
||||||
|
channel := make(chan struct{})
|
||||||
|
close(channel)
|
||||||
|
return channel
|
||||||
|
}
|
||||||
40
capacity_error.go
Normal file
40
capacity_error.go
Normal file
@@ -0,0 +1,40 @@
|
|||||||
|
package promptkit
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// CapacityError reports bounded admission rejected for a selected backend.
|
||||||
|
//
|
||||||
|
// Engine-produced values identify only rejection at Promptkit's bounded
|
||||||
|
// [Engine.Run] or [Engine.RunPrepared] admission boundary. BackendID is the
|
||||||
|
// normalized registered backend ID used for routing and capacity; endpoint
|
||||||
|
// overrides do not change it. Every engine-produced value is nonnil and has a
|
||||||
|
// nonblank BackendID. Provider errors, active-generation waiting, and caller
|
||||||
|
// cancellation are not represented by this type.
|
||||||
|
//
|
||||||
|
// Callers own returned values and may mutate BackendID without affecting engine
|
||||||
|
// state or another error. CapacityError and its default Go encoding have no
|
||||||
|
// stable JSON contract. Consumer-constructed values do not establish that an
|
||||||
|
// engine rejected work.
|
||||||
|
type CapacityError struct {
|
||||||
|
// BackendID is the normalized registered backend ID whose admission was
|
||||||
|
// rejected.
|
||||||
|
BackendID string
|
||||||
|
}
|
||||||
|
|
||||||
|
// Error returns diagnostic wording that is not a parsing contract. It is safe
|
||||||
|
// to call on a nil receiver or a value with a blank BackendID.
|
||||||
|
func (e *CapacityError) Error() string {
|
||||||
|
if e == nil || strings.TrimSpace(e.BackendID) == "" {
|
||||||
|
return ErrCapacityExceeded.Error()
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("backend %q admission: %v", e.BackendID, ErrCapacityExceeded)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Unwrap returns ErrCapacityExceeded so errors.Is and errors.As can be used
|
||||||
|
// together. It is safe to call on a nil receiver or a zero value.
|
||||||
|
func (e *CapacityError) Unwrap() error {
|
||||||
|
return ErrCapacityExceeded
|
||||||
|
}
|
||||||
53
convert.go
53
convert.go
@@ -4,6 +4,7 @@ import (
|
|||||||
"reflect"
|
"reflect"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/jsonvalue"
|
||||||
)
|
)
|
||||||
|
|
||||||
func toDomainRunRequest(req RunRequest) (domain.RunRequest, error) {
|
func toDomainRunRequest(req RunRequest) (domain.RunRequest, error) {
|
||||||
@@ -15,12 +16,12 @@ func toDomainRunRequest(req RunRequest) (domain.RunRequest, error) {
|
|||||||
PromptID: req.PromptID,
|
PromptID: req.PromptID,
|
||||||
PromptVersion: req.PromptVersion,
|
PromptVersion: req.PromptVersion,
|
||||||
ProfileID: req.ProfileID,
|
ProfileID: req.ProfileID,
|
||||||
|
SessionID: req.SessionID,
|
||||||
APIKey: req.APIKey,
|
APIKey: req.APIKey,
|
||||||
Inputs: toDomainArtifactRefMap(req.Inputs),
|
Inputs: toDomainArtifactRefMap(req.Inputs),
|
||||||
Vars: copyStringMap(req.Vars),
|
Vars: copyStringMap(req.Vars),
|
||||||
Execution: execution,
|
Execution: execution,
|
||||||
Validation: toDomainOutputContractPtr(req.Validation),
|
Validation: toDomainOutputContractPtr(req.Validation),
|
||||||
Metadata: copyStringMap(req.Metadata),
|
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -33,6 +34,7 @@ func fromDomainPreparedRun(prepared *domain.PreparedRun) *PreparedRun {
|
|||||||
PromptVersion: prepared.PromptVersion,
|
PromptVersion: prepared.PromptVersion,
|
||||||
PromptHash: prepared.PromptHash,
|
PromptHash: prepared.PromptHash,
|
||||||
SelectedProfileID: prepared.SelectedProfileID,
|
SelectedProfileID: prepared.SelectedProfileID,
|
||||||
|
SelectedBackendID: prepared.SelectedBackendID,
|
||||||
EffectiveModelParams: fromDomainExecutionTarget(prepared.EffectiveModelParams),
|
EffectiveModelParams: fromDomainExecutionTarget(prepared.EffectiveModelParams),
|
||||||
OutputContract: fromDomainOutputContract(prepared.OutputContract),
|
OutputContract: fromDomainOutputContract(prepared.OutputContract),
|
||||||
StructuredOutput: fromDomainStructuredOutputSpec(prepared.StructuredOutput),
|
StructuredOutput: fromDomainStructuredOutputSpec(prepared.StructuredOutput),
|
||||||
@@ -58,8 +60,10 @@ func fromDomainRunResult(result *domain.RunResult) *RunResult {
|
|||||||
PromptID: result.PromptID,
|
PromptID: result.PromptID,
|
||||||
PromptVersion: result.PromptVersion,
|
PromptVersion: result.PromptVersion,
|
||||||
PromptHash: result.PromptHash,
|
PromptHash: result.PromptHash,
|
||||||
|
SessionID: result.SessionID,
|
||||||
RenderedPromptHash: result.RenderedPromptHash,
|
RenderedPromptHash: result.RenderedPromptHash,
|
||||||
SelectedProfileID: result.SelectedProfileID,
|
SelectedProfileID: result.SelectedProfileID,
|
||||||
|
SelectedBackendID: result.SelectedBackendID,
|
||||||
ModelName: result.ModelName,
|
ModelName: result.ModelName,
|
||||||
Endpoint: result.Endpoint,
|
Endpoint: result.Endpoint,
|
||||||
EffectiveModelParams: fromDomainExecutionTarget(result.EffectiveModelParams),
|
EffectiveModelParams: fromDomainExecutionTarget(result.EffectiveModelParams),
|
||||||
@@ -132,7 +136,7 @@ func toDomainExecutionTargetOverride(override *ExecutionTargetOverride) (*domain
|
|||||||
if override == nil {
|
if override == nil {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
extraParams, err := copyPublicJSONMap(override.ExtraParams)
|
extraParams, err := jsonvalue.CopyMap(override.ExtraParams)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -144,7 +148,7 @@ func toDomainExecutionTargetOverride(override *ExecutionTargetOverride) (*domain
|
|||||||
TopP: copyFloat64Ptr(override.TopP),
|
TopP: copyFloat64Ptr(override.TopP),
|
||||||
TimeoutSeconds: copyIntPtr(override.TimeoutSeconds),
|
TimeoutSeconds: copyIntPtr(override.TimeoutSeconds),
|
||||||
ServiceTier: override.ServiceTier,
|
ServiceTier: override.ServiceTier,
|
||||||
ReasoningEffort: override.ReasoningEffort,
|
ReasoningEffort: copyStringPtr(override.ReasoningEffort),
|
||||||
APIKeyEnv: override.APIKeyEnv,
|
APIKeyEnv: override.APIKeyEnv,
|
||||||
ExtraParams: extraParams,
|
ExtraParams: extraParams,
|
||||||
}, nil
|
}, nil
|
||||||
@@ -152,6 +156,7 @@ func toDomainExecutionTargetOverride(override *ExecutionTargetOverride) (*domain
|
|||||||
|
|
||||||
func fromDomainExecutionTarget(target domain.ExecutionTarget) ExecutionTarget {
|
func fromDomainExecutionTarget(target domain.ExecutionTarget) ExecutionTarget {
|
||||||
return ExecutionTarget{
|
return ExecutionTarget{
|
||||||
|
BackendID: target.BackendID,
|
||||||
Endpoint: target.Endpoint,
|
Endpoint: target.Endpoint,
|
||||||
Model: target.Model,
|
Model: target.Model,
|
||||||
Temperature: target.Temperature,
|
Temperature: target.Temperature,
|
||||||
@@ -165,6 +170,40 @@ func fromDomainExecutionTarget(target domain.ExecutionTarget) ExecutionTarget {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func fromDomainProfileInspection(inspection *domain.ProfileInspection) *ProfileInspection {
|
||||||
|
if inspection == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return &ProfileInspection{
|
||||||
|
ProfileID: inspection.ProfileID,
|
||||||
|
EffectiveModelParams: fromDomainExecutionTarget(inspection.EffectiveModelParams),
|
||||||
|
APIKeyRequired: inspection.APIKeyRequired,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func fromDomainPromptInspection(inspection *domain.PromptInspection) *PromptInspection {
|
||||||
|
if inspection == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
inputs := make([]PromptInputDefinition, len(inspection.Inputs))
|
||||||
|
for i, input := range inspection.Inputs {
|
||||||
|
inputs[i] = PromptInputDefinition{
|
||||||
|
Name: input.Name,
|
||||||
|
Required: input.Required,
|
||||||
|
ContentType: input.ContentType,
|
||||||
|
Description: input.Description,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return &PromptInspection{
|
||||||
|
PromptID: inspection.PromptID,
|
||||||
|
PromptVersion: inspection.PromptVersion,
|
||||||
|
PromptHash: inspection.PromptHash,
|
||||||
|
DefaultProfileID: inspection.DefaultProfileID,
|
||||||
|
Inputs: inputs,
|
||||||
|
OutputContract: fromDomainOutputContract(inspection.OutputContract),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func fromDomainExecutionTargetPresence(presence domain.ExecutionTargetPresence) ExecutionTargetPresence {
|
func fromDomainExecutionTargetPresence(presence domain.ExecutionTargetPresence) ExecutionTargetPresence {
|
||||||
return ExecutionTargetPresence{
|
return ExecutionTargetPresence{
|
||||||
Temperature: presence.Temperature,
|
Temperature: presence.Temperature,
|
||||||
@@ -397,6 +436,14 @@ func copyFloat64Ptr(src *float64) *float64 {
|
|||||||
return &v
|
return &v
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func copyStringPtr(src *string) *string {
|
||||||
|
if src == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
v := *src
|
||||||
|
return &v
|
||||||
|
}
|
||||||
|
|
||||||
func copyIntPtr(src *int) *int {
|
func copyIntPtr(src *int) *int {
|
||||||
if src == nil {
|
if src == nil {
|
||||||
return nil
|
return nil
|
||||||
|
|||||||
64
doc.go
64
doc.go
@@ -1,8 +1,64 @@
|
|||||||
// Package promptkit provides an embeddable engine for preparing and executing
|
// Package promptkit provides an embeddable engine for preparing and executing
|
||||||
// prompt-defined LLM workflows.
|
// prompt-defined LLM workflows.
|
||||||
//
|
//
|
||||||
// Applications construct an Engine with NewEngine, select filesystem or
|
// Applications construct an [Engine] with [NewEngine], select filesystem or
|
||||||
// in-memory definition sources with options, and use Prepare or Run to execute
|
// in-memory sources and optional engine-scoped [Backend] registrations, and
|
||||||
// requests. Concrete repositories, validators, and outbound clients remain
|
// call [Engine.InspectPrompt], [Engine.InspectProfile], [Engine.Prepare],
|
||||||
// internal implementation details.
|
// [Engine.PrepareExecution], [Engine.Run], or [Engine.RunPrepared]. Concrete
|
||||||
|
// registries, repositories, validators, and the built-in OpenAI-compatible
|
||||||
|
// client remain internal implementation details.
|
||||||
|
//
|
||||||
|
// # Concurrency and ownership
|
||||||
|
//
|
||||||
|
// An Engine supports concurrent InspectPrompt, InspectProfile, Prepare,
|
||||||
|
// PrepareExecution, Run, and RunPrepared calls. Engine-local backend policies
|
||||||
|
// bound admitted Run and RunPrepared calls and model generations where
|
||||||
|
// configured, while different backend pools and unlimited backends continue
|
||||||
|
// independently. An injected [LLMClient] or [ArtifactReader] can therefore
|
||||||
|
// still receive concurrent calls and must be safe for that use.
|
||||||
|
//
|
||||||
|
// NewEngine copies in-memory profiles and backend definitions. Prepare,
|
||||||
|
// PrepareExecution, and Run copy request maps, slices, pointer values, and
|
||||||
|
// JSON-compatible extra parameters before using them. InspectPrompt and
|
||||||
|
// InspectProfile return copied inspection values. Returned values and values
|
||||||
|
// passed to extension interfaces are likewise isolated from engine state.
|
||||||
|
// Callers own those copies and may mutate them after the call that supplied or
|
||||||
|
// returned them. Returned structured errors are likewise caller-owned and may
|
||||||
|
// be mutated without affecting engine state or another error.
|
||||||
|
//
|
||||||
|
// # Security and sensitive data
|
||||||
|
//
|
||||||
|
// The default artifact reader treats [File] paths as caller-selected operating
|
||||||
|
// system paths. It does not restrict them to an application root or impose an
|
||||||
|
// inbound request-size policy. Promptkit is not an inbound request or
|
||||||
|
// untrusted-input security boundary. Applications must validate and restrict
|
||||||
|
// untrusted input before constructing a request, or install an [ArtifactReader]
|
||||||
|
// that enforces their filesystem, authorization, and size policies.
|
||||||
|
//
|
||||||
|
// Rendered messages, input and output [Artifact] bodies, [RunResult.RawOutput],
|
||||||
|
// and [ValidationResult.Errors] may contain sensitive data. Credential
|
||||||
|
// exclusion and redaction do not sanitize those values. Applications and
|
||||||
|
// injected collaborators are responsible for access control, retention,
|
||||||
|
// logging, and secret handling appropriate to their data.
|
||||||
|
//
|
||||||
|
// # JSON
|
||||||
|
//
|
||||||
|
// Stable JSON representations are provided for [PreparedRun], [RunResult],
|
||||||
|
// [Artifact], [ExecutionTarget], [OutputContract], [ValidationResult],
|
||||||
|
// [TokenUsage], [RenderedPrompt], [RenderedMessage], [CacheControl],
|
||||||
|
// [StructuredOutputSpec], [StructuredOutputJSONSpec], [GenerateRequest],
|
||||||
|
// [GenerateResponse], [ExecutionTargetPresence], and the string value types
|
||||||
|
// used by those values.
|
||||||
|
//
|
||||||
|
// Construction, inspection, handle, and error values, including [Config],
|
||||||
|
// [Backend], [RunRequest], [ArtifactRef], [ExecutionTargetOverride], [Profile],
|
||||||
|
// [OpenAICompatibleProfileConfig], [ProfileInspection],
|
||||||
|
// [PromptInputDefinition], [PromptInspection], [PreparedExecution], and
|
||||||
|
// [CapacityError], do not have stable JSON representations. Direct API keys
|
||||||
|
// are nevertheless excluded from JSON for every public value.
|
||||||
|
//
|
||||||
|
// JSON timestamps use time.Time's RFC 3339 encoding and are omitted when zero.
|
||||||
|
// PreparedRun and RunResult durations are encoded as integer milliseconds in
|
||||||
|
// duration_ms and omitted when zero. Run IDs and all exposed hashes are opaque:
|
||||||
|
// their spelling, length, character set, and algorithm are not API contracts.
|
||||||
package promptkit
|
package promptkit
|
||||||
|
|||||||
@@ -1,151 +1,431 @@
|
|||||||
# Package `promptkit`
|
# Package `promptkit`
|
||||||
|
|
||||||
Import path:
|
## Purpose
|
||||||
|
|
||||||
|
This guide helps Go consumers assemble Promptkit and choose the main
|
||||||
|
preparation or execution workflow. The declarations and GoDoc in the
|
||||||
|
[root package](../../doc.go) own exact field, option, serialization,
|
||||||
|
concurrency, ownership, failure, and cancellation semantics. The
|
||||||
|
[framework format reference](../formats.md) owns prompt, profile, and schema
|
||||||
|
file contracts.
|
||||||
|
|
||||||
|
Import the package as:
|
||||||
|
|
||||||
```go
|
```go
|
||||||
import "gitea.maximumdirect.net/eric/promptkit"
|
import "gitea.maximumdirect.net/eric/promptkit"
|
||||||
```
|
```
|
||||||
|
|
||||||
Package `promptkit` is the supported Go contract for in-process prompt
|
The following Go fragments are illustrative and omit surrounding package,
|
||||||
preparation and execution. The declarations and their GoDoc in the
|
import, and error-handling code. Use the maintained examples for complete
|
||||||
[root package](../../doc.go) own the exact API; this guide explains how the
|
programs.
|
||||||
pieces are used together. The [framework format reference](../formats.md) owns
|
|
||||||
prompt, profile, and schema file contracts.
|
|
||||||
|
|
||||||
## Engine Construction And Sources
|
## Construct An Engine
|
||||||
|
|
||||||
Construct an engine with [`NewEngine`, `Config`, and
|
Create an engine with
|
||||||
`Option`](../../engine.go). `PromptDir` is required unless a prompt source
|
[`NewEngine`](../../engine.go). A directory-backed setup supplies a prompt
|
||||||
option is supplied. `ProfileDir` optionally overlays built-in profiles, and an
|
directory and may supply profile and schema directories:
|
||||||
empty `SchemaDir` uses the current directory. `Timeout` is the transport-wide
|
|
||||||
safety cap for the built-in OpenAI-compatible client. An optional `HTTPClient`
|
|
||||||
is cloned; its positive timeout takes precedence.
|
|
||||||
|
|
||||||
Nil options are ignored. Invalid construction, including a nil injected client
|
```go
|
||||||
or artifact reader, returns an error matching `ErrInvalidConfig`.
|
engine, err := promptkit.NewEngine(promptkit.Config{
|
||||||
|
PromptDir: "prompts",
|
||||||
|
ProfileDir: "profiles",
|
||||||
|
SchemaDir: "schemas",
|
||||||
|
})
|
||||||
|
```
|
||||||
|
|
||||||
The [source options](../../engine.go) replace their matching directory source:
|
Options support single-file or `fs.FS` sources, in-memory profiles,
|
||||||
|
engine-scoped backends, and injected artifact or model clients. Consult the
|
||||||
- `WithPromptFS` and `WithPromptFile` select prompt definitions;
|
[constructor and option GoDoc](../../engine.go) for composition, precedence,
|
||||||
- `WithProfileFS` and `WithProfileFile` overlay built-in profiles;
|
validation, and default transport behavior. Source discovery, format
|
||||||
- `WithProfiles` adds in-memory profiles ahead of file and built-in profiles;
|
validation, and profile precedence are defined by the
|
||||||
- `WithSchemaFS` and `WithSchemaFile` select JSON Schema documents;
|
|
||||||
- `WithLLMClient` replaces the built-in model client; and
|
|
||||||
- `WithArtifactReader` replaces the default reader for every input.
|
|
||||||
|
|
||||||
Source selection, path resolution, strict decoding, profile overlays, and
|
|
||||||
file-to-request precedence are defined in the
|
|
||||||
[framework format reference](../formats.md).
|
[framework format reference](../formats.md).
|
||||||
|
|
||||||
Per-generation timeout values from profiles or requests are independent of
|
## Supply Embedded Application Defaults
|
||||||
the transport cap and caller context. An explicit request value of zero
|
|
||||||
disables only the per-generation deadline. The
|
|
||||||
[outbound integration contract](../integrations/openai-compatible-chat.md#timeout-and-cancellation)
|
|
||||||
defines the complete timeout layering.
|
|
||||||
|
|
||||||
## Preparation And Execution
|
Use `WithFallbackProfileFS` when an application packages profile definitions
|
||||||
|
that should apply unless an operator provides an ordinary configured profile
|
||||||
|
with the same ID. For example, an application can embed its defaults while
|
||||||
|
continuing to use `ProfileDir` for operator overrides:
|
||||||
|
|
||||||
[`Engine.Prepare` and `Engine.Run`](../../engine.go) accept the public
|
```go
|
||||||
[`RunRequest`](../../types.go). `Prepare` resolves the prompt, profile, input
|
//go:embed profiles/*.yaml
|
||||||
artifacts, validation contract, and rendered messages without calling an LLM.
|
var applicationProfiles embed.FS
|
||||||
`Run` performs the same preparation, calls the configured client, and validates
|
|
||||||
the generated content. The maintained
|
engine, err := promptkit.NewEngine(promptkit.Config{
|
||||||
|
PromptDir: "prompts",
|
||||||
|
ProfileDir: operatorProfileDir,
|
||||||
|
},
|
||||||
|
promptkit.WithFallbackProfileFS(applicationProfiles, "profiles"),
|
||||||
|
)
|
||||||
|
```
|
||||||
|
|
||||||
|
Keep application-owned profile IDs and definitions in the embedded source.
|
||||||
|
Use the ordinary configured profile source for operator overrides. Leave
|
||||||
|
`operatorProfileDir` empty when the operator did not configure an override
|
||||||
|
directory; a non-empty path names an authoritative higher-precedence source,
|
||||||
|
so an unavailable or unreadable directory is an error rather than a reason to
|
||||||
|
fall back. The
|
||||||
|
[framework format reference](../formats.md#source-and-profile-precedence)
|
||||||
|
owns the exact profile format and lookup order; the
|
||||||
|
[`WithFallbackProfileFS` GoDoc](../../engine.go) owns its option contract and
|
||||||
|
validation rules.
|
||||||
|
|
||||||
|
## Inspect A Prompt Before Preparation
|
||||||
|
|
||||||
|
Use [`Engine.InspectPrompt`](../../engine.go) to check one configured prompt's
|
||||||
|
declared inputs and output workflow without creating placeholder inputs or
|
||||||
|
resolving a profile:
|
||||||
|
|
||||||
|
```go
|
||||||
|
inspection, err := engine.InspectPrompt(ctx, "meeting.summary", "")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, input := range inspection.Inputs {
|
||||||
|
// Compare the declared input with application configuration.
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Use this configuration-time boundary when the application needs only the
|
||||||
|
declared prompt interface. Use `InspectProfile` separately when it must also
|
||||||
|
check a configured profile. Use `Prepare` when it needs inputs, schemas, or
|
||||||
|
rendered messages, and use prepared execution when that work must remain tied
|
||||||
|
to later execution. The method's [GoDoc](../../engine.go) owns exact fields,
|
||||||
|
hash, ownership, and error semantics.
|
||||||
|
|
||||||
|
## Prepare Without Model Execution
|
||||||
|
|
||||||
|
[`Engine.Prepare`](../../engine.go) resolves the selected prompt and profile,
|
||||||
|
loads inputs and any structured-output schema, and renders messages without
|
||||||
|
calling a model client. Choose it when the prepared value is the final
|
||||||
|
inspection or persistence result and no later execution must be tied to that
|
||||||
|
exact snapshot:
|
||||||
|
|
||||||
|
```go
|
||||||
|
prepared, err := engine.Prepare(ctx, promptkit.RunRequest{
|
||||||
|
PromptID: "meeting.summary",
|
||||||
|
Inputs: map[string]promptkit.ArtifactRef{
|
||||||
|
"note": promptkit.Inline("Synthetic meeting notes"),
|
||||||
|
},
|
||||||
|
})
|
||||||
|
```
|
||||||
|
|
||||||
|
The maintained
|
||||||
[offline preparation example](../../examples/go-library/prepare/main.go)
|
[offline preparation example](../../examples/go-library/prepare/main.go)
|
||||||
provides a complete runnable workflow using a prompt file, in-memory profile,
|
shows a complete runnable setup with a prompt file, in-memory profile, and
|
||||||
and inline input.
|
inline input. Exact request requirements and prepared-result fields belong to
|
||||||
|
the [`RunRequest` and `PreparedRun` GoDoc](../../types.go).
|
||||||
|
|
||||||
[`PreparedRun` and `RunResult`](../../types.go) expose copied public values.
|
## Prepare Now And Execute The Same Snapshot Later
|
||||||
Preparation returns effective settings, hashes, rendered messages, selected
|
|
||||||
profile, structured-output information, and timing without resolved secrets or
|
|
||||||
model output. Execution adds the generated artifact and raw output, validation
|
|
||||||
state, model metadata, usage, run ID, and duration.
|
|
||||||
|
|
||||||
A generated-content validation failure returns a result with
|
Use [`Engine.PrepareExecution`](../../engine.go) when an application must
|
||||||
`Validation.Status == ValidationFailed`. An inability to perform validation
|
inspect or persist preflight details before deciding whether to start model
|
||||||
returns an error matching `ErrValidation`.
|
work, while ensuring that later execution uses those exact rendered messages,
|
||||||
|
inputs, target settings, and validation resources:
|
||||||
|
|
||||||
## Requests, Inputs, And Overrides
|
```go
|
||||||
|
preparedExecution, err := engine.PrepareExecution(ctx, promptkit.RunRequest{
|
||||||
|
PromptID: "meeting.summary",
|
||||||
|
Inputs: map[string]promptkit.ArtifactRef{
|
||||||
|
"note": promptkit.Inline("Synthetic meeting notes"),
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer preparedExecution.Discard()
|
||||||
|
|
||||||
The [request and value declarations](../../types.go) own the available fields,
|
details := preparedExecution.Details()
|
||||||
serialized constants, and result shapes. Use `File`, `Inline`, or
|
// Inspect or persist an application-selected safe subset of details.
|
||||||
`InlineWithURI` to construct artifact references. The
|
|
||||||
[framework format reference](../formats.md) defines declared inputs, template
|
|
||||||
references, output contracts, and the relationship between file values and
|
|
||||||
request overrides.
|
|
||||||
|
|
||||||
`ExecutionTargetOverride` uses pointers for numeric settings so an explicit
|
result, err := engine.RunPrepared(ctx, preparedExecution)
|
||||||
zero remains distinct from no override. `ExtraParams` accepts JSON-compatible
|
```
|
||||||
strings, booleans, finite numbers, string-keyed objects, arrays or slices, and
|
|
||||||
nil. Unsupported values, non-string map keys, non-finite numbers, and cycles
|
|
||||||
match `ErrInvalidConfig` in profiles or `ErrInvalidRequest` in request
|
|
||||||
overrides.
|
|
||||||
|
|
||||||
Returned requests, profiles, prepared values, results, artifacts, maps, and
|
Preparation does not call the model or reserve backend capacity.
|
||||||
slices are isolated from internal engine state. Consumers and injected
|
`RunPrepared` executes from the retained snapshot rather than reloading
|
||||||
extensions should not retain or mutate values owned by another caller.
|
consumer sources. The handle is opaque in-process state, while `Details`
|
||||||
|
contains rendered content and remains subject to the application's data
|
||||||
|
handling policy. The
|
||||||
|
[`PreparedExecution` and method GoDoc](../../prepared_execution.go) and
|
||||||
|
[engine operation GoDoc](../../engine.go) own exact lifecycle, engine-binding,
|
||||||
|
credential, cancellation, timing, and error semantics.
|
||||||
|
|
||||||
## Profiles And Credentials
|
## Execute And Validate
|
||||||
|
|
||||||
[`OpenAICompatibleProfile`](../../profiles.go) constructs an ordinary
|
[`Engine.Run`](../../engine.go) performs the same preparation, invokes the
|
||||||
in-memory profile for an OpenAI-compatible chat-completions endpoint.
|
configured model client, classifies the generated artifact, and validates the
|
||||||
`WithProfiles` rejects duplicate IDs in one call and gives in-memory profiles
|
content in one call. Choose it when the application does not need a preflight
|
||||||
precedence over explicit file sources and built-ins.
|
boundary tied to the eventual execution. A completed content check may return
|
||||||
|
`ValidationFailed` in the result; an operational inability to validate returns
|
||||||
|
an error.
|
||||||
|
|
||||||
Raw API keys do not belong in profiles. File-backed profiles may name an
|
The maintained
|
||||||
environment variable, while an in-memory profile can require a request key.
|
[offline execution example](../../examples/go-library/run/main.go) injects a
|
||||||
A direct `RunRequest.APIKey` is request-scoped and takes precedence over an
|
deterministic model client and exercises `Run` without credentials, network
|
||||||
environment lookup for the built-in client. Profile fields, ranges, built-ins,
|
access, or paid calls. It is intentionally separate from the preparation
|
||||||
precedence, and credential rules are owned by the
|
example so each workflow and its small prompt fixture can be copied and run on
|
||||||
[framework format reference](../formats.md).
|
its own.
|
||||||
|
|
||||||
API keys are excluded from JSON, prepared values, and results. The public
|
Use the [`RunResult` and `ValidationResult` GoDoc](../../types.go) for the
|
||||||
`String` and `GoString` methods report only whether a direct key is present.
|
returned data and the `Engine.Run` GoDoc for failure and cancellation
|
||||||
Avoid reflection-based dumps of request structs, which can bypass that
|
semantics. The
|
||||||
redaction.
|
[OpenAI-compatible integration contract](../integrations/openai-compatible-chat.md)
|
||||||
|
owns the built-in client's outbound HTTP behavior.
|
||||||
|
|
||||||
|
## Inputs, Profiles, And Overrides
|
||||||
|
|
||||||
|
Use `File`, `Inline`, or `InlineWithURI` to construct request inputs. A request
|
||||||
|
can select a profile explicitly or use the prompt's default profile, and can
|
||||||
|
replace execution settings or the complete output contract.
|
||||||
|
|
||||||
|
The [public value GoDoc](../../types.go) defines nil, empty, zero, replacement,
|
||||||
|
copy, and credential behavior. The
|
||||||
|
[framework format reference](../formats.md) defines how those request values
|
||||||
|
interact with prompt definitions, file-backed and application fallback
|
||||||
|
profiles, built-ins, schemas, and framework defaults.
|
||||||
|
|
||||||
|
For programmatic profiles,
|
||||||
|
[`OpenAICompatibleProfile`](../../profiles.go) converts ordinary
|
||||||
|
OpenAI-compatible settings into a value accepted by `WithProfiles`.
|
||||||
|
|
||||||
|
### Inspect A Profile Before Prompt Work
|
||||||
|
|
||||||
|
Use [`Engine.InspectProfile`](../../engine.go) to validate one configured
|
||||||
|
profile without constructing a synthetic prompt or placeholder inputs. It
|
||||||
|
resolves the profile's effective target but does not prepare or execute a
|
||||||
|
prompt:
|
||||||
|
|
||||||
|
```go
|
||||||
|
inspection, err := engine.InspectProfile(ctx, profileID)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
target := inspection.EffectiveModelParams
|
||||||
|
if target.APIKeyEnv != "" {
|
||||||
|
// Apply application policy for the named environment variable.
|
||||||
|
} else if inspection.APIKeyRequired {
|
||||||
|
// Arrange a direct credential before later execution.
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Use this configuration-time boundary when only the profile and its target need
|
||||||
|
checking. Use `Prepare` when the application also needs prompt, input, schema,
|
||||||
|
or rendering work; use prepared execution when that work must remain tied to a
|
||||||
|
later execution. Inspection reports credential requirements but leaves the
|
||||||
|
timing of credential enforcement to the application. The method's
|
||||||
|
[GoDoc](../../engine.go) owns its exact result and error contract.
|
||||||
|
|
||||||
|
### Set A Per-Run Session And Reasoning
|
||||||
|
|
||||||
|
Supply a direct session ID when one prompt should be correlated with a
|
||||||
|
consumer-managed conversation or workflow without changing prompt variables:
|
||||||
|
|
||||||
|
```go
|
||||||
|
reasoning := "high"
|
||||||
|
result, err := engine.Run(ctx, promptkit.RunRequest{
|
||||||
|
PromptID: "meeting.summary",
|
||||||
|
SessionID: "conversation-42",
|
||||||
|
Inputs: map[string]promptkit.ArtifactRef{
|
||||||
|
"note": promptkit.Inline("Synthetic meeting notes"),
|
||||||
|
},
|
||||||
|
Execution: &promptkit.ExecutionTargetOverride{
|
||||||
|
ReasoningEffort: &reasoning,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
```
|
||||||
|
|
||||||
|
A nil reasoning pointer inherits the selected profile, a pointer to a
|
||||||
|
nonblank string replaces it, and a pointer to a blank string disables
|
||||||
|
reasoning for that run. Session IDs are correlation metadata, not credentials;
|
||||||
|
use stable, non-secret values that are safe to expose to collaborators and
|
||||||
|
providers. The
|
||||||
|
[`RunRequest` and `ExecutionTargetOverride` GoDoc](../../types.go) owns the
|
||||||
|
exact normalization, precedence, error, copying, and exposure contract.
|
||||||
|
|
||||||
|
### Configure A Local OpenAI-Compatible Endpoint
|
||||||
|
|
||||||
|
Choose the smallest configuration that fits how the endpoint will be reused.
|
||||||
|
|
||||||
|
#### Use An Endpoint-Only Profile
|
||||||
|
|
||||||
|
Put the endpoint directly on an in-memory profile when only that profile needs
|
||||||
|
it and shared backend identity or capacity policy is unnecessary:
|
||||||
|
|
||||||
|
```go
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{
|
||||||
|
PromptDir: "prompts",
|
||||||
|
},
|
||||||
|
promptkit.WithProfiles(promptkit.Profile{
|
||||||
|
ID: "local-summary",
|
||||||
|
Endpoint: "http://localhost:8000/v1",
|
||||||
|
Model: "example-model",
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
```
|
||||||
|
|
||||||
|
Endpoint-only profiles have an empty backend ID and remain unrestricted by
|
||||||
|
backend capacity policy.
|
||||||
|
|
||||||
|
#### Use The Conventional Local Backend
|
||||||
|
|
||||||
|
Use `LocalBackend` when profiles should share the conventional `local`
|
||||||
|
identity, endpoint, and concurrency limit:
|
||||||
|
|
||||||
|
```go
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{
|
||||||
|
PromptDir: "prompts",
|
||||||
|
},
|
||||||
|
promptkit.WithBackend(
|
||||||
|
promptkit.LocalBackend("http://localhost:8000/v1", 2),
|
||||||
|
),
|
||||||
|
promptkit.WithProfiles(promptkit.Profile{
|
||||||
|
ID: "local-summary",
|
||||||
|
BackendID: promptkit.BackendLocal,
|
||||||
|
Model: "example-model",
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
```
|
||||||
|
|
||||||
|
The helper is explicit: it does not pre-register a backend or read environment
|
||||||
|
variables. Supplying a positive limit leaves queue capacity omitted, so normal
|
||||||
|
backend registration selects the existing default waiting capacity of 1024.
|
||||||
|
The returned value still enters the engine through `WithBackend`.
|
||||||
|
|
||||||
|
#### Configure A Complete Backend
|
||||||
|
|
||||||
|
Use a keyed `Backend` value for authentication, extra request parameters, an
|
||||||
|
explicit queue capacity, a custom ID, or multiple local endpoints:
|
||||||
|
|
||||||
|
```go
|
||||||
|
noWaiting := 0
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{
|
||||||
|
PromptDir: "prompts",
|
||||||
|
},
|
||||||
|
promptkit.WithBackend(promptkit.Backend{
|
||||||
|
ID: "local-gpu",
|
||||||
|
Endpoint: "http://gpu-host:8000/v1",
|
||||||
|
APIKeyEnv: "LOCAL_GPU_API_KEY",
|
||||||
|
ExtraParams: map[string]any{"provider_option": "enabled"},
|
||||||
|
ConcurrencyLimit: 2,
|
||||||
|
QueueCapacity: &noWaiting,
|
||||||
|
}),
|
||||||
|
promptkit.WithProfiles(promptkit.Profile{
|
||||||
|
ID: "gpu-summary",
|
||||||
|
BackendID: "local-gpu",
|
||||||
|
Model: "example-model",
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
```
|
||||||
|
|
||||||
|
Use distinct custom IDs when registering multiple local endpoints.
|
||||||
|
Registrations belong to one engine and custom IDs cannot replace built-ins.
|
||||||
|
The [`Backend`, `LocalBackend`, and `WithBackend` GoDoc](../../backends.go)
|
||||||
|
defines exact construction, validation, copying, uniqueness, concurrency, and
|
||||||
|
request-default behavior.
|
||||||
|
|
||||||
|
Both file-backed and in-memory profiles select a registration through
|
||||||
|
`backend` or `Profile.BackendID`. Profile and request endpoint overrides retain
|
||||||
|
that routing and capacity identity. `PreparedRun.SelectedBackendID`,
|
||||||
|
`RunResult.SelectedBackendID`, and the effective `ExecutionTarget.BackendID`
|
||||||
|
expose it to consumers and injected model clients. Endpoint-only profiles
|
||||||
|
remain supported and expose an empty backend ID.
|
||||||
|
|
||||||
|
### Limit Backend Concurrency
|
||||||
|
|
||||||
|
Set `Backend.ConcurrencyLimit` when a backend needs protection from too many
|
||||||
|
simultaneous model calls. Leaving `QueueCapacity` nil, as in the local-backend
|
||||||
|
example above, selects the default waiting capacity of 1024.
|
||||||
|
|
||||||
|
To accept no waiting backlog beyond the active calls, provide an explicit
|
||||||
|
zero:
|
||||||
|
|
||||||
|
```go
|
||||||
|
noWaiting := 0
|
||||||
|
backend := promptkit.Backend{
|
||||||
|
ID: "local-gpu",
|
||||||
|
Endpoint: "http://gpu-host:8000/v1",
|
||||||
|
ConcurrencyLimit: 2,
|
||||||
|
QueueCapacity: &noWaiting,
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
The pointer distinguishes an explicit zero from omission. Keep using keyed
|
||||||
|
`Backend` literals so additive configuration fields remain source-compatible.
|
||||||
|
Capacity belongs to one engine and the selected backend ID; endpoint-only
|
||||||
|
profiles and custom backends without a configured limit remain unrestricted.
|
||||||
|
Exact validation, defaulting, ownership, and concurrency semantics belong to
|
||||||
|
the [`Backend` GoDoc](../../backends.go).
|
||||||
|
|
||||||
|
## Credentials
|
||||||
|
|
||||||
|
File-backed profiles name an environment variable; in-memory profiles can
|
||||||
|
require a direct request key. Direct keys are request-scoped and are excluded
|
||||||
|
from supported JSON values and the package's `String` and `GoString`
|
||||||
|
summaries. The exact precedence and redaction guarantees belong to
|
||||||
|
[`RunRequest`, `GenerateRequest`, and the profile GoDoc](../../types.go).
|
||||||
|
|
||||||
|
## Protect Files And Generated Data
|
||||||
|
|
||||||
|
The default artifact reader opens a `File` reference as a caller-selected
|
||||||
|
operating-system path. It does not constrain paths to an application root,
|
||||||
|
impose an inbound request-size policy, or establish an untrusted-input security
|
||||||
|
boundary. Applications must validate and restrict untrusted paths and payloads
|
||||||
|
before constructing a request, or inject an artifact reader that enforces
|
||||||
|
their filesystem, authorization, and size policies.
|
||||||
|
|
||||||
|
Rendered messages, input and output artifact bodies, raw model output, and
|
||||||
|
validation diagnostics can contain sensitive data. API-key redaction does not
|
||||||
|
sanitize those values. Treat prepared values, results, collaborator requests,
|
||||||
|
errors, and logs according to the application's data-access, retention, and
|
||||||
|
secret-handling policies.
|
||||||
|
|
||||||
## Extension Interfaces
|
## Extension Interfaces
|
||||||
|
|
||||||
The [`LLMClient`, `GenerateRequest`, and
|
Inject an [`LLMClient` or `ArtifactReader`](../../types.go) when the built-in
|
||||||
`GenerateResponse`](../../types.go) boundary lets a consumer replace model
|
behavior does not fit the application. Their GoDoc defines concurrent use,
|
||||||
generation. Injected clients receive copied rendered messages, effective
|
context handling, ownership of copied values, nil responses, and preservation
|
||||||
settings, explicit numeric-setting presence, structured-output constraints,
|
of collaborator errors. Implementations must honor cancellation, safely manage
|
||||||
and the request-scoped key. They return generated content and token usage.
|
copies they retain, avoid unsafe logging of content or credentials, and enforce
|
||||||
|
the application policy that motivated the injection.
|
||||||
|
|
||||||
The [`ArtifactReader`](../../types.go) boundary replaces the default inline and
|
## Handle Errors
|
||||||
file reader for every input. Readers provide artifact content and metadata; the
|
|
||||||
engine fills an empty artifact name from the input-map key. A reader error
|
|
||||||
matches `ErrArtifactLoad` while preserving the original identity for
|
|
||||||
`errors.Is`. A nil artifact with a nil error is also an artifact-load failure.
|
|
||||||
|
|
||||||
Extensions should honor context cancellation and avoid logging raw prompts,
|
Use `errors.Is` with the
|
||||||
artifacts, or credentials.
|
[public error sentinels and operation GoDoc](../../engine.go). The declarations
|
||||||
|
distinguish invalid construction, invalid requests, absent sources,
|
||||||
|
source-loading failures, collaborator failures, and operational validation
|
||||||
|
failures. Specific request conditions may also match the broader
|
||||||
|
`ErrInvalidRequest`, and injected collaborator identities are preserved where
|
||||||
|
documented. Invalid or duplicate backend registrations match
|
||||||
|
`ErrInvalidConfig`; selecting an unknown backend matches `ErrProfileLoad`.
|
||||||
|
|
||||||
## Errors
|
When a limited backend has admitted all active and waiting calls, handle
|
||||||
|
`ErrCapacityExceeded` separately from request errors and provider failures:
|
||||||
|
|
||||||
The [public error declarations](../../engine.go) and
|
```go
|
||||||
[mapping](../../errors.go) preserve these sentinel checks through `errors.Is`:
|
result, err := engine.Run(ctx, request)
|
||||||
|
if errors.Is(err, promptkit.ErrCapacityExceeded) {
|
||||||
|
var capacityErr *promptkit.CapacityError
|
||||||
|
if errors.As(err, &capacityErr) {
|
||||||
|
// Record capacityErr.BackendID using application-owned diagnostics.
|
||||||
|
}
|
||||||
|
|
||||||
- `ErrInvalidConfig`
|
// Apply application policy: shed work, report overload, or retry later.
|
||||||
- `ErrInvalidRequest`
|
}
|
||||||
- `ErrPromptNotFound`
|
```
|
||||||
- `ErrProfileNotFound`
|
|
||||||
- `ErrProfileRequired`
|
|
||||||
- `ErrPromptLoad`
|
|
||||||
- `ErrProfileLoad`
|
|
||||||
- `ErrAPIKeyEnvMissing`
|
|
||||||
- `ErrArtifactLoad`
|
|
||||||
- `ErrPromptRender`
|
|
||||||
- `ErrLLMGenerate`
|
|
||||||
- `ErrValidation`
|
|
||||||
|
|
||||||
`ErrProfileRequired` and `ErrAPIKeyEnvMissing` also match
|
A rejected call returns no partial result and does not invoke the model
|
||||||
`ErrInvalidRequest`, allowing either broad request handling or a specific
|
client. Promptkit does not prescribe retries or map this error to an HTTP
|
||||||
condition. Wrapped collaborator errors retain their identity where the public
|
status; those choices remain with the consuming application. The
|
||||||
contract promises it.
|
[`CapacityError` GoDoc](../../capacity_error.go) owns the exact typed-error
|
||||||
|
contract, while the [`Engine.Run` and error GoDoc](../../engine.go) owns broad
|
||||||
|
error and cancellation identities.
|
||||||
|
|
||||||
## Consumer Boundary
|
## Application Boundary
|
||||||
|
|
||||||
Promptkit is an importable library. It does not own a command, inbound HTTP
|
Promptkit is an importable library. It does not own a command, inbound HTTP
|
||||||
API, process configuration, or deployment policy. Scriptorium is one
|
API, process configuration, or deployment policy. Applications map the root
|
||||||
downstream application that maps this root package contract into those
|
package's results and errors into those concerns, including inbound size and
|
||||||
application concerns.
|
trust policy.
|
||||||
|
|||||||
@@ -45,11 +45,17 @@ For cross-cutting changes, follow every applicable row. Do not create
|
|||||||
placeholder documents for packages, APIs, or integrations that do not yet
|
placeholder documents for packages, APIs, or integrations that do not yet
|
||||||
exist.
|
exist.
|
||||||
|
|
||||||
## Maintainer-Run Validation
|
## Maintainer Validation
|
||||||
|
|
||||||
Promptkit does not currently use hosted CI. Maintainers are responsible for
|
This section is the canonical local validation workflow for Promptkit. Run
|
||||||
running the documented checks before accepting changes. Run the default Go
|
every command from the repository root before accepting a change. The test
|
||||||
validation from the Promptkit repository root:
|
suite and maintained examples are deterministic, offline, and require no real
|
||||||
|
provider credentials.
|
||||||
|
|
||||||
|
### Tests, Analysis, Build, And Examples
|
||||||
|
|
||||||
|
Run the ordinary and race-enabled suites, static analysis, the build, and both
|
||||||
|
maintained consumer examples:
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
go test ./...
|
go test ./...
|
||||||
@@ -57,85 +63,171 @@ go test -race ./...
|
|||||||
go vet ./...
|
go vet ./...
|
||||||
go build ./...
|
go build ./...
|
||||||
go run ./examples/go-library/prepare
|
go run ./examples/go-library/prepare
|
||||||
|
go run ./examples/go-library/run
|
||||||
```
|
```
|
||||||
|
|
||||||
Check formatting across every tracked Go file:
|
Both examples must exit successfully. Review their JSON output: preparation
|
||||||
|
must report the selected offline prompt, profile, model, and message count;
|
||||||
|
execution must report the deterministic generated output, successful
|
||||||
|
validation, selected offline model, and usage. Neither command may contact a
|
||||||
|
provider or require credentials.
|
||||||
|
|
||||||
|
### Go Formatting
|
||||||
|
|
||||||
|
Check every tracked Go file. The final command must succeed and the captured
|
||||||
|
list must be empty:
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
gofmt -l $(git ls-files '*.go')
|
unformatted=$(
|
||||||
|
git ls-files '*.go' |
|
||||||
|
while IFS= read -r go_file
|
||||||
|
do
|
||||||
|
gofmt -l "$go_file"
|
||||||
|
done
|
||||||
|
)
|
||||||
|
test -z "$unformatted"
|
||||||
```
|
```
|
||||||
|
|
||||||
The formatting command must produce no paths. Follow every added or changed
|
### Local Markdown Links
|
||||||
Markdown link and confirm its target exists. Finally, check whitespace:
|
|
||||||
|
Use the Python standard library to verify every repository-relative Markdown
|
||||||
|
target and local heading fragment. The check is offline and prints nothing on
|
||||||
|
success:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
python3 - <<'PY'
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
from urllib.parse import unquote
|
||||||
|
|
||||||
|
root = Path.cwd().resolve()
|
||||||
|
markdown_files = [
|
||||||
|
root / name
|
||||||
|
for name in subprocess.check_output(
|
||||||
|
["git", "ls-files", "*.md"], text=True
|
||||||
|
).splitlines()
|
||||||
|
]
|
||||||
|
link_pattern = re.compile(r"!?\[[^]]*\]\(([^)]+)\)")
|
||||||
|
heading_pattern = re.compile(r"^#{1,6}\s+(.+?)\s*#*\s*$")
|
||||||
|
scheme_pattern = re.compile(r"^[a-z][a-z0-9+.-]*:", re.IGNORECASE)
|
||||||
|
|
||||||
|
|
||||||
|
def markdown_lines(path):
|
||||||
|
in_fence = False
|
||||||
|
fence = ""
|
||||||
|
for line in path.read_text(encoding="utf-8").splitlines():
|
||||||
|
stripped = line.lstrip()
|
||||||
|
marker = stripped[:3]
|
||||||
|
if marker in {"```", "~~~"}:
|
||||||
|
if not in_fence:
|
||||||
|
in_fence = True
|
||||||
|
fence = marker
|
||||||
|
elif marker == fence:
|
||||||
|
in_fence = False
|
||||||
|
fence = ""
|
||||||
|
continue
|
||||||
|
if not in_fence:
|
||||||
|
yield line
|
||||||
|
|
||||||
|
|
||||||
|
anchor_cache = {}
|
||||||
|
|
||||||
|
|
||||||
|
def anchors(path):
|
||||||
|
if path in anchor_cache:
|
||||||
|
return anchor_cache[path]
|
||||||
|
found = set()
|
||||||
|
counts = {}
|
||||||
|
for line in markdown_lines(path):
|
||||||
|
match = heading_pattern.match(line)
|
||||||
|
if not match:
|
||||||
|
continue
|
||||||
|
heading = re.sub(r"<[^>]+>", "", match.group(1)).replace("`", "")
|
||||||
|
base = re.sub(r"[^\w\- ]", "", heading.lower()).replace(" ", "-")
|
||||||
|
count = counts.get(base, 0)
|
||||||
|
counts[base] = count + 1
|
||||||
|
found.add(base if count == 0 else f"{base}-{count}")
|
||||||
|
anchor_cache[path] = found
|
||||||
|
return found
|
||||||
|
|
||||||
|
|
||||||
|
failures = []
|
||||||
|
for source in markdown_files:
|
||||||
|
text = "\n".join(markdown_lines(source))
|
||||||
|
for match in link_pattern.finditer(text):
|
||||||
|
target = match.group(1).strip()
|
||||||
|
if target.startswith("<") and target.endswith(">"):
|
||||||
|
target = target[1:-1]
|
||||||
|
if scheme_pattern.match(target) or target.startswith("//"):
|
||||||
|
continue
|
||||||
|
path_text, separator, fragment = target.partition("#")
|
||||||
|
destination = source if not path_text else source.parent / unquote(path_text)
|
||||||
|
try:
|
||||||
|
destination = destination.resolve()
|
||||||
|
destination.relative_to(root)
|
||||||
|
except ValueError:
|
||||||
|
failures.append(f"{source.relative_to(root)}: escapes repository: {target}")
|
||||||
|
continue
|
||||||
|
if not destination.exists():
|
||||||
|
failures.append(f"{source.relative_to(root)}: missing target: {target}")
|
||||||
|
continue
|
||||||
|
if separator and destination.suffix.lower() == ".md":
|
||||||
|
fragment = unquote(fragment).lower()
|
||||||
|
if fragment not in anchors(destination):
|
||||||
|
failures.append(f"{source.relative_to(root)}: missing anchor: {target}")
|
||||||
|
|
||||||
|
if failures:
|
||||||
|
print("\n".join(failures), file=sys.stderr)
|
||||||
|
raise SystemExit(1)
|
||||||
|
PY
|
||||||
|
```
|
||||||
|
|
||||||
|
### Repository Hygiene And Review
|
||||||
|
|
||||||
|
Reject an active Go workspace, tracked workspace files, a vendor tree, or a
|
||||||
|
module replacement:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
case "$(go env GOWORK)" in
|
||||||
|
''|off) ;;
|
||||||
|
*) printf '%s\n' 'an active Go workspace is not allowed' >&2; exit 1 ;;
|
||||||
|
esac
|
||||||
|
test -z "$(git ls-files go.work go.work.sum)"
|
||||||
|
test ! -e vendor
|
||||||
|
if grep -Eq '^[[:space:]]*replace([[:space:]]|\()' go.mod
|
||||||
|
then
|
||||||
|
printf '%s\n' 'go.mod contains a replacement' >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
```
|
||||||
|
|
||||||
|
Check whitespace in both unstaged and staged changes. List ignored files and
|
||||||
|
scan tracked content for common credential forms:
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
git diff --check
|
git diff --check
|
||||||
|
git diff --cached --check
|
||||||
|
test -z "$(git ls-files --others --ignored --exclude-standard)"
|
||||||
|
credential_pattern='-----BEGIN ([A-Z0-9]+ )?PRIV''ATE KEY-----|AKI''A[0-9A-Z]{16}|gh[pousr]_[A-Za-z0-9]{36,}|sk-[A-Za-z0-9]{32,}'
|
||||||
|
if git grep -nEI -e "$credential_pattern" -- .
|
||||||
|
then
|
||||||
|
printf '%s\n' 'possible credential found' >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
```
|
```
|
||||||
|
|
||||||
Documentation-only work does not require unrelated new tests, but it still
|
Inspect `git status --short --untracked-files=all` and the complete diff before
|
||||||
requires link validation and `git diff --check`. Run the Go validation whenever
|
accepting a change. The status may contain only the intended source changes
|
||||||
documentation changes commands, examples, generated output, or another
|
during development. Reject credentials, private keys, environment files,
|
||||||
behavior checked by the module.
|
generated binaries, test or coverage output, downloaded assets, template
|
||||||
|
residue, and any other artifact that does not belong in source control. The
|
||||||
|
credential scan catches common forms but does not replace inspection of the
|
||||||
|
actual change.
|
||||||
|
|
||||||
## Focused Validation
|
After committing the accepted change, require a clean candidate:
|
||||||
|
|
||||||
Use focused checks while iterating, then run the complete validation sequence
|
|
||||||
before accepting the change. The root package supports:
|
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
go test .
|
test -z "$(git status --porcelain)"
|
||||||
go vet .
|
|
||||||
go build .
|
|
||||||
```
|
```
|
||||||
|
|
||||||
Filter tests by name without assuming a fixed internal package layout:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
go test ./... -run 'TestName'
|
|
||||||
```
|
|
||||||
|
|
||||||
Replace `TestName` with a useful regular expression. Target only paths that
|
|
||||||
exist, and consult the internal component overview for their owning
|
|
||||||
documentation. A filtered or package-specific run does not replace the
|
|
||||||
complete repository validation.
|
|
||||||
|
|
||||||
## Coordinated Work With Scriptorium
|
|
||||||
|
|
||||||
Promptkit and Scriptorium must remain independently valid. For temporary local
|
|
||||||
integration, use either a Go workspace outside both repositories or an
|
|
||||||
uncommitted replacement in the consuming module.
|
|
||||||
|
|
||||||
If the repositories are sibling directories, run the workspace commands from
|
|
||||||
their parent directory:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
go work init ./promptkit ./scriptorium
|
|
||||||
go work sync
|
|
||||||
```
|
|
||||||
|
|
||||||
Use the workspace only for coordinated local checks. From the same parent
|
|
||||||
directory, remove it when finished:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
rm -f go.work go.work.sum
|
|
||||||
```
|
|
||||||
|
|
||||||
Alternatively, from the Scriptorium repository root, temporarily point its
|
|
||||||
Promptkit dependency at the sibling checkout:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
go mod edit -replace gitea.maximumdirect.net/eric/promptkit=../promptkit
|
|
||||||
```
|
|
||||||
|
|
||||||
After coordinated checks, remove the replacement and reconcile module
|
|
||||||
metadata:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
go mod edit -dropreplace gitea.maximumdirect.net/eric/promptkit
|
|
||||||
go mod tidy
|
|
||||||
```
|
|
||||||
|
|
||||||
Never commit `go.work`, `go.work.sum`, or a local filesystem `replace`
|
|
||||||
directive. Before committing in either repository, inspect its module files and
|
|
||||||
working tree independently. Published consumer versions must depend on a tagged
|
|
||||||
Promptkit version, not a workspace, local replacement, or unpublished commit.
|
|
||||||
|
|||||||
117
docs/formats.md
117
docs/formats.md
@@ -9,9 +9,10 @@ explains how to select these sources and invoke the engine. The
|
|||||||
owns the resulting outbound wire behavior.
|
owns the resulting outbound wire behavior.
|
||||||
|
|
||||||
Prompt and profile sources recursively discover files ending in `.yaml` or
|
Prompt and profile sources recursively discover files ending in `.yaml` or
|
||||||
`.yml`. YAML decoding is strict: unknown fields are errors for the selected
|
`.yml`. Each prompt-definition and profile file contains exactly one YAML
|
||||||
definition. Definitions are selected by their YAML `id`, not their file name
|
document; comments and trailing whitespace are allowed. YAML decoding is
|
||||||
or directory.
|
strict: unknown fields are errors for the selected definition. Definitions are
|
||||||
|
selected by their YAML `id`, not their file name or directory.
|
||||||
|
|
||||||
## Prompt Definitions
|
## Prompt Definitions
|
||||||
|
|
||||||
@@ -58,6 +59,11 @@ When a request omits a version, the selected prompt ID must identify exactly
|
|||||||
one definition. When it supplies a version, the ID and version pair must be
|
one definition. When it supplies a version, the ID and version pair must be
|
||||||
unique.
|
unique.
|
||||||
|
|
||||||
|
Exact prompt inspection uses this same configured source, strict decoding,
|
||||||
|
referenced content-file resolution, and ID/version selection. It reports the
|
||||||
|
selected definition's declared metadata without changing the prompt format or
|
||||||
|
executing the definition.
|
||||||
|
|
||||||
### Inputs
|
### Inputs
|
||||||
|
|
||||||
Each `inputs` item has these fields:
|
Each `inputs` item has these fields:
|
||||||
@@ -81,9 +87,16 @@ Each message has a non-empty `role` and exactly one of:
|
|||||||
- `content`, containing an inline Go template; or
|
- `content`, containing an inline Go template; or
|
||||||
- `content_file`, naming a file whose contents are the Go template.
|
- `content_file`, naming a file whose contents are the Go template.
|
||||||
|
|
||||||
For directory and `fs.FS` prompt sources, `content_file` resolves relative to
|
`content_file` must be a relative path. It resolves from the directory that
|
||||||
the prompt file and remains within the source root. `WithPromptFile` also
|
contains the prompt file and must remain within the configured prompt source
|
||||||
resolves it relative to that file.
|
root; parent components are allowed only when the resolved target remains
|
||||||
|
inside that root. Absolute paths and paths that escape the root are rejected.
|
||||||
|
Operating-system directory and single-file sources also reject symlink targets
|
||||||
|
outside the root, while injected `fs.FS` sources apply containment in that
|
||||||
|
filesystem's relative path namespace. For `WithPromptFile`, the source root is
|
||||||
|
the directory containing the selected prompt file. Promptkit uses the parsed
|
||||||
|
path text exactly after checking separately that it is not blank, so leading
|
||||||
|
and trailing whitespace can name real filesystem entries.
|
||||||
|
|
||||||
Request variables are the template data, so a variable named `audience` is
|
Request variables are the template data, so a variable named `audience` is
|
||||||
referenced as `{{.audience}}`. The `{{input "note"}}` helper renders the body
|
referenced as `{{.audience}}`. The `{{input "note"}}` helper renders the body
|
||||||
@@ -91,7 +104,8 @@ of a named input. Missing variables and input references are errors.
|
|||||||
|
|
||||||
The optional `session_id` uses the same template data and input helper. Its
|
The optional `session_id` uses the same template data and input helper. Its
|
||||||
rendered value is trimmed, omitted when empty, and limited to 256 Unicode code
|
rendered value is trimmed, omitted when empty, and limited to 256 Unicode code
|
||||||
points.
|
points. A nonblank direct request session ID bypasses this template completely;
|
||||||
|
a blank direct value leaves the template behavior unchanged.
|
||||||
|
|
||||||
### Cache Control
|
### Cache Control
|
||||||
|
|
||||||
@@ -136,7 +150,7 @@ A profile supplies model execution settings:
|
|||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
id: local-summary
|
id: local-summary
|
||||||
endpoint: http://localhost:8000/v1
|
backend: openrouter
|
||||||
model: example-model
|
model: example-model
|
||||||
temperature: 0.2
|
temperature: 0.2
|
||||||
max_tokens: 500
|
max_tokens: 500
|
||||||
@@ -144,15 +158,15 @@ top_p: 0.95
|
|||||||
timeout_seconds: 90
|
timeout_seconds: 90
|
||||||
service_tier: flex
|
service_tier: flex
|
||||||
reasoning_effort: medium
|
reasoning_effort: medium
|
||||||
api_key_env: EXAMPLE_API_KEY
|
|
||||||
extra_params:
|
extra_params:
|
||||||
provider_option: enabled
|
provider_option: enabled
|
||||||
```
|
```
|
||||||
|
|
||||||
| Field | Required | Meaning |
|
| Field | Required | Meaning |
|
||||||
| --- | --- | --- |
|
| --- | --- | --- |
|
||||||
| `id` | yes | Non-empty profile identifier. IDs must be unique within one source. |
|
| `id` | yes | Profile identifier, trimmed before selection and publication. It must be non-empty after trimming and unique within one source after normalization. |
|
||||||
| `endpoint` | yes | Non-empty OpenAI-compatible base URL, including an API version path when required. |
|
| `backend` | unless `endpoint` is present | Backend registry ID. It is trimmed and registry membership is checked when the profile is prepared or inspected. |
|
||||||
|
| `endpoint` | unless `backend` is present | OpenAI-compatible base URL, including an API version path when required. A nonempty value is trimmed and must be absolute HTTP or HTTPS with a host and without user information, a query, or a fragment. When both connection fields are present, this overrides the backend endpoint without changing backend identity. |
|
||||||
| `model` | yes | Non-empty provider model name. |
|
| `model` | yes | Non-empty provider model name. |
|
||||||
| `temperature` | no | Number from 0 through 2. |
|
| `temperature` | no | Number from 0 through 2. |
|
||||||
| `max_tokens` | no | Integer zero or greater. |
|
| `max_tokens` | no | Integer zero or greater. |
|
||||||
@@ -166,60 +180,89 @@ extra_params:
|
|||||||
Raw `api_key` is prohibited in profile YAML. Store only an environment
|
Raw `api_key` is prohibited in profile YAML. Store only an environment
|
||||||
variable name in `api_key_env`.
|
variable name in `api_key_env`.
|
||||||
|
|
||||||
|
Promptkit does not infer a backend from a model or endpoint. Endpoint-only
|
||||||
|
profiles remain supported and have no effective backend ID.
|
||||||
|
The engine always provides the built-in `openrouter` ID. Consumers can add
|
||||||
|
engine-scoped IDs with
|
||||||
|
[`WithBackend`](../backends.go); exact registration validation belongs to its
|
||||||
|
GoDoc.
|
||||||
|
|
||||||
`extra_params` accepts null, booleans, finite numbers, strings, arrays, and
|
`extra_params` accepts null, booleans, finite numbers, strings, arrays, and
|
||||||
objects with string keys. Keys must be non-empty. With the built-in client,
|
objects with string keys. Keys must be non-empty. With the built-in client,
|
||||||
they also cannot collide with the standard fields listed in the
|
they also cannot collide with the standard fields listed in the
|
||||||
[outbound request contract](integrations/openai-compatible-chat.md#request-body).
|
[outbound request contract](integrations/openai-compatible-chat.md#request-body).
|
||||||
|
Excessively deep or large JSON-shaped values are rejected for safety.
|
||||||
|
|
||||||
### Defaults And Overrides
|
### Defaults And Overrides
|
||||||
|
|
||||||
Execution settings resolve in this order:
|
Execution settings resolve in this order:
|
||||||
|
|
||||||
1. framework defaults;
|
1. the framework timeout baseline;
|
||||||
2. the selected profile; and
|
2. the selected backend, when the profile names one;
|
||||||
3. request `ExecutionTargetOverride` values.
|
3. the selected profile; and
|
||||||
|
4. request `ExecutionTargetOverride` values.
|
||||||
|
|
||||||
The framework defaults are:
|
The framework baseline is:
|
||||||
|
|
||||||
| Setting | Default |
|
| Setting | Default |
|
||||||
| --- | --- |
|
| --- | --- |
|
||||||
| `temperature` | `0` |
|
| `temperature` | Unspecified and omitted from compatible provider requests unless a profile or runtime override selects it. |
|
||||||
| `max_tokens` | `0` |
|
| `max_tokens` | Unspecified and omitted from compatible provider requests unless a profile or runtime override selects it. |
|
||||||
| `top_p` | `1` |
|
| `top_p` | Unspecified and omitted from compatible provider requests unless a profile or runtime override selects it. |
|
||||||
| `timeout_seconds` | `600` |
|
| `timeout_seconds` | `600` |
|
||||||
|
|
||||||
Numeric zero in a file or in-memory profile means that the profile does not
|
Numeric zero in a file or in-memory profile does not select a numeric value.
|
||||||
replace the framework default. Numeric request overrides use pointers, so an
|
For `temperature`, `max_tokens`, and `top_p`, it leaves the provider control
|
||||||
explicit zero is preserved. In particular, an explicit request
|
unspecified. For `timeout_seconds`, it retains the framework deadline. Numeric
|
||||||
`timeout_seconds` of zero disables the per-generation deadline while leaving
|
request overrides use pointers, so an explicit zero is retained and sent to
|
||||||
the caller context and transport timeout intact.
|
compatible providers. In particular, an explicit request `timeout_seconds` of
|
||||||
|
zero disables the per-generation deadline while leaving the caller context and
|
||||||
|
transport timeout intact.
|
||||||
|
|
||||||
Non-empty request strings replace profile strings. A non-empty request
|
Non-empty profile strings replace backend defaults, and non-empty request
|
||||||
`ExtraParams` map replaces the profile map rather than merging keys.
|
strings replace both. Request reasoning is the exception: a nil
|
||||||
|
`ReasoningEffort` pointer inherits the profile, a pointer to a nonblank string
|
||||||
|
trims and replaces it, and a pointer to a blank string clears it. Backend
|
||||||
|
identity is retained when either layer overrides the endpoint, so the override
|
||||||
|
also retains any engine-local capacity policy configured for that backend.
|
||||||
|
Capacity configuration belongs to the Go
|
||||||
|
[`Backend` API](../backends.go), not prompt or profile YAML. A non-empty
|
||||||
|
`extra_params` map at each layer replaces the entire lower-precedence map
|
||||||
|
rather than merging keys.
|
||||||
The [outbound integration contract](integrations/openai-compatible-chat.md)
|
The [outbound integration contract](integrations/openai-compatible-chat.md)
|
||||||
defines how the effective settings are serialized.
|
defines how the effective settings are serialized.
|
||||||
|
|
||||||
### Source And Profile Precedence
|
### Source And Profile Precedence
|
||||||
|
|
||||||
An explicit request profile ID takes precedence over the prompt's
|
An explicit request profile ID takes precedence over the prompt's
|
||||||
`default_profile`. If neither is present, preparation fails.
|
`default_profile`. If neither is present, preparation fails. Exact profile
|
||||||
|
inspection instead takes one explicit profile ID and does not use a prompt
|
||||||
|
default.
|
||||||
|
|
||||||
Profile sources resolve matching IDs in this order:
|
Profile sources resolve matching IDs in this order:
|
||||||
|
|
||||||
1. in-memory profiles supplied with `WithProfiles`;
|
1. in-memory profiles supplied with `WithProfiles`;
|
||||||
2. a profile file, `fs.FS`, or configured profile directory; and
|
2. the ordinary configured source selected by a profile file, `fs.FS`, or
|
||||||
3. embedded built-in profiles.
|
configured profile directory;
|
||||||
|
3. application fallback profiles supplied with `WithFallbackProfileFS`; and
|
||||||
|
4. embedded built-in profiles.
|
||||||
|
|
||||||
A higher-precedence source falls back only when the profile is absent. An
|
A profile source supplies a complete definition; definitions and their fields
|
||||||
invalid matching profile is an error and does not fall back. In-memory
|
are not merged across sources. A higher-precedence source falls back only when
|
||||||
`Profile` values follow the same ranges as YAML profiles. They use
|
the requested profile ID is absent. An invalid matching profile is an error and
|
||||||
`APIKeyRequired` for request-scoped credentials instead of `api_key_env`.
|
does not fall back. In-memory `Profile` values follow the same ranges as YAML
|
||||||
|
profiles. They use `APIKeyRequired` for request-scoped credentials instead of
|
||||||
|
`api_key_env`. Preparation and exact profile inspection use this same source
|
||||||
|
precedence.
|
||||||
|
|
||||||
## Built-In Profile Catalog
|
## Built-In Profile Catalog
|
||||||
|
|
||||||
Built-ins use the OpenRouter-compatible endpoint and
|
Every built-in selects the `openrouter` backend. The engine's built-in backend
|
||||||
`OPENROUTER_API_KEY`. A custom or in-memory profile with the same ID takes
|
registry supplies `https://openrouter.ai/api/v1` and the environment-variable
|
||||||
precedence.
|
name `OPENROUTER_API_KEY`, so individual profiles contain only model and
|
||||||
|
generation settings. Built-in profile files do not repeat those connection
|
||||||
|
values. A configured, application fallback, or in-memory profile with the same
|
||||||
|
profile ID takes precedence.
|
||||||
|
|
||||||
| Provider | ID | Model |
|
| Provider | ID | Model |
|
||||||
| --- | --- | --- |
|
| --- | --- | --- |
|
||||||
@@ -270,6 +313,10 @@ prompt, profile, schema, or example files:
|
|||||||
- a request can provide a direct `APIKey` or override `APIKeyEnv`; and
|
- a request can provide a direct `APIKey` or override `APIKeyEnv`; and
|
||||||
- a direct request key takes precedence over environment lookup.
|
- a direct request key takes precedence over environment lookup.
|
||||||
|
|
||||||
|
After a direct request key, the credential-source precedence is request
|
||||||
|
`APIKeyEnv`, profile `api_key_env`, then the backend default. An in-memory
|
||||||
|
profile with `APIKeyRequired` clears an inherited backend environment name and
|
||||||
|
requires a direct key unless the request explicitly supplies `APIKeyEnv`.
|
||||||
Promptkit validates required credential availability during preparation.
|
Promptkit validates required credential availability during preparation.
|
||||||
Direct keys are excluded from JSON results and redacted by public string
|
Direct keys are excluded from JSON results and redacted by public string
|
||||||
formatters. Environment-variable names may appear in prepared metadata, but
|
formatters. Environment-variable names may appear in prepared metadata, but
|
||||||
|
|||||||
@@ -13,10 +13,21 @@ that produce these outbound settings.
|
|||||||
## Endpoint And Method
|
## Endpoint And Method
|
||||||
|
|
||||||
Generation sends an HTTP `POST` with `Content-Type: application/json`.
|
Generation sends an HTTP `POST` with `Content-Type: application/json`.
|
||||||
A non-empty endpoint from the execution target overrides the client's
|
Before the client is called, the engine resolves framework, backend, profile,
|
||||||
configured base URL. After trailing slashes are removed,
|
and request values into one execution target. Endpoint configuration is trimmed
|
||||||
`/chat/completions` is appended. Generation fails before sending when neither
|
and must be an absolute HTTP or HTTPS URL with a host and without user
|
||||||
source supplies an endpoint.
|
information, a query, or a fragment. A non-empty endpoint from the target
|
||||||
|
overrides the client's configured base URL. The final selected endpoint is
|
||||||
|
validated again before transport.
|
||||||
|
|
||||||
|
The completion URL is composed through parsed URL path operations. Nested base
|
||||||
|
paths are retained, repeated trailing slashes are normalized, and the result
|
||||||
|
has exactly one appended `/chat/completions` suffix. Generation fails before
|
||||||
|
sending when neither source supplies a valid endpoint.
|
||||||
|
|
||||||
|
The target's backend ID is routing metadata for prepared values, results, and
|
||||||
|
injected clients. The built-in client does not derive the URL from that ID and
|
||||||
|
does not serialize it in the provider request.
|
||||||
|
|
||||||
## Authentication
|
## Authentication
|
||||||
|
|
||||||
@@ -26,6 +37,12 @@ the client reads that variable and requires a non-empty value. The selected
|
|||||||
key is sent as `Authorization: Bearer <key>`. No authorization header is sent
|
key is sent as `Authorization: Bearer <key>`. No authorization header is sent
|
||||||
when neither mechanism is configured.
|
when neither mechanism is configured.
|
||||||
|
|
||||||
|
The target contains the already resolved environment-variable name: an
|
||||||
|
explicit request override takes precedence over profile metadata, which takes
|
||||||
|
precedence over the backend default. Only the name reaches prepared metadata;
|
||||||
|
the environment value is read just before the provider call and is never added
|
||||||
|
to the JSON body.
|
||||||
|
|
||||||
## Request Body
|
## Request Body
|
||||||
|
|
||||||
The request body always contains `model` and `messages`. The execution
|
The request body always contains `model` and `messages`. The execution
|
||||||
@@ -36,20 +53,25 @@ Each ordinary message contains its `role` and string `content`. A
|
|||||||
cache-controlled message instead uses a text content block containing `type`,
|
cache-controlled message instead uses a text content block containing `type`,
|
||||||
`text`, and `cache_control`; an empty cache-control TTL is omitted.
|
`text`, and `cache_control`; an empty cache-control TTL is omitted.
|
||||||
|
|
||||||
A non-empty session ID is trimmed, checked against the internal domain limit,
|
The effective direct or prompt-rendered session ID is trimmed, limited to 256
|
||||||
and sent as top-level `session_id`. It is not sent as a session header.
|
Unicode code points, and sent when nonempty as top-level `session_id`. It is
|
||||||
|
never also sent as a session header.
|
||||||
|
|
||||||
The client conditionally includes:
|
The client conditionally includes:
|
||||||
|
|
||||||
- `temperature`, `max_tokens`, and `top_p` when non-zero or explicitly
|
- `temperature`, `max_tokens`, and `top_p` only when selected by a profile or
|
||||||
present;
|
runtime override, including an explicit runtime zero; they are absent when
|
||||||
- non-empty `service_tier` and `reasoning_effort`; and
|
unspecified;
|
||||||
|
- non-empty `service_tier` and effective `reasoning_effort`; an explicitly
|
||||||
|
disabled reasoning setting is empty and therefore omitted; and
|
||||||
- `response_format` for JSON Schema structured output, including its name,
|
- `response_format` for JSON Schema structured output, including its name,
|
||||||
strict flag, and schema document.
|
strict flag, and schema document.
|
||||||
|
|
||||||
Extra parameters are merged directly into the top-level body after JSON
|
The engine resolves backend, profile, and request extra-parameter maps by
|
||||||
serialization is verified. Empty keys and collisions with these reserved
|
whole-map replacement rather than key merging. The resulting effective map is
|
||||||
fields are rejected before any provider call:
|
then merged directly into the top-level body after JSON serialization is
|
||||||
|
verified. Empty keys and collisions with these reserved fields are rejected
|
||||||
|
before any provider call:
|
||||||
|
|
||||||
- `model`
|
- `model`
|
||||||
- `session_id`
|
- `session_id`
|
||||||
@@ -61,15 +83,34 @@ fields are rejected before any provider call:
|
|||||||
- `reasoning_effort`
|
- `reasoning_effort`
|
||||||
- `response_format`
|
- `response_format`
|
||||||
|
|
||||||
|
`backend_id`, `api_key_env`, and resolved credential values are not provider
|
||||||
|
request fields.
|
||||||
|
|
||||||
## Response Handling
|
## Response Handling
|
||||||
|
|
||||||
Any 2xx response is decoded as an OpenAI-compatible chat response. The client
|
Any 2xx response body is limited to 16 MiB (16,777,216 bytes). A larger
|
||||||
returns the first choice's non-empty message content and maps prompt,
|
declared `Content-Length` is rejected before the body is read, and streamed,
|
||||||
completion, total, cached, and cache-write token counts.
|
chunked, or underreported bodies are read through the same bound with at most
|
||||||
|
one additional byte used to detect overflow. A body exactly at the limit is
|
||||||
|
allowed. The body is closed on every outcome and an oversized stream is not
|
||||||
|
drained.
|
||||||
|
|
||||||
Invalid JSON, absent choices, and empty first-choice content are malformed
|
The bounded body must contain exactly one OpenAI-compatible JSON response
|
||||||
responses. For a non-2xx status, the error includes the status code but never
|
object followed only by JSON whitespace and EOF. The client returns the first
|
||||||
the provider response body.
|
choice's non-empty message content and maps prompt, completion, total, cached,
|
||||||
|
and cache-write token counts. Invalid or truncated JSON, trailing non-whitespace
|
||||||
|
data, a second JSON value, absent choices, empty first-choice content, and size
|
||||||
|
overflow are malformed responses and return no partial result.
|
||||||
|
|
||||||
|
For a non-2xx status, the error includes the status code but never the provider
|
||||||
|
response body. Promptkit does not yet parse provider error envelopes; bounded
|
||||||
|
non-success parsing belongs to the
|
||||||
|
[structured-generation-error roadmap](../roadmap/structured-generation-errors.md).
|
||||||
|
|
||||||
|
An outbound `http.Client.Do` failure retains both Promptkit's request-failure
|
||||||
|
identity and the exact transport error for `errors.Is` and `errors.As` checks.
|
||||||
|
The rendered error does not include the selected endpoint, request headers,
|
||||||
|
request content, credentials, or provider body.
|
||||||
|
|
||||||
## Timeout And Cancellation
|
## Timeout And Cancellation
|
||||||
|
|
||||||
@@ -84,5 +125,7 @@ Timeouts are layered:
|
|||||||
timeout when the supplied value is not positive.
|
timeout when the supplied value is not positive.
|
||||||
|
|
||||||
The earliest applicable caller, generation, or transport deadline controls the
|
The earliest applicable caller, generation, or transport deadline controls the
|
||||||
request. Constructing the internal client does not mutate a supplied
|
request. Caller cancellation retains `context.Canceled`; caller, generation,
|
||||||
`http.Client`.
|
and whole-request timeout failures retain `context.DeadlineExceeded`, together
|
||||||
|
with the request-failure identity. Constructing the internal client does not
|
||||||
|
mutate a supplied `http.Client`.
|
||||||
|
|||||||
118
docs/internal/capacity.md
Normal file
118
docs/internal/capacity.md
Normal file
@@ -0,0 +1,118 @@
|
|||||||
|
# Internal Capacity Management
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
|
||||||
|
This document describes the implemented engine-local capacity coordination in
|
||||||
|
`internal/capacity`. The [architecture policy](../policy/architecture.md) owns
|
||||||
|
component boundaries, the [backend GoDoc](../../backends.go) owns exact public
|
||||||
|
configuration semantics, and the
|
||||||
|
[internal runner document](runner.md) owns orchestration around admission.
|
||||||
|
|
||||||
|
Capacity scheduling is outside the provider wire contract. It does not add
|
||||||
|
fields to execution targets, generated requests, prompt or profile YAML, or
|
||||||
|
stable JSON values.
|
||||||
|
|
||||||
|
## Construction And Pool Lifecycle
|
||||||
|
|
||||||
|
Each root `NewEngine` call obtains a normalized capacity-policy snapshot from
|
||||||
|
its immutable backend registry and constructs a new `Manager`. The manager
|
||||||
|
creates one pool for each limited backend ID. It has no package-global mutable
|
||||||
|
state, background workers, shutdown protocol, or persistence, so engines with
|
||||||
|
the same registrations still have independent capacity.
|
||||||
|
|
||||||
|
Unlimited registered backends and endpoint-only profiles have no pool. Their
|
||||||
|
admission and generation calls take the unrestricted fast path. An endpoint
|
||||||
|
override does not change the selected backend ID and therefore does not change
|
||||||
|
the pool.
|
||||||
|
|
||||||
|
One pool owns immutable active and total limits plus mutex-protected admission
|
||||||
|
count, active count, and ordered waiter list. Pool state exists only for the
|
||||||
|
lifetime of its engine.
|
||||||
|
|
||||||
|
## Bounded Execution Admission
|
||||||
|
|
||||||
|
For ordinary `Run`, the runner asks the manager to admit after resolving the
|
||||||
|
prompt, profile, selected backend, effective execution target, credentials, and
|
||||||
|
output contract, but before schema loading, artifact loading, or rendering.
|
||||||
|
`PrepareExecution` performs no admission. `RunPrepared` claims its handle,
|
||||||
|
rechecks credential availability, and then asks the manager to admit the
|
||||||
|
frozen backend before generation.
|
||||||
|
|
||||||
|
Admission is immediate: a limited pool either reserves a slot or returns only
|
||||||
|
the internal `ErrCapacityExceeded` identity. The runner attaches the selected
|
||||||
|
backend identity at its use-case boundary, and the root facade translates that
|
||||||
|
typed value without treating it as an invalid request or generation failure.
|
||||||
|
|
||||||
|
The total admitted bound is the active-generation limit plus its configured
|
||||||
|
waiting capacity. The returned release function is idempotent. The runner
|
||||||
|
defers it as soon as admission succeeds. An ordinary run holds the lease across
|
||||||
|
remaining preparation, initial generation, validation, every repair attempt,
|
||||||
|
and all failure or cancellation exits. Prepared execution holds the normal
|
||||||
|
lease across generation, validation, every internal repair attempt, and all
|
||||||
|
execution exits. A repair is part of its original admission and does not
|
||||||
|
reserve another bounded slot.
|
||||||
|
|
||||||
|
## FIFO Generation Permits
|
||||||
|
|
||||||
|
`NewClient` wraps the engine's selected internal model client after public
|
||||||
|
client adaptation or built-in client construction. Initial generation and the
|
||||||
|
default repairer receive the same wrapper. Their requests retain the same
|
||||||
|
effective backend, credential, numeric-presence metadata, and structured-output
|
||||||
|
settings, so scheduling does not change provider omission semantics between
|
||||||
|
calls.
|
||||||
|
|
||||||
|
For each `Generate` call, the wrapper selects a pool from the request's
|
||||||
|
effective backend ID. An unlimited call passes directly to the next client. A
|
||||||
|
limited call acquires an active permit, invokes the next client, and defers
|
||||||
|
permit release so ordinary returns and panic unwinding both restore capacity.
|
||||||
|
Preparation and validation never hold an active permit.
|
||||||
|
|
||||||
|
When all active permits are occupied, calls join a mutex-protected FIFO waiter
|
||||||
|
list. Releasing a permit transfers it directly to the oldest remaining waiter
|
||||||
|
before making it generally available. Pools do not order work relative to
|
||||||
|
other backend IDs.
|
||||||
|
|
||||||
|
The wrapper passes generation requests, responses, and collaborator errors
|
||||||
|
through unchanged. It owns scheduling only; the concrete model client remains
|
||||||
|
responsible for provider transport behavior. The runner, rather than the
|
||||||
|
capacity layer, sums all five usage fields from the initial response and every
|
||||||
|
completed repair response into the successful run result.
|
||||||
|
|
||||||
|
## Cancellation And Release
|
||||||
|
|
||||||
|
Admission checks the caller context before reserving a slot. A call canceled
|
||||||
|
while waiting for an active permit removes its waiter under the same pool lock
|
||||||
|
used to grant permits. If cancellation removes the waiter first, the wrapped
|
||||||
|
client is not invoked. If a concurrent grant wins first, the call owns the
|
||||||
|
permit and invokes the client with the original context, allowing the client
|
||||||
|
to observe cancellation normally.
|
||||||
|
|
||||||
|
This grant-or-cancel decision prevents lost and double-released permits.
|
||||||
|
Admission leases and active permits are released after success, collaborator
|
||||||
|
errors, validation failures, cancellation, and panic unwinding. Canceled
|
||||||
|
waiters are unlinked so their contexts and requests are not retained by the
|
||||||
|
pool.
|
||||||
|
|
||||||
|
## Test Ownership
|
||||||
|
|
||||||
|
The [manager tests](../../internal/capacity/manager_test.go) own policy
|
||||||
|
validation, bounded admission, idempotent release, context handling, and
|
||||||
|
unlimited admission. The
|
||||||
|
[client tests](../../internal/capacity/client_test.go) own peak enforcement,
|
||||||
|
FIFO transfer, canceled-waiter removal, grant/cancel races, independent pools,
|
||||||
|
unlimited calls, passthrough behavior, and panic release.
|
||||||
|
|
||||||
|
The [runner tests](../../internal/usecase/runner_test.go) own ordinary early
|
||||||
|
admission, lease lifetime, failure release, and shared initial/repair
|
||||||
|
scheduling. The
|
||||||
|
[prepared-execution use-case tests](../../internal/usecase/prepared_execution_test.go)
|
||||||
|
own deferred admission, credential ordering, and prepared-execution lease
|
||||||
|
release. The
|
||||||
|
[external package capacity tests](../../capacity_contract_test.go) own the
|
||||||
|
assembled public-engine behavior for configured limits, capacity errors,
|
||||||
|
endpoint identity, engine independence, and injected clients. The
|
||||||
|
[prepared-execution contract tests](../../prepared_execution_contract_test.go)
|
||||||
|
own the public prepared-capacity boundary. The
|
||||||
|
[root error-boundary tests](../../errors_internal_test.go) own preservation of
|
||||||
|
the public generation category and context identity when generation is
|
||||||
|
canceled.
|
||||||
@@ -20,20 +20,48 @@ orchestration. `OpenAICompatibleClient` is the built-in implementation. It
|
|||||||
uses internal domain values for rendered prompts, execution targets,
|
uses internal domain values for rendered prompts, execution targets,
|
||||||
structured output, responses, and token usage.
|
structured output, responses, and token usage.
|
||||||
|
|
||||||
Construction validates the configured base URL and clones any supplied
|
The runner supplies a fully resolved target after applying backend, profile,
|
||||||
`http.Client` so Promptkit can apply its timeout default without mutating the
|
and request precedence. The client uses its endpoint, credential metadata,
|
||||||
caller's client. Generation then:
|
generation fields, and extra parameters. `BackendID` remains routing metadata
|
||||||
|
for the generation boundary and is not mapped into the provider payload.
|
||||||
|
|
||||||
1. validates request-level timeout and endpoint requirements;
|
Construction trims and validates a nonempty configured base URL and clones any
|
||||||
|
supplied `http.Client` so Promptkit can apply its timeout default without
|
||||||
|
mutating the caller's client. An empty configured base remains valid because a
|
||||||
|
resolved request target may supply the endpoint. Generation then:
|
||||||
|
|
||||||
|
1. validates shared execution-setting invariants and the final selected base
|
||||||
|
endpoint;
|
||||||
2. maps the internal request into the OpenAI-compatible chat payload;
|
2. maps the internal request into the OpenAI-compatible chat payload;
|
||||||
3. validates and merges extra parameters;
|
3. validates and merges extra parameters;
|
||||||
4. resolves authentication;
|
4. composes `/chat/completions` through parsed URL path operations;
|
||||||
5. performs the outbound request under the applicable deadlines; and
|
5. resolves authentication;
|
||||||
6. decodes the first response choice and token usage.
|
6. performs the outbound request under the applicable deadlines; and
|
||||||
|
7. decodes one strictly framed, size-bounded response object and maps its first
|
||||||
|
choice and token usage.
|
||||||
|
|
||||||
|
`internal/llm` owns the set of reserved OpenAI-compatible request fields used
|
||||||
|
when validating extra parameters. Backend registration consumes the same rule
|
||||||
|
without making the model client depend on registry configuration.
|
||||||
|
|
||||||
The implementation has no retry loop, tool-call support, provider catalog,
|
The implementation has no retry loop, tool-call support, provider catalog,
|
||||||
inbound HTTP behavior, or durable session store.
|
inbound HTTP behavior, or durable session store.
|
||||||
|
|
||||||
|
## Prepared Generation
|
||||||
|
|
||||||
|
For [`RunPrepared`](../../engine.go), the runner supplies the model client with
|
||||||
|
the target, rendered messages, and structured-output constraint retained by
|
||||||
|
executable preparation. Execution does not reopen or rerender consumer
|
||||||
|
sources.
|
||||||
|
|
||||||
|
Before backend admission, the runner rechecks that the frozen credential
|
||||||
|
environment-variable name is available. The handle does not retain the
|
||||||
|
environment value; the model client resolves the value visible when generation
|
||||||
|
begins. A direct request key remains in private execution state only until the
|
||||||
|
claimed execution finishes or an unclaimed handle is discarded. Exact public
|
||||||
|
ownership and redaction semantics belong to the
|
||||||
|
[`PreparedExecution` GoDoc](../../prepared_execution.go).
|
||||||
|
|
||||||
## Failure Categories
|
## Failure Categories
|
||||||
|
|
||||||
The package preserves distinct error identities for invalid client
|
The package preserves distinct error identities for invalid client
|
||||||
@@ -41,9 +69,28 @@ configuration, invalid generation requests, request execution failures,
|
|||||||
non-success provider statuses, and malformed successful responses. Provider
|
non-success provider statuses, and malformed successful responses. Provider
|
||||||
response bodies are not included in non-success errors.
|
response bodies are not included in non-success errors.
|
||||||
|
|
||||||
Caller cancellation and deadline failures during the outbound request are
|
Invalid nonempty configured endpoints are configuration failures. A missing or
|
||||||
reported as request execution failures. The runner classifies these identities
|
invalid final selected endpoint is an invalid generation request and is
|
||||||
without depending on HTTP status mapping.
|
rejected before transport.
|
||||||
|
|
||||||
|
Successful response bodies have a fixed 16 MiB limit enforced by declared
|
||||||
|
length and by reading at most one byte beyond the boundary. The decoder accepts
|
||||||
|
exactly one JSON object plus trailing whitespace and EOF. Size overflow,
|
||||||
|
truncation, malformed JSON, trailing data, and a second value are malformed
|
||||||
|
responses with no partial result or provider content in the error. Every body
|
||||||
|
is closed, and an unbounded oversized stream is not drained. Non-success
|
||||||
|
responses remain status-only; bounded provider error-envelope parsing belongs
|
||||||
|
to the
|
||||||
|
[structured-generation-error roadmap](../roadmap/structured-generation-errors.md).
|
||||||
|
|
||||||
|
An `http.Client.Do` failure is represented by a redacting multi-cause error:
|
||||||
|
the package request-failure sentinel and the exact returned transport error are
|
||||||
|
both available through `errors.Is` and `errors.As`, while the rendered text
|
||||||
|
does not expose the endpoint, headers, request content, credential, transport
|
||||||
|
detail, or provider body. Caller cancellation retains `context.Canceled`;
|
||||||
|
caller deadlines, generation deadlines, and whole-request client timeouts
|
||||||
|
retain `context.DeadlineExceeded`. The runner adds its generation category
|
||||||
|
without discarding those identities or depending on HTTP status mapping.
|
||||||
|
|
||||||
## Test Ownership
|
## Test Ownership
|
||||||
|
|
||||||
@@ -51,5 +98,11 @@ The
|
|||||||
[OpenAI-compatible client tests](../../internal/llm/openai_compatible_client_test.go)
|
[OpenAI-compatible client tests](../../internal/llm/openai_compatible_client_test.go)
|
||||||
own configuration, client cloning, deterministic deadline precedence,
|
own configuration, client cloning, deterministic deadline precedence,
|
||||||
authentication, request and response mapping, malformed data, error identity,
|
authentication, request and response mapping, malformed data, error identity,
|
||||||
cancellation, and response-body suppression. They use local test servers and
|
cancellation, endpoint selection and composition, pre-transport rejection, and
|
||||||
test transports; the default suite makes no live or paid provider requests.
|
bounded single-document response framing, closure, and response-body
|
||||||
|
suppression. The root
|
||||||
|
transport contract tests also verify that resolved backend settings reach this
|
||||||
|
client without serializing backend identity and that ordinary-run cancellation
|
||||||
|
retains its public generation and context identities. All use local test
|
||||||
|
servers or controlled test transports; the default suite makes no live or paid
|
||||||
|
provider requests.
|
||||||
|
|||||||
@@ -11,19 +11,23 @@ contributor workflow and validation.
|
|||||||
|
|
||||||
| Component | Implemented responsibility | References |
|
| Component | Implemented responsibility | References |
|
||||||
| --- | --- | --- |
|
| --- | --- | --- |
|
||||||
| Root `promptkit` package | Provides the supported engine facade, source and injection options, public request and result values, built-in profile construction, extension interfaces, value conversion, redacted formatting, and public error mapping. | [Package GoDoc](../../doc.go), [engine assembly](../../engine.go) |
|
| Root `promptkit` package | Provides the supported engine facade, source, backend-registration, and injection options, public request, result, prompt-inspection, and profile-inspection values, opaque prepared-execution handles, profile construction, extension interfaces, value conversion, redacted formatting, typed capacity errors, public error mapping, and engine-local profile-source assembly including application fallbacks. | [Package GoDoc](../../doc.go), [prepared execution](../../prepared_execution.go), [backend API](../../backends.go), [engine assembly](../../engine.go) |
|
||||||
| `examples/go-library/prepare` | Demonstrates an offline downstream consumer using a prompt file, in-memory profile, inline input, and `Prepare`. It is not a public library package. | [Example program](../../examples/go-library/prepare/main.go) |
|
| `examples/go-library/prepare` | Demonstrates an offline downstream consumer using a prompt file, in-memory profile, inline input, and `Prepare`. It is not a public library package. | [Example program](../../examples/go-library/prepare/main.go) |
|
||||||
| `internal/domain` | Defines internal framework values for requests, artifacts, prompt definitions, profiles, execution targets, rendering, generation, and validation. | [Domain declarations](../../internal/domain/domain.go) |
|
| `examples/go-library/run` | Demonstrates an offline downstream consumer using a prompt file, in-memory profile, inline input, an injected deterministic model client, and `Run`. It is not a public library package. | [Example program](../../examples/go-library/run/main.go) |
|
||||||
|
| `internal/backend` | Constructs each engine's immutable registry from the built-in OpenRouter definition and consumer additions, validates and defensively copies definitions through the shared JSON-value package, and consumes the LLM-owned OpenAI-compatible reserved request-field rule. | [Backend registry](../../internal/backend/registry.go) |
|
||||||
|
| `internal/capacity` | Owns engine-local bounded execution admission and FIFO model-generation permits for limited backend IDs, including cancellation-safe waiter removal and client wrapping. | [Internal capacity management](capacity.md) |
|
||||||
|
| `internal/domain` | Defines internal framework values for requests, artifacts, prompt definitions, profiles, execution targets, rendering, generation, and validation, and owns source-neutral invariants for shared execution settings, OpenAI-compatible base endpoints, session identifiers, and output contracts. Source parsing, required fields, other source-specific normalization, defaulting, and boundary-specific error classification remain with their callers. | [Domain declarations](../../internal/domain/domain.go), [endpoint invariant](../../internal/domain/endpoint.go) |
|
||||||
| `internal/defaults` | Defines application-neutral framework constants and constructs the default execution target. It contains no CLI, server, or inbound HTTP limits. | [Framework defaults](../../internal/defaults/defaults.go) |
|
| `internal/defaults` | Defines application-neutral framework constants and constructs the default execution target. It contains no CLI, server, or inbound HTTP limits. | [Framework defaults](../../internal/defaults/defaults.go) |
|
||||||
| `internal/filecatalog` | Provides deterministic YAML discovery and path helpers for operating-system filesystems and `fs.FS` sources. | [File catalog](../../internal/filecatalog/catalog.go) |
|
| `internal/filecatalog` | Provides deterministic YAML discovery and path helpers for operating-system filesystems and `fs.FS` sources. | [File catalog](../../internal/filecatalog/catalog.go) |
|
||||||
|
| `internal/jsonvalue` | Validates and deeply copies bounded JSON-compatible extra-parameter and prepared-schema trees while preserving supported concrete value types and rejecting cycles or excessive depth and work. | [JSON values](../../internal/jsonvalue/jsonvalue.go) |
|
||||||
| `internal/promptdef` | Loads strictly decoded, validated prompt definitions from filesystem and `fs.FS` sources, including version selection and contained file-backed message content. | [Framework formats](../formats.md), [prompt-definition repository](../../internal/promptdef/filesystem_repository.go) |
|
| `internal/promptdef` | Loads strictly decoded, validated prompt definitions from filesystem and `fs.FS` sources, including version selection and contained file-backed message content. | [Framework formats](../formats.md), [prompt-definition repository](../../internal/promptdef/filesystem_repository.go) |
|
||||||
| `internal/profile` | Loads strictly decoded, validated execution profiles from filesystem and `fs.FS` sources and composes repositories with error-preserving fallback. | [Framework formats](../formats.md), [profile repositories](../../internal/profile/filesystem_repository.go) |
|
| `internal/profile` | Loads strictly decoded, validated execution profiles, including backend selection, from filesystem and `fs.FS` sources and composes repositories with error-preserving fallback. | [Framework formats](../formats.md), [profile repositories](../../internal/profile/filesystem_repository.go) |
|
||||||
| `internal/profile/builtin` | Embeds the built-in execution profile catalog and combines it with an optional primary repository. | [Built-in catalog](../formats.md#built-in-profile-catalog), [repository](../../internal/profile/builtin/repository.go) |
|
| `internal/profile/builtin` | Embeds the built-in profile catalog, whose entries select OpenRouter. | [Built-in catalog](../formats.md#built-in-profile-catalog), [repository](../../internal/profile/builtin/repository.go) |
|
||||||
| `internal/prompt` | Renders prompt messages from Go templates with artifact, variable, session, and cache-control data. | [Go-template renderer](../../internal/prompt/go_renderer.go) |
|
| `internal/prompt` | Renders prompt messages from Go templates with artifact, variable, session, and cache-control data. | [Go-template renderer](../../internal/prompt/go_renderer.go) |
|
||||||
| `internal/artifact` | Resolves ordinary inline and unrestricted caller-selected file references into copied artifacts with metadata and hashes. | [Internal sources and validation](sources.md) |
|
| `internal/artifact` | Resolves ordinary inline and unrestricted caller-selected file references into copied artifacts with metadata and hashes. | [Internal sources and validation](sources.md) |
|
||||||
| `internal/validate` | Validates basic, JSON, and JSON Schema output using operating-system filesystem or `fs.FS` schema sources. | [Framework formats](../formats.md#schemas), [internal sources and validation](sources.md) |
|
| `internal/validate` | Validates basic, JSON, and JSON Schema output using operating-system filesystem or `fs.FS` schema sources and creates operation-local validation plans with canonical contained schema resources. | [Framework formats](../formats.md#schemas), [internal sources and validation](sources.md) |
|
||||||
| `internal/llm` | Defines the internal generation boundary and implements outbound OpenAI-compatible chat requests, response decoding, authentication, and deadline handling. | [Internal model client](llm.md) |
|
| `internal/llm` | Defines the internal generation boundary and implements outbound OpenAI-compatible chat requests from resolved execution targets, including response decoding, authentication, deadline handling, and ownership of the OpenAI-compatible reserved request-field policy. | [Internal model client](llm.md) |
|
||||||
| `internal/usecase` | Coordinates preparation and execution across internal sources, rendering, artifact loading, generation, validation, and optional repair. | [Internal runner](runner.md) |
|
| `internal/usecase` | Resolves prompt definitions and hashes, profiles, backends, and targets for exact inspection and request settings for preparation, and coordinates ordinary execution and one-attempt prepared execution across internal sources, rendering, artifact loading, operation-local validation plans, generation, capacity, and optional repair. | [Internal runner](runner.md), [prepared-execution implementation](../../internal/usecase/prepared_execution.go) |
|
||||||
|
|
||||||
The root package assembles these internal components without exposing their
|
The root package assembles these internal components without exposing their
|
||||||
representations. Consumers depend only on the root facade.
|
representations. Consumers depend only on the root facade.
|
||||||
|
|||||||
@@ -17,51 +17,154 @@ and override semantics consumed by the runner.
|
|||||||
## Collaborators
|
## Collaborators
|
||||||
|
|
||||||
`Runner` coordinates narrow internal interfaces for prompt definitions,
|
`Runner` coordinates narrow internal interfaces for prompt definitions,
|
||||||
profiles, artifacts, rendering, model generation, and validation. Schema
|
profiles, backend resolution, artifacts, rendering, model generation, and
|
||||||
documents are loaded through the validator's optional schema-loader interface.
|
validation. The root engine supplies one immutable registry containing the
|
||||||
|
built-in backend and validated consumer additions, one engine-local run
|
||||||
|
admitter, and a model client wrapped by the same capacity manager. Validation
|
||||||
|
plans and provider-facing schema metadata come from the validator's preparation
|
||||||
|
interface.
|
||||||
An output repairer can be injected internally, but the ordinary runner
|
An output repairer can be injected internally, but the ordinary runner
|
||||||
constructor does not enable one.
|
constructor does not enable one.
|
||||||
|
|
||||||
Each invocation carries its state in request, prepared-run, and result values.
|
Each invocation carries its state in request, prepared-run, and result values.
|
||||||
The runner has no durable run or session store.
|
The runner has no durable run or session store.
|
||||||
|
|
||||||
## Preparation Flow
|
## Shared Prompt Selection
|
||||||
|
|
||||||
`Prepare` performs the reusable pre-generation workflow:
|
The runner uses one prompt-selection and hashing boundary for ordinary
|
||||||
|
preparation and exact prompt inspection. Preparation retains its early
|
||||||
|
request-ID check before direct-session normalization; both operations then use
|
||||||
|
the configured prompt repository to select one definition, load referenced
|
||||||
|
message content, and calculate the same prompt hash.
|
||||||
|
|
||||||
1. validate the prompt selection and load the prompt definition;
|
Inspection stops after that structural lookup. It does not parse templates or
|
||||||
2. hash the loaded definition;
|
touch profile, artifact, schema, renderer, validator, admission, or model
|
||||||
|
collaborators. The root [`Engine.InspectPrompt`](../../engine.go) GoDoc owns
|
||||||
|
the public operation's exact contract.
|
||||||
|
|
||||||
|
## Shared Profile Selection
|
||||||
|
|
||||||
|
The runner uses one profile-selection and target-resolution boundary for
|
||||||
|
ordinary preparation and exact profile inspection. Preparation first selects a
|
||||||
|
request profile or a prompt default; inspection begins with its required
|
||||||
|
explicit profile ID. Both then apply the ordinary source precedence, resolve a
|
||||||
|
named backend, and construct the effective target from framework, backend, and
|
||||||
|
profile values.
|
||||||
|
|
||||||
|
Inspection stops after the resulting endpoint and model are structurally
|
||||||
|
validated. It does not check credential availability or perform prompt,
|
||||||
|
artifact, schema, rendering, admission, or model-client work. The root
|
||||||
|
[`Engine.InspectProfile`](../../engine.go) GoDoc owns the public operation's
|
||||||
|
exact contract.
|
||||||
|
|
||||||
|
## Shared Preparation Pipeline
|
||||||
|
|
||||||
|
`Prepare` and `Run` share one private preparation pipeline split at the point
|
||||||
|
where a run can be assigned to its selected backend pool. The resolution phase
|
||||||
|
performs only the work needed to validate routing and admission:
|
||||||
|
|
||||||
|
1. validate the required prompt selection and normalize any direct session ID;
|
||||||
|
2. load the prompt definition and hash the original definition;
|
||||||
3. select the request profile or the prompt's default profile;
|
3. select the request profile or the prompt's default profile;
|
||||||
4. resolve application-neutral defaults, profile values, and explicit request
|
4. resolve the profile's backend ID, when present;
|
||||||
overrides in that order;
|
5. resolve application-neutral defaults, backend defaults, profile values,
|
||||||
5. validate endpoint, model, numeric overrides, and credential requirements;
|
and explicit request overrides in that order;
|
||||||
6. resolve the output contract and load a structured-output schema when
|
6. validate endpoint, model, numeric overrides, and credential requirements;
|
||||||
required;
|
7. resolve and validate the effective output contract without loading its
|
||||||
7. load and hash input artifacts;
|
schema; and
|
||||||
8. render and hash the prompt; and
|
8. retain the definition, source identities, effective settings, output
|
||||||
9. return the effective settings, source identities, messages, hashes, and
|
contract, and preparation start time in invocation-local state.
|
||||||
preparation timing.
|
|
||||||
|
The completion phase consumes that state without reloading the prompt,
|
||||||
|
profile, or backend:
|
||||||
|
|
||||||
|
1. create one operation-local validation plan and derive structured-output
|
||||||
|
schema metadata from it when required;
|
||||||
|
2. load and hash input artifacts;
|
||||||
|
3. render messages and the prompt-defined session;
|
||||||
|
4. apply any direct session ID;
|
||||||
|
5. hash the effective rendered prompt; and
|
||||||
|
6. construct the prepared value and preparation timing.
|
||||||
|
|
||||||
|
`Prepare` runs both phases consecutively and never performs capacity admission.
|
||||||
|
`Run` performs backend admission between the phases. This structure preserves
|
||||||
|
one execution-precedence and error-ordering implementation while allowing a
|
||||||
|
full backend pool to reject work before expensive schema, artifact, and
|
||||||
|
rendering operations. `Prepare` discards the plan after returning its public
|
||||||
|
metadata. `Run` retains the plan through initial and repaired-output validation
|
||||||
|
and discards it when the operation ends. Prepared execution stores the same
|
||||||
|
kind of plan only in its private payload.
|
||||||
|
|
||||||
Pointer-based numeric overrides preserve an explicit zero. Invalid negative or
|
Pointer-based numeric overrides preserve an explicit zero. Invalid negative or
|
||||||
out-of-range values fail as invalid requests. A direct API key takes
|
out-of-range values fail as invalid requests. Endpoint overrides do not change
|
||||||
precedence over environment lookup for execution; secret values remain
|
the selected backend identity. Non-empty extra-parameter maps replace whole
|
||||||
excluded from serialized metadata.
|
lower-precedence maps. A direct API key takes precedence over environment
|
||||||
|
lookup; otherwise request, profile, and backend environment-variable names
|
||||||
|
apply in that order. A profile requiring a direct key clears an inherited
|
||||||
|
backend environment name unless the request supplies its own name. Secret
|
||||||
|
values remain excluded from serialized metadata.
|
||||||
|
|
||||||
|
Reasoning overrides are tri-state: nil inherits the profile, a pointer to a
|
||||||
|
nonblank string trims and replaces it, and a pointer to a blank string clears
|
||||||
|
it. A nonblank direct session is normalized before source loading, bypasses
|
||||||
|
the prompt session template, and is applied after ordinary message rendering.
|
||||||
|
A blank direct value retains prompt-template behavior. The runner clears the
|
||||||
|
template only on a value copy of the definition, so the definition hash always
|
||||||
|
describes the original source while the rendered-prompt hash includes the
|
||||||
|
effective direct or rendered session.
|
||||||
|
|
||||||
|
The registry is read-only after engine construction. Concurrent `Prepare` and
|
||||||
|
`Run` calls resolve independent defensive backend values and keep all
|
||||||
|
invocation state local.
|
||||||
|
|
||||||
## Run Flow
|
## Run Flow
|
||||||
|
|
||||||
`Run` calls `Prepare` rather than maintaining a second preparation path. It
|
`Run` records its start time, performs the shared resolution phase, and asks
|
||||||
performs one initial generation call, builds the named output artifact, and
|
its `RunAdmitter` to reserve capacity for the effective backend ID. A nil
|
||||||
validates that artifact. Invalid generated content remains a validation result;
|
admitter is an internal unlimited fallback. After successful admission, `Run`
|
||||||
an inability to perform validation is an operational error.
|
immediately defers the returned release function, performs the completion
|
||||||
|
phase, makes one initial generation call, builds the named output artifact,
|
||||||
|
and validates that artifact with the plan compiled during completion. Invalid
|
||||||
|
generated content remains a validation result; an inability to perform
|
||||||
|
validation is an operational error.
|
||||||
|
|
||||||
|
Validation preparation and execution honor cancellation at every
|
||||||
|
Promptkit-controlled boundary and do not publish a partial plan or result.
|
||||||
|
Schema reads are bounded and context-checked between chunks; JSON decoding,
|
||||||
|
schema compilation, and schema execution are checked immediately before and
|
||||||
|
after their synchronous calls. Promptkit does not move arbitrary filesystem or
|
||||||
|
JSON Schema work to background goroutines, so an already-blocked dependency
|
||||||
|
method must return before cancellation can take precedence over its outcome.
|
||||||
|
|
||||||
|
The admission lease covers completion-phase preparation, initial generation,
|
||||||
|
validation, every repair, and every exit. It bounds accepted work without
|
||||||
|
serializing preparation or validation behind the active-generation limit.
|
||||||
|
The wrapped model client separately acquires a FIFO active permit only around
|
||||||
|
each actual generation call.
|
||||||
|
|
||||||
When an internal repairer is present, a JSON or JSON Schema content failure can
|
When an internal repairer is present, a JSON or JSON Schema content failure can
|
||||||
trigger bounded repair attempts. Repair receives the effective execution
|
trigger bounded repair attempts. Repair receives the effective execution
|
||||||
target, validation errors, prior output, and structured-output specification.
|
target, explicit numeric-presence bits, credential, backend identity, session
|
||||||
|
ID, validation errors, prior output, and structured-output specification. One
|
||||||
|
request constructor supplies those common fields to initial and repair
|
||||||
|
generation while their rendered prompts remain intentionally distinct. The
|
||||||
|
default repairer uses the same wrapped client as initial generation, so each
|
||||||
|
repair reacquires the selected backend's active permit while remaining inside
|
||||||
|
its original admission lease. Repair never performs a second bounded
|
||||||
|
admission, and repaired outputs use the operation's existing validation plan.
|
||||||
This capability remains internal and is not a public option.
|
This capability remains internal and is not a public option.
|
||||||
|
|
||||||
A successful result includes the output artifact and raw output, validation
|
A successful result includes the output artifact and raw output, validation
|
||||||
state, prompt and rendered-prompt hashes, selected profile, effective settings,
|
state, effective session ID, prompt and rendered-prompt hashes, selected
|
||||||
input hashes, token usage, a generated run identifier, and UTC timing.
|
profile and backend, effective settings, input hashes, token usage, a generated
|
||||||
|
run identifier, and UTC timing. The same effective session reaches initial
|
||||||
|
generation and any repair attempt through the rendered prompt. The same
|
||||||
|
effective target and presence metadata, including backend identity and direct
|
||||||
|
credential during execution, reaches generation and every repair attempt.
|
||||||
|
Result usage is the field-wise sum of all five usage values from the initial
|
||||||
|
response and every completed repair response. Final raw output, artifact, and
|
||||||
|
validation state still come from the last candidate. A repair error returns no
|
||||||
|
partial run result or partial usage.
|
||||||
|
|
||||||
## Failure Categories
|
## Failure Categories
|
||||||
|
|
||||||
@@ -69,17 +172,42 @@ Package errors distinguish invalid requests, required profile selection,
|
|||||||
credential failures, and prompt, profile, artifact, rendering, generation, and
|
credential failures, and prompt, profile, artifact, rendering, generation, and
|
||||||
validation failures. Wrapping preserves the package identities mapped by the
|
validation failures. Wrapping preserves the package identities mapped by the
|
||||||
public facade and retains collaborator identities where they are part of the
|
public facade and retains collaborator identities where they are part of the
|
||||||
internal contract. Context cancellation propagates through the invoked
|
internal contract.
|
||||||
collaborator and is classified by the owning operation.
|
|
||||||
|
Admission capacity exhaustion retains the internal capacity identity. At the
|
||||||
|
use-case boundary, the runner attaches the selected backend ID in an internal
|
||||||
|
typed error, and the root facade copies that value into the public
|
||||||
|
[`CapacityError`](../../capacity_error.go) without parsing diagnostic text. It
|
||||||
|
is not recategorized as an invalid request or generation failure, and no
|
||||||
|
partial result is returned. A context already done at admission retains its
|
||||||
|
context identity directly. Cancellation while waiting for an active generation
|
||||||
|
permit prevents client invocation when it wins the grant race; the model-client
|
||||||
|
boundary then preserves the context error through the generation-failure
|
||||||
|
category. Deferred release restores the admission lease on preparation,
|
||||||
|
generation, validation, repair, and cancellation failures.
|
||||||
|
|
||||||
|
Other context cancellation propagates through the invoked collaborator and is
|
||||||
|
classified by the owning operation. In particular, cancellation observed by
|
||||||
|
validation retains the context identity through the validation error category.
|
||||||
|
An overlong direct session is an invalid request before source loading, while
|
||||||
|
an invalid or overlong prompt session template remains a prompt-render failure.
|
||||||
|
An unknown selected backend, or a selected backend with no configured resolver,
|
||||||
|
is classified as a profile-load failure.
|
||||||
|
|
||||||
## Test Ownership And Changes
|
## Test Ownership And Changes
|
||||||
|
|
||||||
The [runner tests](../../internal/usecase/runner_test.go) own preparation order,
|
The [runner tests](../../internal/usecase/runner_test.go) own preparation order,
|
||||||
selection and override precedence, schema-before-generation behavior, hashing,
|
selection and override precedence, the two-phase boundary, early admission,
|
||||||
generation and validation outcomes, bounded repair, credentials and redaction,
|
lease lifetime and release, direct-session resolution, schema-before-generation
|
||||||
error categories, artifact metadata, usage, and timing.
|
behavior, hashing, generation and validation outcomes, backend propagation,
|
||||||
|
bounded repair progression, initial/repair request parity, cumulative usage,
|
||||||
|
shared initial/repair capacity, credentials and redaction, error categories,
|
||||||
|
artifact metadata, and timing. The
|
||||||
|
[capacity subsystem document](capacity.md) identifies the focused pool,
|
||||||
|
waiter, and wrapped-client tests.
|
||||||
|
|
||||||
Changes to orchestration should continue to use the existing package
|
Changes to orchestration should continue to use the existing package
|
||||||
interfaces, keep request state local to an invocation, and preserve `Run`'s use
|
interfaces, keep request state local to an invocation, and preserve the shared
|
||||||
of `Prepare`. Source, renderer, validator, or model-client contract changes
|
resolution and completion pipeline. Source, renderer, validator, or
|
||||||
belong first in their owning package and document.
|
model-client contract changes belong first in their owning package and
|
||||||
|
document.
|
||||||
|
|||||||
@@ -12,9 +12,26 @@ validation modes, built-in catalog, and source precedence.
|
|||||||
|
|
||||||
## Prompt Definitions
|
## Prompt Definitions
|
||||||
|
|
||||||
`internal/promptdef` discovers YAML deterministically, decodes and validates
|
`internal/promptdef` uses one source-neutral flow for prompt selection and
|
||||||
definitions, selects an ID and optional version, and resolves file-backed
|
normalization. That flow scans normalized YAML ID and version metadata,
|
||||||
message content within the selected operating-system or `fs.FS` source.
|
requires one strictly decoded document per file, classifies errors for the
|
||||||
|
selected definition, detects duplicates, and normalizes the exact match.
|
||||||
|
Small operating-system and `fs.FS` adapters own discovery, byte reads, display
|
||||||
|
paths, content opening, and root containment. Each lookup remains a
|
||||||
|
point-in-time scan: definitions and catalogs are not cached, and file-backed
|
||||||
|
message content is opened only for the exact selected candidate.
|
||||||
|
|
||||||
|
Operating-system sources enforce containment against canonical roots and
|
||||||
|
targets so symlinks cannot escape. Injected `fs.FS` sources enforce containment
|
||||||
|
in their clean relative path namespace. A single-file source uses the selected
|
||||||
|
prompt file's containing directory as its root. Every content path must be
|
||||||
|
relative and is opened from its exact parsed text after a separate blank check;
|
||||||
|
contained parent components and whitespace-bearing names remain valid.
|
||||||
|
|
||||||
|
Exact prompt inspection performs one point-in-time lookup through that same
|
||||||
|
repository and validates referenced message content before returning declared
|
||||||
|
metadata. It does not parse templates or read profile, input, or schema
|
||||||
|
sources, and it does not retain the definition for a later execution.
|
||||||
|
|
||||||
Its package tests own prompt selection, strict decoding, definition validation,
|
Its package tests own prompt selection, strict decoding, definition validation,
|
||||||
duplicate detection, and source containment:
|
duplicate detection, and source containment:
|
||||||
@@ -23,22 +40,56 @@ duplicate detection, and source containment:
|
|||||||
## Profiles And Built-Ins
|
## Profiles And Built-Ins
|
||||||
|
|
||||||
`internal/profile` loads and validates execution profiles from an
|
`internal/profile` loads and validates execution profiles from an
|
||||||
operating-system filesystem or an `fs.FS`. It supports a primary repository
|
operating-system filesystem or an `fs.FS`. A file contains exactly one YAML
|
||||||
with fallback only when the primary reports that a profile is absent.
|
document and its trimmed YAML `id` is its only selection identity; filenames do
|
||||||
|
not confer authority. Each point lookup reads discovered files once for their
|
||||||
|
metadata and reuses the selected file's bytes for strict decoding; unrelated
|
||||||
|
profiles are not fully decoded. Strict selected decoding recognizes the
|
||||||
|
optional `backend` field, trims its value, and requires a model plus at least
|
||||||
|
one non-blank backend or endpoint. File-backed `extra_params` values are
|
||||||
|
validated and defensively copied through the shared bounded JSON-value owner
|
||||||
|
before a profile is published. OpenAI-compatible reserved-field policy remains
|
||||||
|
with the model-client and backend-registry owners.
|
||||||
|
|
||||||
`internal/profile/builtin` embeds the maintained built-in profile catalog and
|
The overlay repository consults the next repository only when the
|
||||||
can place a caller-selected repository ahead of that catalog. Profile behavior
|
higher-precedence repository reports that a profile is absent. A reliably
|
||||||
is owned by the
|
selected malformed profile stops fallback, while an unrelated malformed file
|
||||||
[profile repository tests](../../internal/profile/repository_test.go), while
|
does not become authoritative through its filename. Loading does not check
|
||||||
catalog completeness, duplicate IDs, and overlay behavior are owned by the
|
backend registry membership because the available registry belongs to the
|
||||||
|
assembled engine; the runner checks membership during preparation and exact
|
||||||
|
profile inspection.
|
||||||
|
|
||||||
|
The root engine assembles profile repositories in precedence order: in-memory
|
||||||
|
profiles, one ordinary configured source, an application fallback source, then
|
||||||
|
the embedded built-in catalog. An explicit file or `fs.FS` profile source
|
||||||
|
replaces `Config.ProfileDir` within the ordinary configured-source category.
|
||||||
|
|
||||||
|
Exact profile inspection performs one point-in-time lookup through those
|
||||||
|
profile sources and checks the resolved target without reading prompt, input,
|
||||||
|
or schema sources. It does not retain that lookup for a later execution.
|
||||||
|
|
||||||
|
`internal/profile/builtin` embeds the maintained built-in profile catalog.
|
||||||
|
Every embedded profile selects `openrouter` and inherits its endpoint and
|
||||||
|
credential environment-variable name from the built-in backend registry rather
|
||||||
|
than repeating those values. Profile loading and overlay behavior are owned by
|
||||||
|
the [profile repository tests](../../internal/profile/repository_test.go),
|
||||||
|
while catalog completeness, the backend-selection invariant, and duplicate IDs
|
||||||
|
are owned by the
|
||||||
[built-in repository tests](../../internal/profile/builtin/repository_test.go).
|
[built-in repository tests](../../internal/profile/builtin/repository_test.go).
|
||||||
|
|
||||||
## Ordinary Artifacts
|
## Ordinary Artifacts
|
||||||
|
|
||||||
`internal/artifact` resolves inline references and unrestricted,
|
`internal/artifact` accepts explicitly typed inline references even when their
|
||||||
caller-selected file paths. It copies content into an artifact, records
|
body is empty. It also resolves unrestricted, caller-selected paths only when
|
||||||
metadata and a content hash, applies a content-type fallback, and honors
|
they identify regular operating-system files, checking that condition before
|
||||||
context cancellation.
|
and after opening the file. It copies content into an artifact, records
|
||||||
|
metadata and an opaque content-equality value, and applies a content-type
|
||||||
|
fallback.
|
||||||
|
|
||||||
|
Regular files are read synchronously in bounded chunks. Cancellation is
|
||||||
|
checked before opening, before and after every read, and before publishing the
|
||||||
|
artifact, so a canceled read never publishes partial content. The ordinary
|
||||||
|
reader does not detach file reads into background goroutines.
|
||||||
|
|
||||||
This ordinary reader does not implement an inbound HTTP security boundary. In
|
This ordinary reader does not implement an inbound HTTP security boundary. In
|
||||||
particular, it does not constrain files to an application root or impose an
|
particular, it does not constrain files to an application root or impose an
|
||||||
@@ -50,8 +101,17 @@ implemented reader behavior and failures.
|
|||||||
## Rendering
|
## Rendering
|
||||||
|
|
||||||
`internal/prompt` renders definition messages as Go templates using named
|
`internal/prompt` renders definition messages as Go templates using named
|
||||||
artifacts and variables. It carries message roles, session IDs, and cache
|
artifacts and variables. Within one render, each referenced artifact body is
|
||||||
control into the rendered prompt. The
|
converted to text lazily and cached by input name for reuse across the session
|
||||||
|
and every message; the cache is not shared across renders. Conversion uses
|
||||||
|
bounded chunks and preserves the artifact bytes exactly.
|
||||||
|
|
||||||
|
Session and message parsing and execution remain synchronous. The renderer
|
||||||
|
checks cancellation before and after each parse and execution boundary,
|
||||||
|
between artifact conversion chunks, around each message, and before publishing
|
||||||
|
the complete prompt. It cannot interrupt template work already in progress and
|
||||||
|
never publishes a partial prompt after observing cancellation. It carries
|
||||||
|
message roles, session IDs, and cache control into the rendered prompt. The
|
||||||
[renderer tests](../../internal/prompt/renderer_test.go) own rendering behavior.
|
[renderer tests](../../internal/prompt/renderer_test.go) own rendering behavior.
|
||||||
|
|
||||||
## Schemas And Output Validation
|
## Schemas And Output Validation
|
||||||
@@ -61,6 +121,35 @@ filesystem or an `fs.FS`. Invalid generated content is returned as a validation
|
|||||||
result; inability to load, register, or compile a schema is an operational
|
result; inability to load, register, or compile a schema is an operational
|
||||||
error.
|
error.
|
||||||
|
|
||||||
|
Every preparation operation creates one operation-local validation plan. None,
|
||||||
|
basic, and JSON modes retain the effective output contract without source
|
||||||
|
access. JSON Schema mode loads the root document once, resolves and compiles
|
||||||
|
each transitive reference, and retains the compiled validator. Schema compiler
|
||||||
|
resources use canonical escaped file or private-scheme URLs; loaders decode
|
||||||
|
their paths once and enforce the configured source boundary. The
|
||||||
|
provider-facing structured-output metadata uses the root document captured by
|
||||||
|
the same plan.
|
||||||
|
|
||||||
|
Schema preparation and execution remain synchronous. Promptkit checks
|
||||||
|
cancellation before and after source resolution, JSON decoding, compilation,
|
||||||
|
and validation, and between bounded schema-read chunks. Once cancellation is
|
||||||
|
observed it returns the context error without publishing a partial plan or
|
||||||
|
validation result, even when a compiler or validator has just returned a
|
||||||
|
different error or a successful result. An `fs.FS` method or JSON Schema
|
||||||
|
dependency call already in progress cannot be preempted; Promptkit waits for
|
||||||
|
that call to return and then gives cancellation precedence. Validation does
|
||||||
|
not detach dependency work into background goroutines.
|
||||||
|
|
||||||
|
`Prepare` discards its validation plan after returning metadata. `Run` retains
|
||||||
|
its plan for initial and repaired-output validation, then discards it with the
|
||||||
|
operation. `PrepareExecution` retains the plan in its private frozen payload;
|
||||||
|
`RunPrepared` uses that plan without reopening prompt, profile, input, or
|
||||||
|
schema sources or rerendering the request. A later ordinary `Run` always
|
||||||
|
performs fresh source resolution and preparation.
|
||||||
|
|
||||||
The [validator tests](../../internal/validate/standard_validator_test.go) own
|
The [validator tests](../../internal/validate/standard_validator_test.go) own
|
||||||
basic, JSON, JSON Schema, source resolution, schema loading, compilation, and
|
basic, JSON, JSON Schema, source resolution, schema loading, compilation,
|
||||||
content-failure behavior.
|
frozen-reference behavior, content-failure behavior, and the synchronous
|
||||||
|
cancellation boundary. Prepared execution
|
||||||
|
orchestration is owned by the
|
||||||
|
[use-case tests](../../internal/usecase/prepared_execution_test.go).
|
||||||
|
|||||||
@@ -19,12 +19,18 @@ results, public values, extension interfaces, profiles, and error sentinels.
|
|||||||
|
|
||||||
The implemented internal components consist of:
|
The implemented internal components consist of:
|
||||||
|
|
||||||
- `internal/domain`, which owns framework data values shared by later internal
|
- `internal/domain`, which owns framework data values and source-neutral
|
||||||
components;
|
invariants shared by later internal components;
|
||||||
|
- `internal/backend`, which owns validated immutable OpenAI-compatible backend
|
||||||
|
definitions and the built-in OpenRouter definition;
|
||||||
|
- `internal/capacity`, which owns engine-local bounded run admission and
|
||||||
|
model-generation scheduling for limited backends;
|
||||||
- `internal/defaults`, which owns application-neutral framework defaults and
|
- `internal/defaults`, which owns application-neutral framework defaults and
|
||||||
constructs the default execution target;
|
constructs the default execution target;
|
||||||
- `internal/filecatalog`, which discovers YAML files and provides source-path
|
- `internal/filecatalog`, which discovers YAML files and provides source-path
|
||||||
helpers for filesystem and `fs.FS` consumers;
|
helpers for filesystem and `fs.FS` consumers;
|
||||||
|
- `internal/jsonvalue`, which validates and defensively copies JSON-compatible
|
||||||
|
extra-parameter trees;
|
||||||
- `internal/promptdef`, which loads and validates prompt definitions from
|
- `internal/promptdef`, which loads and validates prompt definitions from
|
||||||
filesystem and `fs.FS` sources;
|
filesystem and `fs.FS` sources;
|
||||||
- `internal/profile`, which loads, validates, and overlays execution profiles
|
- `internal/profile`, which loads, validates, and overlays execution profiles
|
||||||
@@ -41,21 +47,25 @@ The implemented internal components consist of:
|
|||||||
- `internal/usecase`, which coordinates preparation and execution across the
|
- `internal/usecase`, which coordinates preparation and execution across the
|
||||||
internal framework components.
|
internal framework components.
|
||||||
|
|
||||||
The `examples/go-library/prepare` package is a maintained downstream consumer
|
The `examples/go-library/prepare` and `examples/go-library/run` packages are
|
||||||
of the root facade. It does not expose a library package or participate in
|
maintained downstream consumers of the root facade. They do not expose library
|
||||||
internal assembly.
|
packages or participate in internal assembly.
|
||||||
|
|
||||||
The root facade assembles the internal repositories, renderer, validator,
|
The root facade assembles one immutable backend registry, one capacity manager,
|
||||||
outbound client, and use-case runner while translating public values and
|
the internal repositories, renderer, validator, outbound client, and use-case
|
||||||
errors at the library boundary. The defaults and renderer depend on the domain
|
runner while translating public values and errors at the library boundary. The
|
||||||
model. Prompt-definition and profile repositories use the domain model, file
|
registry contains built-ins plus validated engine-scoped consumer additions.
|
||||||
catalog, and YAML decoder. The built-in profile repository supplies an
|
The facade constructs the capacity manager from the registry's immutable
|
||||||
embedded `fs.FS` to the profile package. Artifact reading uses the domain model
|
policy snapshot, wraps the selected built-in or injected model client, and
|
||||||
and application-neutral defaults. Validation uses the domain model, file
|
supplies bounded admission to the runner. The defaults and renderer depend on
|
||||||
|
the domain model. Prompt-definition and profile repositories use the domain
|
||||||
|
model, file catalog, and YAML decoder. The built-in profile repository supplies
|
||||||
|
an embedded `fs.FS` to the profile package. Artifact reading uses the domain
|
||||||
|
model and application-neutral defaults. Validation uses the domain model, file
|
||||||
catalog, and JSON Schema implementation. The model client uses the domain
|
catalog, and JSON Schema implementation. The model client uses the domain
|
||||||
model, application-neutral defaults, and an injected or standard-library HTTP
|
model, application-neutral defaults, and an injected or standard-library HTTP
|
||||||
client. The use-case runner depends on the narrow interfaces owned by each
|
client. The use-case runner depends on the narrow interfaces owned by each
|
||||||
internal component.
|
internal component, including backend lookup and run admission.
|
||||||
|
|
||||||
The current implementation follows this dependency direction:
|
The current implementation follows this dependency direction:
|
||||||
|
|
||||||
@@ -72,9 +82,22 @@ downstream consumers, including Scriptorium
|
|||||||
narrow injected abstractions
|
narrow injected abstractions
|
||||||
```
|
```
|
||||||
|
|
||||||
The facade coordinates internal components and adapts the supported public
|
The backend registry depends on the domain model and shared JSON-value
|
||||||
extension interfaces to narrow internal abstractions. Internal components must
|
validation, has no mutation API after construction, and consumes the
|
||||||
not depend on consumers or on Scriptorium.
|
OpenAI-compatible reserved request-field rule owned by the model client. The
|
||||||
|
capacity component depends on the domain model and the narrow internal
|
||||||
|
model-client boundary, not on provider transport implementation. The model
|
||||||
|
client does not depend on registry or capacity configuration. The facade
|
||||||
|
coordinates internal components and adapts the supported public extension
|
||||||
|
interfaces to narrow internal abstractions. Internal components must not depend
|
||||||
|
on consumers or on Scriptorium.
|
||||||
|
|
||||||
|
`internal/domain` owns source-neutral invariants for values shared across
|
||||||
|
multiple input and execution boundaries, including execution-setting bounds,
|
||||||
|
OpenAI-compatible base endpoints, session identifiers, and output-contract
|
||||||
|
legality. Callers retain source parsing, required-field rules, other
|
||||||
|
source-specific normalization, defaulting, error classification, and policy
|
||||||
|
specific to their own boundary.
|
||||||
|
|
||||||
## Repository And Consumer Boundary
|
## Repository And Consumer Boundary
|
||||||
|
|
||||||
|
|||||||
@@ -78,6 +78,7 @@ mechanisms, not secret values.
|
|||||||
| Framework file formats | `docs/formats.md` | Prompt-definition and profile YAML fields, schema references, defaults, validation modes, built-in profiles, credentials, and file-to-request precedence. | Exported Go declarations, outbound wire behavior, internal parsing mechanics, and application configuration. |
|
| Framework file formats | `docs/formats.md` | Prompt-definition and profile YAML fields, schema references, defaults, validation modes, built-in profiles, credentials, and file-to-request precedence. | Exported Go declarations, outbound wire behavior, internal parsing mechanics, and application configuration. |
|
||||||
| Consumer guidance | `docs/consumers/`, when consumer workflows require dedicated guidance | Task-oriented use of implemented public APIs, minimal examples, and consumer responsibilities. | Exact exported declarations and internal mechanics. |
|
| Consumer guidance | `docs/consumers/`, when consumer workflows require dedicated guidance | Task-oriented use of implemented public APIs, minimal examples, and consumer responsibilities. | Exact exported declarations and internal mechanics. |
|
||||||
| Durable integration contracts | `docs/integrations/`, when integrations exist | External formats and protocols, compatibility behavior, and upstream or downstream responsibilities. | Internal transformations and public Go declarations. |
|
| Durable integration contracts | `docs/integrations/`, when integrations exist | External formats and protocols, compatibility behavior, and upstream or downstream responsibilities. | Internal transformations and public Go declarations. |
|
||||||
|
| Supplemental release guidance | None. `docs/releases/` may be used when a release benefits from a changelog or migration guide. | No canonical content. These files may briefly summarize release-specific changes, compatibility, and consumer migration paths, and may be corrected, consolidated, archived, or removed when no longer useful. | Public API and behavior contracts, formats, integrations, architecture, release procedure, and the authoritative annotated-tag release record. |
|
||||||
| Implemented component inventory | `docs/internal/overview.md` | Current packages and components, their implemented responsibilities, and links to focused internal documents. | Normative architecture, contributor workflow, external contracts, and proposed components. |
|
| Implemented component inventory | `docs/internal/overview.md` | Current packages and components, their implemented responsibilities, and links to focused internal documents. | Normative architecture, contributor workflow, external contracts, and proposed components. |
|
||||||
| Internal subsystem behavior | Other files under `docs/internal/`, when a subsystem needs durable detail | Implementation flow, internal collaborators and state transitions, package-local guarantees and failures, and relevant tests. | Global architecture invariants, public API definitions, and future package plans. |
|
| Internal subsystem behavior | Other files under `docs/internal/`, when a subsystem needs durable detail | Implementation flow, internal collaborators and state transitions, package-local guarantees and failures, and relevant tests. | Global architecture invariants, public API definitions, and future package plans. |
|
||||||
| Architectural decision history | `docs/adr/`, when repository-local decisions require records | Significant decisions, context, alternatives, rationale, consequences, and supersession history. | Current behavior reference, implementation status, and task sequencing. |
|
| Architectural decision history | `docs/adr/`, when repository-local decisions require records | Significant decisions, context, alternatives, rationale, consequences, and supersession history. | Current behavior reference, implementation status, and task sequencing. |
|
||||||
@@ -85,9 +86,9 @@ mechanisms, not secret values.
|
|||||||
| Complete copyable artifacts | `examples/` | Valid inputs, Go programs, and other files intended to be copied or run. | Field-by-field reference, exact API declarations, and prose explanation. |
|
| Complete copyable artifacts | `examples/` | Valid inputs, Go programs, and other files intended to be copied or run. | Field-by-field reference, exact API declarations, and prose explanation. |
|
||||||
|
|
||||||
Conditional owners do not require placeholder files or directories. Create a
|
Conditional owners do not require placeholder files or directories. Create a
|
||||||
consumer, integration, subsystem, ADR, roadmap, or example document only when
|
consumer, integration, release, subsystem, ADR, roadmap, or example document
|
||||||
the corresponding implemented interface, decision, planned effort, or
|
only when the corresponding implemented interface, release, decision, planned
|
||||||
maintained artifact exists.
|
effort, or maintained artifact exists.
|
||||||
|
|
||||||
## Boundary Rules
|
## Boundary Rules
|
||||||
|
|
||||||
@@ -109,6 +110,23 @@ but must link to its canonical definition rather than restate it.
|
|||||||
The [framework format reference](../formats.md) owns exact prompt, profile, and
|
The [framework format reference](../formats.md) owns exact prompt, profile, and
|
||||||
schema-file contracts. Integration documents own external wire formats.
|
schema-file contracts. Integration documents own external wire formats.
|
||||||
|
|
||||||
|
### Supplemental Release Guidance
|
||||||
|
|
||||||
|
Files under `docs/releases/` may provide changelog-style summaries and
|
||||||
|
migration guidance for a particular release. They are navigation and
|
||||||
|
orientation aids, not canonical owners of public APIs, behavior, formats,
|
||||||
|
integrations, architecture, release procedure, or other durable facts. When a
|
||||||
|
reader needs detail beyond a short release-specific note, the release document
|
||||||
|
must link to the applicable canonical documentation rather than reproduce its
|
||||||
|
contract.
|
||||||
|
|
||||||
|
The annotated tag message required by the
|
||||||
|
[release procedure](../release.md#write-the-release-note) remains the
|
||||||
|
authoritative release record. Supplemental release documents may be corrected,
|
||||||
|
consolidated, archived, or removed at any time when they are no longer useful,
|
||||||
|
provided maintained documentation does not depend on them and the annotated
|
||||||
|
tag record remains intact.
|
||||||
|
|
||||||
### Security Topics
|
### Security Topics
|
||||||
|
|
||||||
This policy owns what documentation and examples may contain. Architecture owns
|
This policy owns what documentation and examples may contain. Architecture owns
|
||||||
@@ -160,6 +178,10 @@ durable owners, update incoming links, and archive or remove the roadmap
|
|||||||
according to repository practice. Do not preserve completed roadmaps as a
|
according to repository practice. Do not preserve completed roadmaps as a
|
||||||
second current-state reference.
|
second current-state reference.
|
||||||
|
|
||||||
|
Supplemental release documents may likewise be removed without preserving a
|
||||||
|
replacement. Before removal, update maintained incoming links so current
|
||||||
|
documentation does not depend on an optional historical guide.
|
||||||
|
|
||||||
Before completing documentation work:
|
Before completing documentation work:
|
||||||
|
|
||||||
- verify affected behavior and examples;
|
- verify affected behavior and examples;
|
||||||
|
|||||||
@@ -50,19 +50,18 @@ Examples of appropriate seams include clocks, randomness, subprocesses, remote A
|
|||||||
## Test execution requirements
|
## Test execution requirements
|
||||||
|
|
||||||
Promptkit currently uses maintainer-run validation rather than hosted CI.
|
Promptkit currently uses maintainer-run validation rather than hosted CI.
|
||||||
Maintainers run the repository-documented test, vet, build, formatting,
|
Maintainers run the complete local workflow in the
|
||||||
documentation-link, and repository-hygiene checks before accepting changes.
|
[development guide](../development.md#maintainer-validation) before accepting
|
||||||
|
changes. That guide is the canonical owner of exact commands, formatting,
|
||||||
|
documentation-link validation, and repository-hygiene checks.
|
||||||
Introducing hosted CI later would supplement, not silently redefine, this
|
Introducing hosted CI later would supplement, not silently redefine, this
|
||||||
documented validation model.
|
documented validation model.
|
||||||
|
|
||||||
The complete test sequence includes ordinary and race-enabled package tests.
|
Maintainer validation must include ordinary and race-enabled package tests,
|
||||||
The maintained offline consumer workflow is also run from the repository root:
|
static analysis, a complete build, and execution of both maintained offline
|
||||||
|
consumer examples. The preparation example protects assembled preparation and
|
||||||
```sh
|
inspection behavior. The execution example separately protects assembled
|
||||||
go test ./...
|
`Run`, injected-client, validation, usage, and result behavior.
|
||||||
go test -race ./...
|
|
||||||
go run ./examples/go-library/prepare
|
|
||||||
```
|
|
||||||
|
|
||||||
Tests in the default suite must be deterministic, offline, and independent of
|
Tests in the default suite must be deterministic, offline, and independent of
|
||||||
real credentials. They must not invoke paid APIs, use live network
|
real credentials. They must not invoke paid APIs, use live network
|
||||||
@@ -88,8 +87,9 @@ Use each test type where it protects a distinct risk:
|
|||||||
interaction, while replacing live or nondeterministic external boundaries.
|
interaction, while replacing live or nondeterministic external boundaries.
|
||||||
- External-package root tests exercise the public facade as a Go consumer,
|
- External-package root tests exercise the public facade as a Go consumer,
|
||||||
while internal package tests own focused implementation behavior.
|
while internal package tests own focused implementation behavior.
|
||||||
- The maintained offline preparation example protects one representative
|
- The maintained offline preparation and execution examples protect distinct
|
||||||
assembled consumer workflow without contacting a model provider.
|
representative assembled consumer workflows without contacting a model
|
||||||
|
provider.
|
||||||
- Fixtures should be minimal, synthetic, versioned with the behavior they
|
- Fixtures should be minimal, synthetic, versioned with the behavior they
|
||||||
exercise, and free of credentials or private data.
|
exercise, and free of credentials or private data.
|
||||||
- Golden files are appropriate only when the complete output is intentionally
|
- Golden files are appropriate only when the complete output is intentionally
|
||||||
|
|||||||
248
docs/release.md
248
docs/release.md
@@ -7,27 +7,91 @@ tags. It does not publish runnable binaries or binary packages and does not
|
|||||||
currently use hosted CI. The release maintainer performs and records the
|
currently use hosted CI. The release maintainer performs and records the
|
||||||
required validation.
|
required validation.
|
||||||
|
|
||||||
The first planned release is `v0.1.0`. Do not create that tag until the
|
`v0.1.0` is the initial published release. Later releases use semantic
|
||||||
framework has been extracted and the resulting public library has passed this
|
`vMAJOR.MINOR.PATCH` tags. Before `v1`, minor releases may change the public
|
||||||
procedure. Later tags use the `vMAJOR.MINOR.PATCH` form. While Promptkit remains
|
API and patch releases preserve compatibility within their minor line. Every
|
||||||
pre-`v1`, release notes must identify intentional public API changes and any
|
pre-`v1` release note must summarize compatibility, identify public API
|
||||||
consumer migration required by them.
|
changes, and state any action required of consumers.
|
||||||
|
|
||||||
## Prepare The Release
|
Promptkit releases are source-only. The annotated tag message is the release
|
||||||
|
note; there is no separate hosted release or binary packaging step.
|
||||||
|
|
||||||
Work from a clean checkout of the intended release commit, outside any Go
|
## Establish The Candidate
|
||||||
workspace and without a local module replacement. Confirm the source commit is
|
|
||||||
already published through the normal branch workflow.
|
|
||||||
|
|
||||||
From the Promptkit repository root, verify the checkout:
|
Choose a version that has not been published and export it as
|
||||||
|
`RELEASE_VERSION`. Run every command in this procedure from the Promptkit
|
||||||
|
repository root in the same POSIX shell. Do not reuse `v0.1.0` or another
|
||||||
|
existing version.
|
||||||
|
|
||||||
|
The following guard derives the release commit from `HEAD` and stops on a
|
||||||
|
missing or malformed version, a checkout other than synchronized `main`,
|
||||||
|
uncommitted changes, an active Go workspace, a module replacement, a vendor
|
||||||
|
tree, or an existing local or remote tag:
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
gowork=$(go env GOWORK)
|
set -eu
|
||||||
test -z "$gowork" || test "$gowork" = off
|
|
||||||
test -z "$(git status --short)"
|
: "${RELEASE_VERSION:?export an unpublished vMAJOR.MINOR.PATCH version}"
|
||||||
git fetch --tags origin
|
if ! printf '%s\n' "$RELEASE_VERSION" |
|
||||||
|
grep -Eq '^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$'
|
||||||
|
then
|
||||||
|
printf '%s\n' "invalid release version: $RELEASE_VERSION" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
RELEASE_COMMIT=$(git rev-parse --verify 'HEAD^{commit}')
|
||||||
|
export RELEASE_COMMIT
|
||||||
|
|
||||||
|
check_release_candidate() {
|
||||||
|
test "$(git branch --show-current)" = main
|
||||||
|
test -z "$(git status --porcelain)"
|
||||||
|
|
||||||
|
gowork_value=$(go env GOWORK)
|
||||||
|
case "$gowork_value" in
|
||||||
|
''|off) ;;
|
||||||
|
*)
|
||||||
|
printf '%s\n' "active Go workspace: $gowork_value" >&2
|
||||||
|
return 1
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
test -z "$(git ls-files go.work go.work.sum)"
|
||||||
|
test ! -e vendor
|
||||||
|
if grep -Eq '^[[:space:]]*replace([[:space:]]|\()' go.mod
|
||||||
|
then
|
||||||
|
printf '%s\n' 'go.mod contains a replacement' >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
git fetch origin main --tags
|
||||||
|
test "$RELEASE_COMMIT" = \
|
||||||
|
"$(git rev-parse --verify 'refs/remotes/origin/main^{commit}')"
|
||||||
|
|
||||||
|
if git show-ref --verify --quiet "refs/tags/$RELEASE_VERSION"
|
||||||
|
then
|
||||||
|
printf '%s\n' "local tag already exists: $RELEASE_VERSION" >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
if test -n "$(
|
||||||
|
git ls-remote --tags origin \
|
||||||
|
"refs/tags/$RELEASE_VERSION" \
|
||||||
|
"refs/tags/$RELEASE_VERSION^{}"
|
||||||
|
)"
|
||||||
|
then
|
||||||
|
printf '%s\n' "remote tag already exists: $RELEASE_VERSION" >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
check_release_candidate
|
||||||
```
|
```
|
||||||
|
|
||||||
|
Do not continue unless the guard completes successfully. In particular, push
|
||||||
|
the intended commit through the normal `main` branch workflow before release;
|
||||||
|
the tag procedure is not a substitute for publishing the source commit.
|
||||||
|
|
||||||
|
## Validate The Candidate
|
||||||
|
|
||||||
Confirm the module and root package metadata:
|
Confirm the module and root package metadata:
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
@@ -42,95 +106,133 @@ gitea.maximumdirect.net/eric/promptkit 1.25.5
|
|||||||
promptkit gitea.maximumdirect.net/eric/promptkit
|
promptkit gitea.maximumdirect.net/eric/promptkit
|
||||||
```
|
```
|
||||||
|
|
||||||
Run the same default Go validation required by the
|
As a release prerequisite, run the complete
|
||||||
[development guide](development.md):
|
[maintainer validation workflow](development.md#maintainer-validation) against
|
||||||
|
the clean candidate. Do not substitute a partial command list: the development
|
||||||
|
guide owns the tests, race checks, analysis, build, both offline examples,
|
||||||
|
formatting, Markdown links, generated-output and credential review, and
|
||||||
|
repository hygiene. Record the successful workflow result with the candidate.
|
||||||
|
|
||||||
```sh
|
## Write The Release Note
|
||||||
go test ./...
|
|
||||||
go test -race ./...
|
Prepare a plain-text annotated-tag message outside the repository and export
|
||||||
go vet ./...
|
its path as `RELEASE_NOTES_FILE`. Use this form, replacing each summary with
|
||||||
go build ./...
|
release-specific text; write `None.` when there are no public API changes or
|
||||||
go run ./examples/go-library/prepare
|
consumer actions:
|
||||||
|
|
||||||
|
```text
|
||||||
|
Promptkit vMAJOR.MINOR.PATCH
|
||||||
|
|
||||||
|
Validated commit: full commit ID
|
||||||
|
Compatibility: compatibility summary
|
||||||
|
Public API changes: changes or None.
|
||||||
|
Consumer action: required action or None.
|
||||||
```
|
```
|
||||||
|
|
||||||
Check every tracked Go file and repository whitespace:
|
After writing it, require all release-note fields, the selected version, and
|
||||||
|
the validated commit to be present:
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
gofmt -l $(git ls-files '*.go')
|
: "${RELEASE_NOTES_FILE:?export the path to the release-note file}"
|
||||||
git diff --check
|
test -f "$RELEASE_NOTES_FILE"
|
||||||
|
test -s "$RELEASE_NOTES_FILE"
|
||||||
|
grep -F "Promptkit $RELEASE_VERSION" "$RELEASE_NOTES_FILE"
|
||||||
|
grep -F "Validated commit: $RELEASE_COMMIT" "$RELEASE_NOTES_FILE"
|
||||||
|
grep -F 'Compatibility:' "$RELEASE_NOTES_FILE"
|
||||||
|
grep -F 'Public API changes:' "$RELEASE_NOTES_FILE"
|
||||||
|
grep -F 'Consumer action:' "$RELEASE_NOTES_FILE"
|
||||||
```
|
```
|
||||||
|
|
||||||
The formatting command must produce no paths. Follow every maintained Markdown
|
Inspect the complete message and confirm that it accurately records the
|
||||||
link and confirm its target exists. Review the repository for generated
|
compatibility impact, public API changes, and required consumer action.
|
||||||
binaries, test or coverage output, credentials, template residue, and other
|
|
||||||
files that do not belong in source control.
|
|
||||||
|
|
||||||
Confirm that no workspace override is tracked and that `go.mod` contains no
|
## Create And Inspect The Tag
|
||||||
`replace` directive:
|
|
||||||
|
Run the candidate guard again immediately before tag creation. This ensures
|
||||||
|
that validation or release-note preparation did not change the checkout and
|
||||||
|
that the commit is still published and untagged:
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
git ls-files go.work go.work.sum
|
check_release_candidate
|
||||||
rg -n '^replace\b' go.mod
|
|
||||||
```
|
```
|
||||||
|
|
||||||
Both commands must produce no output. Re-run `git status --short` and require a
|
Create the annotated tag from the prepared release note and bind it explicitly
|
||||||
clean result after every validation and review check.
|
to the validated commit:
|
||||||
|
|
||||||
## Create And Publish The Tag
|
|
||||||
|
|
||||||
Choose the semantic version from the intended compatibility change. Record the
|
|
||||||
release commit before tagging:
|
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
release_version=v0.1.0
|
git tag --annotate "$RELEASE_VERSION" \
|
||||||
release_commit=$(git rev-parse HEAD)
|
--file "$RELEASE_NOTES_FILE" \
|
||||||
|
"$RELEASE_COMMIT"
|
||||||
```
|
```
|
||||||
|
|
||||||
Replace the example version for later releases and keep both values in the same
|
Inspect both the tag message and its source commit before publication:
|
||||||
shell for the remaining commands. Confirm the tag does not already exist
|
|
||||||
locally or remotely:
|
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
test -z "$(git tag --list "$release_version")"
|
test "$(git cat-file -t "refs/tags/$RELEASE_VERSION")" = tag
|
||||||
test -z "$(git ls-remote --tags origin "refs/tags/$release_version")"
|
git show --no-patch --decorate "refs/tags/$RELEASE_VERSION"
|
||||||
|
test "$(
|
||||||
|
git rev-parse --verify "refs/tags/$RELEASE_VERSION^{commit}"
|
||||||
|
)" = "$RELEASE_COMMIT"
|
||||||
```
|
```
|
||||||
|
|
||||||
Create an annotated tag whose message identifies the release and records that
|
If inspection finds an error, delete the unpublished local tag, correct the
|
||||||
the documented validation passed for the tagged commit:
|
release note or candidate, and repeat the guards. Never move or recreate a tag
|
||||||
|
that has been published.
|
||||||
|
|
||||||
|
## Publish The Selected Tag
|
||||||
|
|
||||||
|
Push only the selected tag ref. Do not use `git push --tags`:
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
git tag --annotate "$release_version" \
|
git push origin \
|
||||||
--message "Promptkit $release_version; documented validation passed for $release_commit"
|
"refs/tags/$RELEASE_VERSION:refs/tags/$RELEASE_VERSION"
|
||||||
```
|
|
||||||
|
|
||||||
Inspect the tag before publication:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
git show --no-patch --decorate "$release_version"
|
|
||||||
test "$(git rev-list -n 1 "$release_version")" = "$release_commit"
|
|
||||||
```
|
|
||||||
|
|
||||||
Publish the tag without relying on a hosting-provider-specific release
|
|
||||||
interface:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
git push origin "refs/tags/$release_version"
|
|
||||||
```
|
```
|
||||||
|
|
||||||
## Verify Publication
|
## Verify Publication
|
||||||
|
|
||||||
Confirm that the remote tag object matches the local annotated tag and still
|
Compare the remote annotated-tag object with the local object, then compare the
|
||||||
resolves to the intended source commit:
|
remote peeled source commit with the validated commit:
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
remote_tag=$(git ls-remote --tags origin "refs/tags/$release_version" | awk '{print $1}')
|
remote_tag=$(
|
||||||
test "$remote_tag" = "$(git rev-parse "refs/tags/$release_version")"
|
git ls-remote --tags origin "refs/tags/$RELEASE_VERSION" |
|
||||||
test "$(git rev-list -n 1 "refs/tags/$release_version")" = "$release_commit"
|
awk 'NR == 1 { print $1 }'
|
||||||
|
)
|
||||||
|
remote_commit=$(
|
||||||
|
git ls-remote --tags origin "refs/tags/$RELEASE_VERSION^{}" |
|
||||||
|
awk 'NR == 1 { print $1 }'
|
||||||
|
)
|
||||||
|
test -n "$remote_tag"
|
||||||
|
test "$remote_tag" = \
|
||||||
|
"$(git rev-parse --verify "refs/tags/$RELEASE_VERSION")"
|
||||||
|
test "$remote_commit" = "$RELEASE_COMMIT"
|
||||||
```
|
```
|
||||||
|
|
||||||
Promptkit must publish the required tag before Scriptorium or another consumer
|
Finally, resolve the version as an ordinary Go module in a temporary module
|
||||||
publishes a release that depends on that version. Released consumer modules
|
outside this repository and without a workspace or replacement:
|
||||||
must not use a local replacement or unpublished Promptkit revision.
|
|
||||||
|
```sh
|
||||||
|
resolution_dir=$(mktemp -d)
|
||||||
|
(
|
||||||
|
trap 'rm -rf "$resolution_dir"' 0 1 2 15
|
||||||
|
cd "$resolution_dir"
|
||||||
|
GOWORK=off go mod init example.com/promptkit-release-check
|
||||||
|
GOWORK=off go mod download \
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit@$RELEASE_VERSION"
|
||||||
|
resolved_version=$(
|
||||||
|
GOWORK=off go list -m -f '{{.Version}}' \
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit@$RELEASE_VERSION"
|
||||||
|
)
|
||||||
|
test "$resolved_version" = "$RELEASE_VERSION"
|
||||||
|
)
|
||||||
|
```
|
||||||
|
|
||||||
|
Promptkit must publish and verify the required version before Scriptorium or
|
||||||
|
another consumer publishes a release that depends on it. This ordering does
|
||||||
|
not replace the consumer project's own release procedure. Released consumers
|
||||||
|
must select the published Promptkit tag through ordinary module resolution,
|
||||||
|
without a workspace, replacement, vendored Promptkit source, or unpublished
|
||||||
|
revision.
|
||||||
|
|
||||||
## Policy Changes
|
## Policy Changes
|
||||||
|
|
||||||
|
|||||||
243
docs/releases/v0.2.0.md
Normal file
243
docs/releases/v0.2.0.md
Normal file
@@ -0,0 +1,243 @@
|
|||||||
|
# Promptkit v0.2.0
|
||||||
|
|
||||||
|
This supplemental changelog and migration guide summarizes the consumer-facing
|
||||||
|
changes from `v0.1.0` to `v0.2.0`. The annotated `v0.2.0` tag is the
|
||||||
|
authoritative release record. Exact current contracts belong to the linked
|
||||||
|
GoDoc and durable documentation.
|
||||||
|
|
||||||
|
## Summary
|
||||||
|
|
||||||
|
`v0.2.0` adds three major capabilities:
|
||||||
|
|
||||||
|
- an engine-scoped registry for reusable OpenAI-compatible backend
|
||||||
|
definitions;
|
||||||
|
- bounded, backend-specific run admission and model-generation concurrency;
|
||||||
|
and
|
||||||
|
- direct per-run session IDs and tri-state reasoning-effort overrides.
|
||||||
|
|
||||||
|
Existing endpoint-only profiles remain supported. Consumers can adopt backend
|
||||||
|
registration and runtime overrides incrementally rather than rewriting all
|
||||||
|
profiles during the upgrade.
|
||||||
|
|
||||||
|
## Compatibility At A Glance
|
||||||
|
|
||||||
|
Promptkit remains pre-`v1`, and this minor release includes source-level and
|
||||||
|
behavioral changes that deserve review.
|
||||||
|
|
||||||
|
| Area | `v0.1.0` consumer impact |
|
||||||
|
| --- | --- |
|
||||||
|
| Endpoint-only profiles | Continue to work without migration. |
|
||||||
|
| Built-in profiles | Continue to use OpenRouter and `OPENROUTER_API_KEY`; they now select the built-in `openrouter` backend. |
|
||||||
|
| Custom backends | Registration is optional. Existing profiles may keep their endpoint and credential configuration. |
|
||||||
|
| Reasoning overrides | String assignments must migrate to the new pointer field. |
|
||||||
|
| `RunRequest.Metadata` | Removed; delete assignments to this field. |
|
||||||
|
| OpenRouter concurrency | Now limited to 16 active generations with waiting capacity of 1024 per engine. |
|
||||||
|
| Public JSON | `v0.2.0` formalizes supported JSON representations; consumers relying on `v0.1.0` encodings should review the notes below. |
|
||||||
|
| Unkeyed public struct literals | May require updates because fields were added. Keyed literals are recommended. |
|
||||||
|
|
||||||
|
## Upgrade
|
||||||
|
|
||||||
|
After the `v0.2.0` tag is published, update the module dependency with:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go get gitea.maximumdirect.net/eric/promptkit@v0.2.0
|
||||||
|
go mod tidy
|
||||||
|
```
|
||||||
|
|
||||||
|
Run the consuming project's ordinary tests and race-enabled tests after the
|
||||||
|
upgrade, especially if it calls one engine concurrently or persists Promptkit
|
||||||
|
JSON values.
|
||||||
|
|
||||||
|
## Backend Registry
|
||||||
|
|
||||||
|
Consumers may now register reusable OpenAI-compatible backend definitions with
|
||||||
|
`WithBackend`, then select them by ID from file-backed or in-memory profiles.
|
||||||
|
A backend can supply its endpoint, API-key environment-variable name,
|
||||||
|
request-wide extra parameters, and optional capacity policy.
|
||||||
|
|
||||||
|
Registrations are immutable and belong to one engine. Consumer registrations
|
||||||
|
can add new IDs but cannot replace Promptkit's reserved `openrouter` backend.
|
||||||
|
Profiles that select a backend may still override its endpoint without losing
|
||||||
|
the backend's routing or capacity identity.
|
||||||
|
|
||||||
|
An existing endpoint-only in-memory profile remains valid:
|
||||||
|
|
||||||
|
```go
|
||||||
|
promptkit.Profile{
|
||||||
|
ID: "local",
|
||||||
|
Endpoint: "http://localhost:8000/v1",
|
||||||
|
Model: "example-model",
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Adopting the registry is optional and can be done when several profiles should
|
||||||
|
share connection or capacity settings:
|
||||||
|
|
||||||
|
```go
|
||||||
|
engine, err := promptkit.NewEngine(
|
||||||
|
promptkit.Config{PromptDir: "prompts"},
|
||||||
|
promptkit.WithBackend(promptkit.Backend{
|
||||||
|
ID: "local",
|
||||||
|
Endpoint: "http://localhost:8000/v1",
|
||||||
|
APIKeyEnv: "LOCAL_LLM_API_KEY",
|
||||||
|
}),
|
||||||
|
promptkit.WithProfiles(promptkit.Profile{
|
||||||
|
ID: "local-summary",
|
||||||
|
BackendID: "local",
|
||||||
|
Model: "example-model",
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
```
|
||||||
|
|
||||||
|
See the
|
||||||
|
[local-endpoint consumer guide](../consumers/pkg-promptkit.md#configure-a-local-openai-compatible-endpoint)
|
||||||
|
for task-oriented usage. The
|
||||||
|
[`Backend` and `WithBackend` GoDoc](../../backends.go) owns exact registration,
|
||||||
|
validation, copying, defaulting, and uniqueness semantics. The
|
||||||
|
[framework format reference](../formats.md) owns the profile `backend` field
|
||||||
|
and execution precedence.
|
||||||
|
|
||||||
|
## Backend-Specific Concurrency
|
||||||
|
|
||||||
|
Each registered backend may now define:
|
||||||
|
|
||||||
|
- an active model-generation limit; and
|
||||||
|
- a bounded number of additional admitted `Run` calls.
|
||||||
|
|
||||||
|
Promptkit owns scheduling for both its built-in model client and an injected
|
||||||
|
`LLMClient`. `Run` remains synchronous: an admitted caller waits for its
|
||||||
|
ordinary result, while a call beyond the bounded admission capacity returns
|
||||||
|
`ErrCapacityExceeded`. Capacity is engine-local and keyed by backend ID.
|
||||||
|
Endpoint-only profiles and custom backends without a configured limit remain
|
||||||
|
unlimited.
|
||||||
|
|
||||||
|
The built-in OpenRouter backend now permits 16 active generations and 1024
|
||||||
|
additional admitted calls per engine. Applications that can exceed this bound
|
||||||
|
should handle capacity exhaustion separately from provider and request
|
||||||
|
failures:
|
||||||
|
|
||||||
|
```go
|
||||||
|
result, err := engine.Run(ctx, request)
|
||||||
|
if errors.Is(err, promptkit.ErrCapacityExceeded) {
|
||||||
|
// Apply application-specific overload or retry policy.
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Promptkit does not prescribe retries or map this error to an HTTP status. See
|
||||||
|
the
|
||||||
|
[concurrency consumer guidance](../consumers/pkg-promptkit.md#limit-backend-concurrency)
|
||||||
|
and the [`Backend` GoDoc](../../backends.go) for the canonical configuration
|
||||||
|
contract. Runtime behavior and public error identities belong to the
|
||||||
|
[`Engine.Run` GoDoc](../../engine.go).
|
||||||
|
|
||||||
|
## Per-Run Session IDs
|
||||||
|
|
||||||
|
`RunRequest.SessionID` can now supply a consumer-managed correlation ID for one
|
||||||
|
`Prepare` or `Run` invocation. A nonblank direct value overrides the prompt's
|
||||||
|
session template and is exposed in prepared values, results, injected-client
|
||||||
|
requests, and provider observability. Session IDs should therefore be stable,
|
||||||
|
non-secret values.
|
||||||
|
|
||||||
|
```go
|
||||||
|
result, err := engine.Run(ctx, promptkit.RunRequest{
|
||||||
|
PromptID: "meeting.summary",
|
||||||
|
SessionID: "conversation-42",
|
||||||
|
})
|
||||||
|
```
|
||||||
|
|
||||||
|
The built-in OpenAI-compatible client sends a nonempty effective session as the
|
||||||
|
top-level `session_id` request-body field, not as an `x-session-id` header. See
|
||||||
|
the
|
||||||
|
[session and reasoning consumer guide](../consumers/pkg-promptkit.md#set-a-per-run-session-and-reasoning),
|
||||||
|
the [`RunRequest` GoDoc](../../types.go), and the
|
||||||
|
[OpenAI-compatible request contract](../integrations/openai-compatible-chat.md#request-body)
|
||||||
|
for exact normalization, length, exposure, and wire behavior.
|
||||||
|
|
||||||
|
## Per-Run Reasoning Effort
|
||||||
|
|
||||||
|
`ExecutionTargetOverride.ReasoningEffort` changed from `string` to `*string` so
|
||||||
|
one request can distinguish inheritance, replacement, and explicit clearing.
|
||||||
|
|
||||||
|
Update a `v0.1.0` override like this:
|
||||||
|
|
||||||
|
```go
|
||||||
|
// v0.1.0
|
||||||
|
Execution: &promptkit.ExecutionTargetOverride{
|
||||||
|
ReasoningEffort: "high",
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
to:
|
||||||
|
|
||||||
|
```go
|
||||||
|
// v0.2.0
|
||||||
|
reasoning := "high"
|
||||||
|
Execution: &promptkit.ExecutionTargetOverride{
|
||||||
|
ReasoningEffort: &reasoning,
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
The three states are:
|
||||||
|
|
||||||
|
- `nil` inherits the selected profile's value;
|
||||||
|
- a pointer to a nonblank string replaces it for that invocation; and
|
||||||
|
- a pointer to an empty or whitespace-only string clears it for that
|
||||||
|
invocation.
|
||||||
|
|
||||||
|
This allows consumers to consolidate profiles that differed only by reasoning
|
||||||
|
effort. The [`ExecutionTargetOverride` GoDoc](../../types.go) owns the exact
|
||||||
|
override contract.
|
||||||
|
|
||||||
|
## Other Migration Notes
|
||||||
|
|
||||||
|
### Remove `RunRequest.Metadata`
|
||||||
|
|
||||||
|
`RunRequest.Metadata` is no longer part of the public request. Remove any
|
||||||
|
assignment to that field. Use application-owned state keyed by `RunResult.RunID`
|
||||||
|
or a direct `SessionID` when correlation is needed; these identifiers have
|
||||||
|
different purposes, so choose according to the application's lifecycle.
|
||||||
|
|
||||||
|
### Review Persisted JSON
|
||||||
|
|
||||||
|
`v0.2.0` defines stable JSON representations for the public result, artifact,
|
||||||
|
execution, validation, and model-client values listed in the
|
||||||
|
[package documentation](../../doc.go). Consumers that treated `v0.1.0`
|
||||||
|
reflection-derived encodings as stable should update fixtures and stored-data
|
||||||
|
adapters.
|
||||||
|
|
||||||
|
In particular:
|
||||||
|
|
||||||
|
- `RunResult` encodes elapsed time as integer milliseconds in `duration_ms`
|
||||||
|
instead of encoding `time.Duration` under `duration`;
|
||||||
|
- result JSON can include the new `session_id` and `selected_backend_id`
|
||||||
|
fields;
|
||||||
|
- execution-target JSON can include `backend_id`; and
|
||||||
|
- artifact and target-presence fields now use their documented lower-case
|
||||||
|
names.
|
||||||
|
|
||||||
|
The `v0.2.0` `RunResult` decoder reads `duration_ms`; it does not translate a
|
||||||
|
persisted `v0.1.0` `duration` field. Transform old payloads before decoding
|
||||||
|
when preserving their elapsed duration matters.
|
||||||
|
|
||||||
|
### Prefer Keyed Struct Literals
|
||||||
|
|
||||||
|
New fields were added to several public structs. Replace positional composite
|
||||||
|
literals with keyed literals so future additive fields do not cause another
|
||||||
|
source migration.
|
||||||
|
|
||||||
|
## Migration Checklist
|
||||||
|
|
||||||
|
- Update the module dependency and run the consumer's tests.
|
||||||
|
- Change reasoning overrides from strings to pointers.
|
||||||
|
- Remove uses of `RunRequest.Metadata`.
|
||||||
|
- Review unkeyed Promptkit struct literals.
|
||||||
|
- Decide whether shared endpoints should move into registered backends.
|
||||||
|
- If using built-in OpenRouter profiles at high concurrency, handle
|
||||||
|
`ErrCapacityExceeded` and review the new engine-local bound.
|
||||||
|
- Review stored JSON, fixtures, and downstream decoders.
|
||||||
|
- Optionally replace profile-specific session or reasoning variants with
|
||||||
|
per-run overrides.
|
||||||
|
|
||||||
|
For complete consumer workflows, use the
|
||||||
|
[package consumer guide](../consumers/pkg-promptkit.md) and maintained
|
||||||
|
[offline execution example](../../examples/go-library/run/main.go).
|
||||||
74
docs/releases/v0.3.0.md
Normal file
74
docs/releases/v0.3.0.md
Normal file
@@ -0,0 +1,74 @@
|
|||||||
|
# Promptkit v0.3.0
|
||||||
|
|
||||||
|
This supplemental changelog summarizes the consumer-facing changes from
|
||||||
|
`v0.2.0` to `v0.3.0`. The annotated `v0.3.0` tag is the authoritative release
|
||||||
|
record. Exact current contracts belong to the linked GoDoc and durable
|
||||||
|
documentation.
|
||||||
|
|
||||||
|
## Summary
|
||||||
|
|
||||||
|
`v0.3.0` adds a concise way to register the common local OpenAI-compatible
|
||||||
|
backend configuration:
|
||||||
|
|
||||||
|
- `BackendLocal` provides the conventional, non-reserved backend ID `"local"`;
|
||||||
|
and
|
||||||
|
- `LocalBackend` constructs an ordinary `Backend` from an endpoint and
|
||||||
|
concurrency limit.
|
||||||
|
|
||||||
|
The helper is explicit and additive. It does not pre-register a backend, read
|
||||||
|
environment variables, select a model, or replace the complete `Backend`
|
||||||
|
configuration interface.
|
||||||
|
|
||||||
|
## Compatibility
|
||||||
|
|
||||||
|
Existing `v0.2.0` consumers require no migration. Endpoint-only profiles,
|
||||||
|
complete custom `Backend` values, the built-in OpenRouter backend, and existing
|
||||||
|
registrations using the literal ID `"local"` continue to work unchanged.
|
||||||
|
|
||||||
|
## Upgrade
|
||||||
|
|
||||||
|
Update the module dependency with:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go get gitea.maximumdirect.net/eric/promptkit@v0.3.0
|
||||||
|
go mod tidy
|
||||||
|
```
|
||||||
|
|
||||||
|
Run the consuming project's ordinary tests and race-enabled tests after the
|
||||||
|
upgrade.
|
||||||
|
|
||||||
|
## Configure A Local Backend
|
||||||
|
|
||||||
|
Register the convenience value through the existing `WithBackend` option and
|
||||||
|
select it from one or more profiles:
|
||||||
|
|
||||||
|
```go
|
||||||
|
engine, err := promptkit.NewEngine(
|
||||||
|
promptkit.Config{PromptDir: "prompts"},
|
||||||
|
promptkit.WithBackend(
|
||||||
|
promptkit.LocalBackend("http://localhost:8000/v1", 2),
|
||||||
|
),
|
||||||
|
promptkit.WithProfiles(promptkit.Profile{
|
||||||
|
ID: "local-summary",
|
||||||
|
BackendID: promptkit.BackendLocal,
|
||||||
|
Model: "example-model",
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
```
|
||||||
|
|
||||||
|
Use an endpoint-only profile when shared backend identity and capacity policy
|
||||||
|
are unnecessary. Continue to use a complete keyed `Backend` value for custom
|
||||||
|
IDs, authentication, extra request parameters, explicit queue capacity, or
|
||||||
|
multiple local endpoints.
|
||||||
|
|
||||||
|
See the
|
||||||
|
[local-endpoint consumer guide](../consumers/pkg-promptkit.md#configure-a-local-openai-compatible-endpoint)
|
||||||
|
for task-oriented configuration choices. The
|
||||||
|
[`BackendLocal`, `LocalBackend`, and `WithBackend` GoDoc](../../backends.go)
|
||||||
|
owns their exact construction, registration, validation, and concurrency
|
||||||
|
semantics.
|
||||||
|
|
||||||
|
## Consumer Action
|
||||||
|
|
||||||
|
None. Adopt the convenience constructor when it simplifies local endpoint
|
||||||
|
configuration.
|
||||||
189
docs/releases/v0.4.0.md
Normal file
189
docs/releases/v0.4.0.md
Normal file
@@ -0,0 +1,189 @@
|
|||||||
|
# Promptkit v0.4.0
|
||||||
|
|
||||||
|
This supplemental changelog and adoption guide summarizes the consumer-facing
|
||||||
|
changes from `v0.3.0` to `v0.4.0`. The annotated `v0.4.0` tag is the
|
||||||
|
authoritative release record. Exact current contracts belong to the linked
|
||||||
|
GoDoc and durable documentation.
|
||||||
|
|
||||||
|
## Summary
|
||||||
|
|
||||||
|
`v0.4.0` adds four complementary capabilities:
|
||||||
|
|
||||||
|
- opaque prepared-execution handles for preparing once, inspecting safe
|
||||||
|
details, and executing the same frozen snapshot;
|
||||||
|
- exact prompt-definition inspection without profile resolution or execution;
|
||||||
|
- exact profile inspection without selecting a prompt or checking credential
|
||||||
|
availability; and
|
||||||
|
- structured backend identity on engine admission-capacity rejection.
|
||||||
|
|
||||||
|
These APIs let consumers perform more precise preflight work and retain useful
|
||||||
|
operational context without reproducing Promptkit's internal resolution logic.
|
||||||
|
|
||||||
|
## Compatibility
|
||||||
|
|
||||||
|
The release is additive for `v0.3.0` consumers. Existing uses of `Prepare`,
|
||||||
|
`Run`, backend registration, endpoint-only profiles, local-backend helpers,
|
||||||
|
runtime overrides, public JSON values, and error sentinels continue to work
|
||||||
|
without migration.
|
||||||
|
|
||||||
|
Capacity rejection now returns a structured error while continuing to match
|
||||||
|
`ErrCapacityExceeded` through `errors.Is`. Error-string wording and direct
|
||||||
|
sentinel equality were not public contracts.
|
||||||
|
|
||||||
|
The new inspection values, capacity error, and prepared-execution handle do not
|
||||||
|
have stable JSON representations. `PreparedExecution.Details` returns the
|
||||||
|
existing stable `PreparedRun` value.
|
||||||
|
|
||||||
|
## Upgrade
|
||||||
|
|
||||||
|
Update the module dependency with:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go get gitea.maximumdirect.net/eric/promptkit@v0.4.0
|
||||||
|
go mod tidy
|
||||||
|
```
|
||||||
|
|
||||||
|
Run the consuming project's ordinary and race-enabled tests after upgrading.
|
||||||
|
No source migration is required.
|
||||||
|
|
||||||
|
## Prepare Once And Execute The Same Snapshot
|
||||||
|
|
||||||
|
Consumers that need to persist preparation details before generation can now
|
||||||
|
prepare an opaque, engine-bound execution:
|
||||||
|
|
||||||
|
```go
|
||||||
|
prepared, err := engine.PrepareExecution(ctx, request)
|
||||||
|
if err != nil {
|
||||||
|
// Handle preparation failure.
|
||||||
|
}
|
||||||
|
defer prepared.Discard()
|
||||||
|
|
||||||
|
details := prepared.Details()
|
||||||
|
// Persist a consumer-selected, appropriately protected preparation record.
|
||||||
|
|
||||||
|
result, err := engine.RunPrepared(ctx, prepared)
|
||||||
|
```
|
||||||
|
|
||||||
|
Preparation freezes the selected sources, rendered messages, effective
|
||||||
|
settings, input content, structured-output metadata, and validation resources
|
||||||
|
needed by execution. `Details` returns a fresh, caller-owned,
|
||||||
|
credential-redacted `PreparedRun`.
|
||||||
|
|
||||||
|
A handle belongs to its creating engine and permits one execution attempt.
|
||||||
|
`RunPrepared` consumes that attempt on success and on operational failure.
|
||||||
|
`Discard` is idempotent and releases an unclaimed handle's execution-only
|
||||||
|
state. Consumers should discard handles they will not execute, particularly
|
||||||
|
when a direct request API key may be retained privately until claim or
|
||||||
|
discard.
|
||||||
|
|
||||||
|
Prepared execution does not reserve backend admission during preparation.
|
||||||
|
Credential availability and backend admission are checked when execution
|
||||||
|
begins. The execution context is independent of the preparation context.
|
||||||
|
|
||||||
|
See the
|
||||||
|
[prepared-execution consumer guide](../consumers/pkg-promptkit.md#prepare-now-and-execute-the-same-snapshot-later),
|
||||||
|
the [`PreparedExecution` GoDoc](../../prepared_execution.go), and the
|
||||||
|
[`Engine.PrepareExecution` and `Engine.RunPrepared` GoDoc](../../engine.go)
|
||||||
|
for the exact lifecycle, ownership, cancellation, capacity, timing, and
|
||||||
|
failure contracts.
|
||||||
|
|
||||||
|
## Inspect A Prompt
|
||||||
|
|
||||||
|
`Engine.InspectPrompt` resolves one prompt ID and optional version through the
|
||||||
|
engine's configured prompt source:
|
||||||
|
|
||||||
|
```go
|
||||||
|
inspection, err := engine.InspectPrompt(ctx, "report.summary", "")
|
||||||
|
```
|
||||||
|
|
||||||
|
The result includes prompt identity, the opaque prompt hash, declared default
|
||||||
|
profile ID, declared input metadata, and normalized output contract. It
|
||||||
|
structurally loads the selected definition and referenced message content but
|
||||||
|
does not resolve a profile, load schemas or artifacts, render templates,
|
||||||
|
reserve capacity, or contact a model.
|
||||||
|
|
||||||
|
Use inspection for exact configuration checks and metadata discovery. Use
|
||||||
|
`PrepareExecution` rather than relying on a prior inspection when later
|
||||||
|
execution must freeze one exact source state, because filesystem-backed
|
||||||
|
inspection is only a point-in-time lookup.
|
||||||
|
|
||||||
|
See the
|
||||||
|
[prompt-inspection consumer guide](../consumers/pkg-promptkit.md#inspect-a-prompt-before-preparation)
|
||||||
|
and [`Engine.InspectPrompt` GoDoc](../../engine.go) for exact selection,
|
||||||
|
ownership, and error behavior.
|
||||||
|
|
||||||
|
## Inspect A Profile
|
||||||
|
|
||||||
|
`Engine.InspectProfile` resolves one explicit profile independently of a
|
||||||
|
prompt:
|
||||||
|
|
||||||
|
```go
|
||||||
|
inspection, err := engine.InspectProfile(ctx, "report-production")
|
||||||
|
```
|
||||||
|
|
||||||
|
The result includes the resolved effective execution target and whether a
|
||||||
|
later request must provide a direct credential. Environment-variable names may
|
||||||
|
be reported, but inspection does not read credential values or require the
|
||||||
|
named variable to be populated.
|
||||||
|
|
||||||
|
Inspection applies the engine's profile source precedence and resolves any
|
||||||
|
selected backend. It does not load a prompt, render content, reserve capacity,
|
||||||
|
or contact a model.
|
||||||
|
|
||||||
|
See the
|
||||||
|
[profile-inspection consumer guide](../consumers/pkg-promptkit.md#inspect-a-profile-before-prompt-work)
|
||||||
|
and [`Engine.InspectProfile` GoDoc](../../engine.go) for the exact resolution,
|
||||||
|
credential, ownership, and error contracts.
|
||||||
|
|
||||||
|
## Identify Capacity-Rejected Backends
|
||||||
|
|
||||||
|
Calls rejected at Promptkit's bounded engine admission boundary continue to
|
||||||
|
match `ErrCapacityExceeded`. Consumers can additionally obtain the selected
|
||||||
|
registered backend ID without parsing diagnostic text:
|
||||||
|
|
||||||
|
```go
|
||||||
|
result, err := engine.Run(ctx, request)
|
||||||
|
if errors.Is(err, promptkit.ErrCapacityExceeded) {
|
||||||
|
var capacityErr *promptkit.CapacityError
|
||||||
|
if errors.As(err, &capacityErr) {
|
||||||
|
// Record capacityErr.BackendID using application-owned diagnostics.
|
||||||
|
}
|
||||||
|
|
||||||
|
// Apply application-owned overload or retry policy.
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
The structured error applies to `Run` and `RunPrepared` admission rejection.
|
||||||
|
It does not represent provider throttling, quota exhaustion, cancellation
|
||||||
|
while waiting for generation capacity, or another model-client failure.
|
||||||
|
Promptkit does not prescribe retry timing or transport status mapping.
|
||||||
|
|
||||||
|
See the
|
||||||
|
[error-handling consumer guide](../consumers/pkg-promptkit.md#handle-errors),
|
||||||
|
the [`CapacityError` GoDoc](../../capacity_error.go), and the
|
||||||
|
[`ErrCapacityExceeded` GoDoc](../../engine.go) for the canonical contracts.
|
||||||
|
|
||||||
|
## Public API Additions
|
||||||
|
|
||||||
|
The release adds:
|
||||||
|
|
||||||
|
- `Engine.PrepareExecution`;
|
||||||
|
- `Engine.RunPrepared`;
|
||||||
|
- `PreparedExecution`, including `Details`, `Discard`, `String`, and
|
||||||
|
`GoString`;
|
||||||
|
- `Engine.InspectPrompt`;
|
||||||
|
- `PromptInspection`;
|
||||||
|
- `PromptInputDefinition`;
|
||||||
|
- `Engine.InspectProfile`;
|
||||||
|
- `ProfileInspection`; and
|
||||||
|
- `CapacityError`.
|
||||||
|
|
||||||
|
No public API was removed.
|
||||||
|
|
||||||
|
## Consumer Action
|
||||||
|
|
||||||
|
None. Existing `v0.3.0` workflows may upgrade without adopting the new APIs.
|
||||||
|
|
||||||
|
Consumers that adopt prepared execution should discard unused handles.
|
||||||
|
Consumers that need backend-specific capacity diagnostics may add an
|
||||||
|
`errors.As` check while retaining their existing `errors.Is` classification.
|
||||||
125
docs/releases/v0.5.0.md
Normal file
125
docs/releases/v0.5.0.md
Normal file
@@ -0,0 +1,125 @@
|
|||||||
|
# Promptkit v0.5.0
|
||||||
|
|
||||||
|
This supplemental changelog and migration guide summarizes the consumer-facing
|
||||||
|
changes from `v0.4.0` to `v0.5.0`. The annotated `v0.5.0` tag is the
|
||||||
|
authoritative release record. Exact current contracts belong to the linked
|
||||||
|
GoDoc and durable documentation.
|
||||||
|
|
||||||
|
## Summary
|
||||||
|
|
||||||
|
`v0.5.0` makes provider requests less prescriptive and adds an application
|
||||||
|
fallback layer for profile definitions:
|
||||||
|
|
||||||
|
- unset optional provider controls are omitted from OpenAI-compatible request
|
||||||
|
bodies instead of being populated with framework values; and
|
||||||
|
- `WithFallbackProfileFS` lets an application package profile defaults that
|
||||||
|
operators can override through the existing ordinary profile sources.
|
||||||
|
|
||||||
|
These changes let compatible providers apply their own model defaults while
|
||||||
|
giving applications stable embedded profile IDs without weakening operator
|
||||||
|
configuration precedence.
|
||||||
|
|
||||||
|
## Compatibility
|
||||||
|
|
||||||
|
The release adds one public function and removes no public declaration.
|
||||||
|
Existing source code should continue to compile.
|
||||||
|
|
||||||
|
There is one intentional behavior change: when no profile or runtime override
|
||||||
|
selects `top_p`, Promptkit no longer sends the former framework value of `1`.
|
||||||
|
It omits `top_p` and lets the provider choose its behavior. Unset
|
||||||
|
`temperature` and `max_tokens` are likewise omitted. Explicit nonzero profile
|
||||||
|
values and runtime values—including explicit runtime zero values—retain their
|
||||||
|
precedence and wire effect.
|
||||||
|
|
||||||
|
Consumers that relied on Promptkit always sending `top_p: 1` should add that
|
||||||
|
value to the relevant profile or runtime override before upgrading. Consumers
|
||||||
|
that did not rely on the implicit sampling value require no migration.
|
||||||
|
|
||||||
|
Application fallback profiles are opt-in. Engines that do not call
|
||||||
|
`WithFallbackProfileFS` retain the previous profile-source behavior.
|
||||||
|
|
||||||
|
## Upgrade
|
||||||
|
|
||||||
|
Update the module dependency with:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go get gitea.maximumdirect.net/eric/promptkit@v0.5.0
|
||||||
|
go mod tidy
|
||||||
|
```
|
||||||
|
|
||||||
|
Run the consuming project's ordinary and race-enabled tests after upgrading.
|
||||||
|
If request payloads or model behavior are asserted in fixtures, review them for
|
||||||
|
the optional-parameter omission described below.
|
||||||
|
|
||||||
|
## Omitted Optional Provider Controls
|
||||||
|
|
||||||
|
The built-in OpenAI-compatible client now includes `temperature`,
|
||||||
|
`max_tokens`, and `top_p` only when a profile or runtime override selects the
|
||||||
|
value. An explicit runtime zero remains present because runtime override
|
||||||
|
pointers distinguish zero from an unspecified value.
|
||||||
|
|
||||||
|
Promptkit's positive generation deadline remains a framework concern and is
|
||||||
|
not a provider request-body default. Required request fields, session IDs,
|
||||||
|
structured output, reasoning selection, credentials, and explicit extra
|
||||||
|
parameters retain their existing behavior.
|
||||||
|
|
||||||
|
See the [framework default and precedence reference](../formats.md#defaults-and-overrides),
|
||||||
|
the [`ExecutionTargetOverride` GoDoc](../../types.go), and the
|
||||||
|
[OpenAI-compatible request-body contract](../integrations/openai-compatible-chat.md#request-body)
|
||||||
|
for current details.
|
||||||
|
|
||||||
|
## Embedded Application Fallback Profiles
|
||||||
|
|
||||||
|
Applications can package ordinary profile YAML in an `fs.FS` and register it
|
||||||
|
as a fallback source:
|
||||||
|
|
||||||
|
```go
|
||||||
|
//go:embed profiles/*.yaml
|
||||||
|
var applicationProfiles embed.FS
|
||||||
|
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{
|
||||||
|
PromptDir: "prompts",
|
||||||
|
ProfileDir: operatorProfileDir,
|
||||||
|
},
|
||||||
|
promptkit.WithFallbackProfileFS(applicationProfiles, "profiles"),
|
||||||
|
)
|
||||||
|
```
|
||||||
|
|
||||||
|
Leave `operatorProfileDir` empty when no operator source is configured. A
|
||||||
|
configured ordinary source is authoritative: a matching definition overrides
|
||||||
|
the application fallback, while a read or validation failure remains an error
|
||||||
|
instead of silently reaching a lower layer.
|
||||||
|
|
||||||
|
Profile definitions resolve in this order:
|
||||||
|
|
||||||
|
1. in-memory profiles supplied with `WithProfiles`;
|
||||||
|
2. the ordinary configured source selected by `WithProfileFile`,
|
||||||
|
`WithProfileFS`, or `Config.ProfileDir`;
|
||||||
|
3. the application source supplied with `WithFallbackProfileFS`; and
|
||||||
|
4. Promptkit's embedded built-in profiles.
|
||||||
|
|
||||||
|
Only an absent profile ID falls through. Sources provide complete profiles and
|
||||||
|
do not merge fields. Loading remains lazy, and the new source uses the existing
|
||||||
|
strict profile YAML and credential rules.
|
||||||
|
|
||||||
|
See the
|
||||||
|
[embedded-default consumer guidance](../consumers/pkg-promptkit.md#supply-embedded-application-defaults),
|
||||||
|
the [`WithFallbackProfileFS` GoDoc](../../engine.go), and the
|
||||||
|
[profile source reference](../formats.md#source-and-profile-precedence) for
|
||||||
|
current details.
|
||||||
|
|
||||||
|
## Public API Changes
|
||||||
|
|
||||||
|
The release adds:
|
||||||
|
|
||||||
|
- `WithFallbackProfileFS`.
|
||||||
|
|
||||||
|
No public declaration was removed or changed.
|
||||||
|
|
||||||
|
## Consumer Action
|
||||||
|
|
||||||
|
- Review any workflow that depended on Promptkit's implicit `top_p: 1` and
|
||||||
|
configure the value explicitly when required.
|
||||||
|
- Optionally adopt `WithFallbackProfileFS` when an application should package
|
||||||
|
overridable profile defaults.
|
||||||
|
- Run consumer tests after updating the module dependency.
|
||||||
131
docs/releases/v0.6.0.md
Normal file
131
docs/releases/v0.6.0.md
Normal file
@@ -0,0 +1,131 @@
|
|||||||
|
# Promptkit v0.6.0
|
||||||
|
|
||||||
|
This supplemental changelog and migration guide summarizes the consumer-facing
|
||||||
|
changes from `v0.5.0` to `v0.6.0`. The annotated `v0.6.0` tag is the
|
||||||
|
authoritative release record. Exact current contracts belong to the linked
|
||||||
|
GoDoc and durable documentation.
|
||||||
|
|
||||||
|
## Summary
|
||||||
|
|
||||||
|
`v0.6.0` is a broad correctness, safety, efficiency, and maintainability
|
||||||
|
release. It does not add or remove public declarations. The release:
|
||||||
|
|
||||||
|
- centralizes shared execution-setting, output-contract, endpoint, and
|
||||||
|
JSON-compatible-value rules;
|
||||||
|
- unifies prompt repository behavior and avoids unnecessary prompt and profile
|
||||||
|
decoding;
|
||||||
|
- bounds consumer-controlled JSON trees and successful provider responses;
|
||||||
|
- hardens prompt content paths, artifact files, provider URLs, JSON framing,
|
||||||
|
and error propagation;
|
||||||
|
- reuses compiled schema plans and rendered artifact text within an operation;
|
||||||
|
and
|
||||||
|
- improves cancellation behavior, prepared-value ownership, deterministic
|
||||||
|
transport testing, and maintainer validation.
|
||||||
|
|
||||||
|
## Compatibility
|
||||||
|
|
||||||
|
No public declaration was added, removed, or changed. Ordinary valid `v0.5.0`
|
||||||
|
configurations and requests should continue to compile and behave as before.
|
||||||
|
|
||||||
|
The release intentionally rejects or reports several inputs that were
|
||||||
|
previously accepted, altered, or misclassified:
|
||||||
|
|
||||||
|
- execution settings must be finite, within their documented ranges, and safe
|
||||||
|
to convert to Go durations;
|
||||||
|
- output formats, validation modes, repair counts, and JSON Schema dependencies
|
||||||
|
are validated consistently;
|
||||||
|
- file-backed prompt and profile identity comes from normalized YAML metadata,
|
||||||
|
not filenames;
|
||||||
|
- prompt `content_file` values must be exact relative paths contained by their
|
||||||
|
configured source root;
|
||||||
|
- built-in file artifacts must resolve to regular files;
|
||||||
|
- selected provider endpoints must be absolute HTTP or HTTPS URLs without user
|
||||||
|
information, query strings, or fragments;
|
||||||
|
- JSON documents and successful provider responses must contain exactly one
|
||||||
|
value, and successful provider bodies are limited to 16 MiB; and
|
||||||
|
- excessively deep or expansive JSON-compatible values fail with ordinary
|
||||||
|
validation errors.
|
||||||
|
|
||||||
|
These are compatibility corrections and safety boundaries rather than new
|
||||||
|
consumer configuration requirements. Consumers relying on an invalid or
|
||||||
|
ambiguous input should correct that input before upgrading.
|
||||||
|
|
||||||
|
## Upgrade
|
||||||
|
|
||||||
|
Update the module dependency with:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go get gitea.maximumdirect.net/eric/promptkit@v0.6.0
|
||||||
|
go mod tidy
|
||||||
|
```
|
||||||
|
|
||||||
|
Run the consuming project's ordinary and race-enabled tests after upgrading.
|
||||||
|
Applications with custom prompt/profile sources, local provider endpoints,
|
||||||
|
unusual artifact paths, or assertions over provider error identities should
|
||||||
|
pay particular attention to the compatibility notes below.
|
||||||
|
|
||||||
|
## Source Loading And Identity
|
||||||
|
|
||||||
|
Prompt definitions now share one source-neutral selection and normalization
|
||||||
|
flow across operating-system and `fs.FS` sources. YAML `id` and `version`
|
||||||
|
metadata are authoritative; filenames do not create a second identity system.
|
||||||
|
Only selected content bodies are loaded, malformed unrelated definitions do
|
||||||
|
not shadow valid exact matches, and per-file read failures are reported as
|
||||||
|
prompt-load failures rather than false absence.
|
||||||
|
|
||||||
|
File-backed profiles likewise use normalized YAML IDs, reuse their metadata
|
||||||
|
read for selected strict decoding, and avoid fully decoding unrelated files.
|
||||||
|
Selected malformed definitions remain authoritative and do not silently fall
|
||||||
|
through to a lower-precedence source.
|
||||||
|
|
||||||
|
Prompt `content_file` paths are opened exactly as declared after a separate
|
||||||
|
blank check. They must remain relative to and contained by the configured
|
||||||
|
prompt source root, including across operating-system symlinks.
|
||||||
|
|
||||||
|
See the [framework source and identity reference](../formats.md) and
|
||||||
|
[internal source overview](../internal/sources.md) for the current contracts.
|
||||||
|
|
||||||
|
## Validation, Cancellation, And Efficiency
|
||||||
|
|
||||||
|
JSON Schema documents preserve exact JSON-number representations. Schema
|
||||||
|
resource URLs safely escape legal filesystem names, and each operation loads
|
||||||
|
and compiles its schema graph once. `Run` and prepared execution reuse that
|
||||||
|
operation-local plan; Promptkit does not introduce a cross-operation cache.
|
||||||
|
|
||||||
|
Artifact reading, rendering, schema loading, compilation, and validation now
|
||||||
|
check cancellation at the synchronous boundaries Promptkit controls. Rendering
|
||||||
|
memoizes each artifact's text within one render operation, while plain JSON
|
||||||
|
validation avoids materializing an unnecessary generic tree.
|
||||||
|
|
||||||
|
The shared JSON-compatible-value owner now limits nesting and produced work so
|
||||||
|
unsafe consumer-controlled structures return errors instead of risking
|
||||||
|
unbounded recursion or allocation. See the
|
||||||
|
[architecture policy](../policy/architecture.md) for invariant ownership and
|
||||||
|
the [format reference](../formats.md) for validation behavior.
|
||||||
|
|
||||||
|
## Provider Transport Hardening
|
||||||
|
|
||||||
|
OpenAI-compatible endpoints are parsed and composed structurally, including
|
||||||
|
nested base paths. Underlying transport cancellation and deadline errors remain
|
||||||
|
discoverable with `errors.Is` through Promptkit's generation error category.
|
||||||
|
|
||||||
|
Successful provider bodies are read with a fixed 16 MiB bound and must contain
|
||||||
|
exactly one JSON response object followed only by whitespace. Oversized,
|
||||||
|
truncated, malformed, or multiply framed responses fail without returning a
|
||||||
|
partial result. See the
|
||||||
|
[OpenAI-compatible integration contract](../integrations/openai-compatible-chat.md)
|
||||||
|
for the canonical request, endpoint, error, and response behavior.
|
||||||
|
|
||||||
|
## Public API Changes
|
||||||
|
|
||||||
|
None.
|
||||||
|
|
||||||
|
## Consumer Action
|
||||||
|
|
||||||
|
- Correct any configuration or request that depends on the formerly permissive
|
||||||
|
cases described under Compatibility.
|
||||||
|
- Confirm custom local provider endpoints are absolute HTTP or HTTPS base URLs
|
||||||
|
without credentials, queries, or fragments.
|
||||||
|
- Confirm prompt content paths remain within their configured source root and
|
||||||
|
file artifacts resolve to regular files.
|
||||||
|
- Run ordinary and race-enabled consumer tests after updating the dependency.
|
||||||
61
docs/roadmap/deferred.md
Normal file
61
docs/roadmap/deferred.md
Normal file
@@ -0,0 +1,61 @@
|
|||||||
|
# Deferred Feature Ideas
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
|
||||||
|
This document catalogs feature ideas that remain potentially useful but have
|
||||||
|
been deliberately postponed. These ideas are not awaiting ordinary selection
|
||||||
|
from the [future feature catalog](future.md); each has a stated reason to wait
|
||||||
|
and should be reconsidered only when its trigger becomes relevant.
|
||||||
|
|
||||||
|
Deferred entries are not commitments, schedules, active implementation plans,
|
||||||
|
or descriptions of current behavior. When an entry is reactivated, move it to
|
||||||
|
`future.md` for evaluation or directly into a focused roadmap after its open
|
||||||
|
design dependencies have been resolved.
|
||||||
|
|
||||||
|
## Deferred Ideas
|
||||||
|
|
||||||
|
### Semantic Execution-Target Fingerprints
|
||||||
|
|
||||||
|
**Reason for deferral:** A stable digest requires a deliberate semantic-
|
||||||
|
equality and versioning design. Notarius can safely use conservative source
|
||||||
|
hashes and a Promptkit release marker today, while Weatherreporter does not
|
||||||
|
currently reuse LLM-dependent checkpoints.
|
||||||
|
|
||||||
|
Promptkit could expose an opaque equality value for a resolved profile and its
|
||||||
|
effective generation target. This would let checkpointing consumers detect
|
||||||
|
generation-affecting configuration changes without hashing YAML presentation
|
||||||
|
or depending on Promptkit's built-in catalog layout.
|
||||||
|
|
||||||
|
The digest should change with semantically relevant state such as the resolved
|
||||||
|
model, endpoint, backend routing identity, request defaults, extra parameters,
|
||||||
|
profile generation settings, and selected built-in profile semantics. It
|
||||||
|
should exclude credential values, concurrency and queue policy, source paths,
|
||||||
|
comments, formatting, and other representation-only changes. Whether a
|
||||||
|
credential environment-variable name affects equality must be decided
|
||||||
|
explicitly. The encoding should remain opaque and internally versioned so
|
||||||
|
Promptkit can deliberately invalidate earlier digests when its resolution
|
||||||
|
semantics change.
|
||||||
|
|
||||||
|
Reconsider this idea when a downstream consumer needs Promptkit-owned
|
||||||
|
checkpoint equality or when a broader semantic identity design is selected.
|
||||||
|
|
||||||
|
### Eager Source Validation
|
||||||
|
|
||||||
|
**Reason for deferral:** Exact prompt and profile inspection may already
|
||||||
|
provide a sufficiently small validation surface. Experience from downstream
|
||||||
|
adoption should establish whether an engine-wide operation would add enough
|
||||||
|
value to justify its broader contract.
|
||||||
|
|
||||||
|
Promptkit could provide an explicit offline operation that discovers and
|
||||||
|
structurally validates configured prompt, profile, and schema sources without
|
||||||
|
model generation. The normal `NewEngine` path would remain lazy.
|
||||||
|
|
||||||
|
An eager operation would need coherent handling for duplicate prompt IDs and
|
||||||
|
versions, strict YAML decoding, referenced content files, profile/backend
|
||||||
|
membership, schema syntax and transitive references, context cancellation,
|
||||||
|
and source-specific public errors. Credential declarations must remain
|
||||||
|
separate from credential values; checking current environment availability,
|
||||||
|
if supported at all, should be an explicit option and must not expose secrets.
|
||||||
|
|
||||||
|
Reconsider this idea after downstream use of `InspectPrompt`,
|
||||||
|
`InspectProfile`, and fixture-based preparation demonstrates a concrete gap.
|
||||||
82
docs/roadmap/future.md
Normal file
82
docs/roadmap/future.md
Normal file
@@ -0,0 +1,82 @@
|
|||||||
|
# Future Feature Ideas
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
|
||||||
|
This document catalogs reasonably specific ideas that may be useful in future
|
||||||
|
Promptkit development. It is an idea pool, not a commitment, schedule, or
|
||||||
|
description of current behavior.
|
||||||
|
|
||||||
|
Ideas belong here while they are worth retaining but have not been selected
|
||||||
|
for active development. Keep each entry at the level of intended capability,
|
||||||
|
consumer value, and important scope boundaries. Defer API design,
|
||||||
|
implementation details, sequencing, and acceptance criteria until an idea is
|
||||||
|
selected.
|
||||||
|
|
||||||
|
Ideas that have been deliberately postponed rather than left available for
|
||||||
|
ordinary selection belong in the [deferred catalog](deferred.md).
|
||||||
|
|
||||||
|
## Using This Catalog
|
||||||
|
|
||||||
|
- Add an idea when its purpose and likely value can be stated clearly.
|
||||||
|
- Keep entries independent enough that maintainers can evaluate and select
|
||||||
|
them individually.
|
||||||
|
- Note significant dependencies or boundary concerns, but do not turn entries
|
||||||
|
into implementation plans.
|
||||||
|
- Treat inclusion as an invitation to evaluate, not as approval or priority.
|
||||||
|
- When an idea is selected, move its active planning to a focused roadmap or,
|
||||||
|
when it requires a durable architectural decision, an ADR. Update
|
||||||
|
current-state documentation only when implementation lands.
|
||||||
|
- Move an idea to `deferred.md` when maintainers decide to retain it but wait
|
||||||
|
for a stated design dependency, demand signal, or reconsideration trigger.
|
||||||
|
- Remove ideas that are no longer relevant. Retain a rejected idea only when
|
||||||
|
its rationale is likely to prevent repeated reconsideration.
|
||||||
|
|
||||||
|
Future capabilities must continue to respect the
|
||||||
|
[architecture policy](../policy/architecture.md), particularly Promptkit's
|
||||||
|
role as an application-neutral library and its boundary with downstream
|
||||||
|
consumers.
|
||||||
|
|
||||||
|
## Ideas
|
||||||
|
|
||||||
|
### Public bounded output repair
|
||||||
|
|
||||||
|
After the codebase-audit remediations are complete, Promptkit should make its
|
||||||
|
bounded output-repair capability available through the public engine. A
|
||||||
|
consumer should be able to request a limited number of corrective generation
|
||||||
|
attempts when JSON or JSON Schema output fails content validation, without
|
||||||
|
having to reproduce Promptkit's generation, validation, capacity, and result-
|
||||||
|
accounting orchestration.
|
||||||
|
|
||||||
|
- Repair is validation recovery, not a general provider retry, failover, or
|
||||||
|
backoff policy. Transport failures, cancellation, and operational schema or
|
||||||
|
validation errors must retain their ordinary error behavior.
|
||||||
|
- Repair must stop after the first valid result or the configured attempt
|
||||||
|
bound. Exhausting the bound should preserve the final invalid result and its
|
||||||
|
validation diagnostics rather than inventing success.
|
||||||
|
- Initial generation and every repair attempt must use the same resolved
|
||||||
|
backend, effective execution settings and presence semantics, session,
|
||||||
|
credential boundary, structured-output contract, and backend-capacity
|
||||||
|
policy.
|
||||||
|
- Results should report the number of repair attempts and cumulative usage for
|
||||||
|
every model call made by the run.
|
||||||
|
- Ordinary and prepared execution should expose coherent behavior, including
|
||||||
|
cancellation, frozen prepared state, error identity, and capacity lifetime.
|
||||||
|
|
||||||
|
Select this work only after the accepted audit findings affecting shared
|
||||||
|
execution invariants, validation, orchestration, transport, and repair
|
||||||
|
internals have been remediated.
|
||||||
|
|
||||||
|
## Entry Format
|
||||||
|
|
||||||
|
Use a short heading followed by a concise summary. Add focused bullets when
|
||||||
|
they help preserve important scope boundaries without becoming an
|
||||||
|
implementation plan:
|
||||||
|
|
||||||
|
```markdown
|
||||||
|
### Idea name
|
||||||
|
|
||||||
|
Describe the intended capability, who benefits, and the most important scope
|
||||||
|
boundary or dependency.
|
||||||
|
|
||||||
|
- Optionally record an important behavior or boundary.
|
||||||
|
```
|
||||||
71
docs/roadmap/structured-generation-errors.md
Normal file
71
docs/roadmap/structured-generation-errors.md
Normal file
@@ -0,0 +1,71 @@
|
|||||||
|
# Structured Generation Errors
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
|
||||||
|
Promptkit should give downstream applications actionable, machine-readable
|
||||||
|
details when the built-in OpenAI-compatible client receives a non-success HTTP
|
||||||
|
response. Today the client reports only the status code and discards the
|
||||||
|
provider response body. This makes ordinary configuration failures—such as an
|
||||||
|
unsupported strict JSON Schema keyword—unnecessarily difficult to diagnose.
|
||||||
|
|
||||||
|
## Target End State
|
||||||
|
|
||||||
|
Failures from the built-in transport are available through a public typed error
|
||||||
|
that works with `errors.As` while continuing to match `ErrLLMGenerate` through
|
||||||
|
`errors.Is`. The error should expose:
|
||||||
|
|
||||||
|
- the HTTP status code;
|
||||||
|
- a normalized provider error code or type when supplied; and
|
||||||
|
- a bounded provider message extracted from a recognized OpenAI-compatible
|
||||||
|
JSON error envelope.
|
||||||
|
|
||||||
|
The ordinary `Error()` string should remain safe and concise: it should include
|
||||||
|
the status and provider code or type, but not automatically include the
|
||||||
|
provider message. Consumers that deliberately want the provider's diagnostic
|
||||||
|
text can retrieve it from the typed error and apply their own disclosure and
|
||||||
|
logging policy.
|
||||||
|
|
||||||
|
This contract should be available for both ordinary and prepared execution.
|
||||||
|
Errors returned by injected model clients must continue to preserve their own
|
||||||
|
identity and should not be converted into fabricated HTTP details.
|
||||||
|
|
||||||
|
## Safety And Compatibility Boundaries
|
||||||
|
|
||||||
|
- Never expose the raw response body, response headers, endpoint, credentials,
|
||||||
|
request messages, schema document, or generated content through this API.
|
||||||
|
- Read only a small fixed maximum response body, reject malformed or
|
||||||
|
unrecognized envelopes, normalize invalid UTF-8 and control characters, and
|
||||||
|
cap every retained diagnostic field independently.
|
||||||
|
- Treat the extracted provider message as untrusted and potentially sensitive:
|
||||||
|
its GoDoc must tell consumers not to log or display it without applying their
|
||||||
|
own policy.
|
||||||
|
- Preserve the existing generic behavior when a response is empty, non-JSON,
|
||||||
|
oversized, or does not match a recognized error envelope.
|
||||||
|
- Do not assign retryability from an HTTP status. Promptkit supplies facts;
|
||||||
|
downstream applications retain retry and presentation policy.
|
||||||
|
|
||||||
|
## Recommended API Direction
|
||||||
|
|
||||||
|
Prefer one immutable public `GenerationError` value, constructed internally and
|
||||||
|
carrying accessors for HTTP status, provider code or type, and provider message.
|
||||||
|
This keeps the exact representation evolvable while giving consumers an
|
||||||
|
idiomatic `errors.As` contract. Public Go declarations and GoDoc should own the
|
||||||
|
final exact names and semantics.
|
||||||
|
|
||||||
|
The internal OpenAI-compatible client should parse only the conventional
|
||||||
|
top-level `error` envelope and pass normalized details through the use-case and
|
||||||
|
public error-mapping layers. The integration documentation should continue to
|
||||||
|
own wire behavior; the public declarations should own the consumer contract.
|
||||||
|
|
||||||
|
## Acceptance Criteria
|
||||||
|
|
||||||
|
- A downstream consumer can distinguish a provider HTTP 400 from other
|
||||||
|
generation failures and obtain a bounded provider explanation when present.
|
||||||
|
- The typed error still satisfies `errors.Is(err, ErrLLMGenerate)`.
|
||||||
|
- Existing cancellation, capacity, validation, and injected-client error
|
||||||
|
identities remain unchanged.
|
||||||
|
- Tests cover recognized string and numeric provider codes, absent and malformed
|
||||||
|
envelopes, oversized bodies and fields, control characters, and error-chain
|
||||||
|
behavior without making live provider requests.
|
||||||
|
- Current-state GoDoc and the OpenAI-compatible integration and internal-client
|
||||||
|
documents are updated only when the implementation lands.
|
||||||
474
engine.go
474
engine.go
@@ -12,7 +12,10 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
artifactadapter "gitea.maximumdirect.net/eric/promptkit/internal/artifact"
|
artifactadapter "gitea.maximumdirect.net/eric/promptkit/internal/artifact"
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/backend"
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/capacity"
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/defaults"
|
"gitea.maximumdirect.net/eric/promptkit/internal/defaults"
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/llm"
|
"gitea.maximumdirect.net/eric/promptkit/internal/llm"
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/profile"
|
"gitea.maximumdirect.net/eric/promptkit/internal/profile"
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/profile/builtin"
|
"gitea.maximumdirect.net/eric/promptkit/internal/profile/builtin"
|
||||||
@@ -22,42 +25,107 @@ import (
|
|||||||
"gitea.maximumdirect.net/eric/promptkit/internal/validate"
|
"gitea.maximumdirect.net/eric/promptkit/internal/validate"
|
||||||
)
|
)
|
||||||
|
|
||||||
// ErrInvalidConfig indicates invalid public engine configuration.
|
// ErrInvalidConfig identifies invalid engine construction, including missing
|
||||||
|
// required configuration, invalid options or backend registrations, and a nil
|
||||||
|
// Engine receiver.
|
||||||
var ErrInvalidConfig = errors.New("invalid engine configuration")
|
var ErrInvalidConfig = errors.New("invalid engine configuration")
|
||||||
|
|
||||||
var (
|
var (
|
||||||
ErrInvalidRequest = errors.New("invalid run request")
|
// ErrInvalidRequest identifies a request whose required values, overrides,
|
||||||
ErrPromptNotFound = errors.New("prompt not found")
|
// credentials, or effective settings are invalid.
|
||||||
ErrProfileNotFound = errors.New("profile not found")
|
ErrInvalidRequest = errors.New("invalid run request")
|
||||||
ErrProfileRequired = errors.New("profile selection is required")
|
// ErrPromptNotFound identifies a requested prompt ID or version that is not
|
||||||
ErrPromptLoad = errors.New("failed to load prompt definition")
|
// present in the selected prompt source. It does not also match
|
||||||
ErrProfileLoad = errors.New("failed to load execution profile")
|
// ErrPromptLoad.
|
||||||
|
ErrPromptNotFound = errors.New("prompt not found")
|
||||||
|
// ErrProfileNotFound identifies a selected profile ID that is absent from
|
||||||
|
// every configured profile source. It does not also match ErrProfileLoad.
|
||||||
|
ErrProfileNotFound = errors.New("profile not found")
|
||||||
|
// ErrProfileRequired identifies a request for which neither RunRequest.ProfileID
|
||||||
|
// nor the selected prompt's default profile is present. Such an error also
|
||||||
|
// matches ErrInvalidRequest.
|
||||||
|
ErrProfileRequired = errors.New("profile selection is required")
|
||||||
|
// ErrPromptLoad identifies a failure to read, decode, validate, select, or
|
||||||
|
// hash a prompt definition, except for the not-found case represented by
|
||||||
|
// ErrPromptNotFound.
|
||||||
|
ErrPromptLoad = errors.New("failed to load prompt definition")
|
||||||
|
// ErrProfileLoad identifies a failure to read, decode, validate, or select
|
||||||
|
// an execution profile or resolve its backend, except for the profile
|
||||||
|
// not-found case represented by ErrProfileNotFound.
|
||||||
|
ErrProfileLoad = errors.New("failed to load execution profile")
|
||||||
|
// ErrAPIKeyEnvMissing identifies an APIKeyEnv whose environment variable is
|
||||||
|
// unset or empty when no direct RunRequest.APIKey takes precedence. Such an
|
||||||
|
// error also matches ErrInvalidRequest.
|
||||||
ErrAPIKeyEnvMissing = errors.New("api_key_env points to an unset environment variable")
|
ErrAPIKeyEnvMissing = errors.New("api_key_env points to an unset environment variable")
|
||||||
ErrArtifactLoad = errors.New("failed to load artifact")
|
// ErrArtifactLoad identifies a failure to resolve an input artifact. Errors
|
||||||
ErrPromptRender = errors.New("failed to render prompt")
|
// returned by an injected ArtifactReader remain available through errors.Is.
|
||||||
ErrLLMGenerate = errors.New("failed to generate output")
|
ErrArtifactLoad = errors.New("failed to load artifact")
|
||||||
ErrValidation = errors.New("failed to validate output")
|
// ErrPromptRender identifies a failure to render prompt messages or the
|
||||||
|
// session ID from the resolved inputs and variables.
|
||||||
|
ErrPromptRender = errors.New("failed to render prompt")
|
||||||
|
// ErrCapacityExceeded identifies a Run or RunPrepared rejected because the
|
||||||
|
// selected backend already admitted ConcurrencyLimit + QueueCapacity calls.
|
||||||
|
// A [CapacityError] reports the selected backend ID. It is not an invalid
|
||||||
|
// request, an LLM or provider rate-limit response, or ErrLLMGenerate.
|
||||||
|
ErrCapacityExceeded = errors.New("backend capacity exceeded")
|
||||||
|
// ErrLLMGenerate identifies a model-client failure or a nil successful
|
||||||
|
// response. Errors returned by an injected LLMClient remain available
|
||||||
|
// through errors.Is.
|
||||||
|
ErrLLMGenerate = errors.New("failed to generate output")
|
||||||
|
// ErrValidation identifies an operational failure to load or compile a
|
||||||
|
// schema or validate output. A completed validation whose Status is
|
||||||
|
// ValidationFailed is returned in RunResult without this error.
|
||||||
|
ErrValidation = errors.New("failed to validate output")
|
||||||
)
|
)
|
||||||
|
|
||||||
// Engine prepares and runs Promptkit prompt requests.
|
// Engine inspects prompts and profiles and prepares and runs Promptkit prompt
|
||||||
|
// requests.
|
||||||
|
//
|
||||||
|
// An Engine is safe for concurrent calls to [Engine.InspectPrompt],
|
||||||
|
// [Engine.InspectProfile], [Engine.Prepare], [Engine.PrepareExecution],
|
||||||
|
// [Engine.Run], and [Engine.RunPrepared]. Each Engine owns independent
|
||||||
|
// backend-capacity pools that coordinate Run and RunPrepared admission and
|
||||||
|
// model generation. Injected collaborators may still be invoked concurrently
|
||||||
|
// across different backend pools or for unlimited backends.
|
||||||
type Engine struct {
|
type Engine struct {
|
||||||
runner *usecase.Runner
|
runner *usecase.Runner
|
||||||
}
|
}
|
||||||
|
|
||||||
// Config configures a public Promptkit engine.
|
// Config selects the directory-backed sources and built-in model-client
|
||||||
|
// transport used by [NewEngine]. Config has no stable JSON representation.
|
||||||
type Config struct {
|
type Config struct {
|
||||||
PromptDir string
|
// PromptDir is the directory searched recursively for prompt definitions.
|
||||||
|
// It is required unless a WithPromptFS or WithPromptFile option supplies the
|
||||||
|
// prompt source.
|
||||||
|
PromptDir string
|
||||||
|
// ProfileDir is an optional ordinary configured source whose profiles take
|
||||||
|
// precedence over application fallback and embedded built-in profiles. An
|
||||||
|
// empty value selects the lower-precedence sources unless a profile-source
|
||||||
|
// option supplies the ordinary source.
|
||||||
ProfileDir string
|
ProfileDir string
|
||||||
SchemaDir string
|
// SchemaDir is the root for JSON Schema files. An empty value uses the
|
||||||
|
// current directory. WithSchemaFS or WithSchemaFile replaces this source.
|
||||||
|
SchemaDir string
|
||||||
// Timeout is the transport-wide safety cap for the built-in LLM client
|
// Timeout is the transport-wide safety cap for the built-in LLM client
|
||||||
// when HTTPClient is absent or has a non-positive timeout.
|
// when HTTPClient is absent or has a non-positive timeout. A zero or negative
|
||||||
|
// value selects the 10-minute default.
|
||||||
Timeout time.Duration
|
Timeout time.Duration
|
||||||
// HTTPClient is cloned for the built-in LLM client. Its positive Timeout
|
// HTTPClient is cloned for the built-in LLM client. Its positive Timeout
|
||||||
// takes precedence over Config.Timeout as the transport-wide safety cap.
|
// takes precedence over Timeout. A zero or negative client Timeout inherits
|
||||||
|
// Timeout or the 10-minute default. The supplied client is not mutated. This
|
||||||
|
// field is ignored when WithLLMClient is used.
|
||||||
HTTPClient *http.Client
|
HTTPClient *http.Client
|
||||||
}
|
}
|
||||||
|
|
||||||
// Option customizes engine construction.
|
// Option customizes engine construction.
|
||||||
|
//
|
||||||
|
// NewEngine applies options in argument order and ignores nil options. Within
|
||||||
|
// each prompt-source, ordinary-profile-source, fallback-profile-source,
|
||||||
|
// in-memory-profile, schema-source, model-client, and artifact-reader
|
||||||
|
// category, the last non-nil valid option replaces earlier options in that
|
||||||
|
// category. WithBackend is the additive exception: unique registrations
|
||||||
|
// accumulate, and a repeated backend ID is an error rather than a replacement.
|
||||||
|
// An invalid option fails construction even if a later option would replace it.
|
||||||
type Option interface {
|
type Option interface {
|
||||||
apply(*engineOptions) error
|
apply(*engineOptions) error
|
||||||
}
|
}
|
||||||
@@ -69,20 +137,30 @@ func (f optionFunc) apply(options *engineOptions) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type engineOptions struct {
|
type engineOptions struct {
|
||||||
llmClient llm.Client
|
llmClient llm.Client
|
||||||
artifactReader artifactadapter.Reader
|
artifactReader artifactadapter.Reader
|
||||||
promptDefs promptdef.Repository
|
promptDefs promptdef.Repository
|
||||||
profiles profile.Repository
|
profiles profile.Repository
|
||||||
memoryProfiles profile.Repository
|
fallbackProfiles profile.Repository
|
||||||
validator validate.Validator
|
memoryProfiles profile.Repository
|
||||||
promptSource bool
|
backends []domain.Backend
|
||||||
profileSource bool
|
validator validate.Validator
|
||||||
memorySource bool
|
promptSource bool
|
||||||
validatorSource bool
|
profileSource bool
|
||||||
artifactSource bool
|
fallbackProfileSource bool
|
||||||
|
memorySource bool
|
||||||
|
validatorSource bool
|
||||||
|
artifactSource bool
|
||||||
}
|
}
|
||||||
|
|
||||||
// WithLLMClient injects a custom LLM client for execution.
|
// WithLLMClient replaces the built-in model client used by [Engine.Run] and
|
||||||
|
// [Engine.RunPrepared].
|
||||||
|
//
|
||||||
|
// A nil client makes NewEngine fail with ErrInvalidConfig. The Engine schedules
|
||||||
|
// Generate calls according to the selected backend's capacity policy, but the
|
||||||
|
// client may still be called concurrently across different backend pools or for
|
||||||
|
// unlimited backends. The client is not used by [Engine.Prepare] or
|
||||||
|
// [Engine.PrepareExecution].
|
||||||
func WithLLMClient(client LLMClient) Option {
|
func WithLLMClient(client LLMClient) Option {
|
||||||
return optionFunc(func(options *engineOptions) error {
|
return optionFunc(func(options *engineOptions) error {
|
||||||
if client == nil {
|
if client == nil {
|
||||||
@@ -93,7 +171,11 @@ func WithLLMClient(client LLMClient) Option {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
// WithArtifactReader injects a reader for every input artifact reference.
|
// WithArtifactReader replaces the default reader for every input artifact
|
||||||
|
// reference, regardless of its ArtifactRef.Type.
|
||||||
|
//
|
||||||
|
// A nil reader makes NewEngine fail with ErrInvalidConfig. The reader may be
|
||||||
|
// called concurrently.
|
||||||
func WithArtifactReader(reader ArtifactReader) Option {
|
func WithArtifactReader(reader ArtifactReader) Option {
|
||||||
return optionFunc(func(options *engineOptions) error {
|
return optionFunc(func(options *engineOptions) error {
|
||||||
if reader == nil {
|
if reader == nil {
|
||||||
@@ -109,6 +191,9 @@ func WithArtifactReader(reader ArtifactReader) Option {
|
|||||||
//
|
//
|
||||||
// The source uses the same strict prompt YAML rules as configured prompt
|
// The source uses the same strict prompt YAML rules as configured prompt
|
||||||
// directories, and prompt content_file paths resolve within this source.
|
// directories, and prompt content_file paths resolve within this source.
|
||||||
|
// fsys must be non-nil and root must be non-empty; otherwise NewEngine fails
|
||||||
|
// with ErrInvalidConfig. This option replaces Config.PromptDir and earlier
|
||||||
|
// prompt-source options.
|
||||||
func WithPromptFS(fsys fs.FS, root string) Option {
|
func WithPromptFS(fsys fs.FS, root string) Option {
|
||||||
return optionFunc(func(options *engineOptions) error {
|
return optionFunc(func(options *engineOptions) error {
|
||||||
if fsys == nil {
|
if fsys == nil {
|
||||||
@@ -125,14 +210,16 @@ func WithPromptFS(fsys fs.FS, root string) Option {
|
|||||||
|
|
||||||
// WithPromptFile loads prompt definitions from the single prompt file at path.
|
// WithPromptFile loads prompt definitions from the single prompt file at path.
|
||||||
//
|
//
|
||||||
// Relative prompt content_file paths resolve from the file's directory.
|
// Relative prompt content_file paths resolve from the file's directory. path
|
||||||
|
// must name an existing non-directory file when NewEngine applies the option.
|
||||||
|
// This option replaces Config.PromptDir and earlier prompt-source options.
|
||||||
func WithPromptFile(path string) Option {
|
func WithPromptFile(path string) Option {
|
||||||
return optionFunc(func(options *engineOptions) error {
|
return optionFunc(func(options *engineOptions) error {
|
||||||
fsys, root, err := fileSource(path)
|
fsys, root, err := fileSource(path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
options.promptDefs = promptdef.NewFSRepository(fsys, root)
|
options.promptDefs = promptdef.NewFileRepository(fsys, root, filepath.Dir(path))
|
||||||
options.promptSource = true
|
options.promptSource = true
|
||||||
return nil
|
return nil
|
||||||
})
|
})
|
||||||
@@ -140,8 +227,12 @@ func WithPromptFile(path string) Option {
|
|||||||
|
|
||||||
// WithProfileFS loads execution profiles from fsys under root.
|
// WithProfileFS loads execution profiles from fsys under root.
|
||||||
//
|
//
|
||||||
// Profiles from this source overlay built-in profiles. Profile YAML must use
|
// Profiles from this ordinary configured source take precedence over
|
||||||
// api_key_env for environment-based credentials; raw API keys are rejected.
|
// application fallback and built-in profiles. Profile YAML must use api_key_env
|
||||||
|
// for environment-based credentials; raw API keys are rejected. fsys must be
|
||||||
|
// non-nil and root must be non-empty; otherwise NewEngine fails with
|
||||||
|
// ErrInvalidConfig. This option replaces Config.ProfileDir and earlier file or
|
||||||
|
// FS profile-source options, but remains below WithProfiles in precedence.
|
||||||
func WithProfileFS(fsys fs.FS, root string) Option {
|
func WithProfileFS(fsys fs.FS, root string) Option {
|
||||||
return optionFunc(func(options *engineOptions) error {
|
return optionFunc(func(options *engineOptions) error {
|
||||||
if fsys == nil {
|
if fsys == nil {
|
||||||
@@ -158,8 +249,12 @@ func WithProfileFS(fsys fs.FS, root string) Option {
|
|||||||
|
|
||||||
// WithProfileFile loads execution profiles from the single profile file at path.
|
// WithProfileFile loads execution profiles from the single profile file at path.
|
||||||
//
|
//
|
||||||
// The profile overlays built-in profiles. Profile YAML must use api_key_env for
|
// The profile takes precedence over application fallback and built-in profiles.
|
||||||
// environment-based credentials; raw API keys are rejected.
|
// Profile YAML must use api_key_env for environment-based credentials; raw API
|
||||||
|
// keys are rejected. path must name an existing non-directory file when
|
||||||
|
// NewEngine applies the option. This option replaces Config.ProfileDir and
|
||||||
|
// earlier file or FS profile-source options, but remains below WithProfiles in
|
||||||
|
// precedence.
|
||||||
func WithProfileFile(path string) Option {
|
func WithProfileFile(path string) Option {
|
||||||
return optionFunc(func(options *engineOptions) error {
|
return optionFunc(func(options *engineOptions) error {
|
||||||
fsys, root, err := fileSource(path)
|
fsys, root, err := fileSource(path)
|
||||||
@@ -172,8 +267,46 @@ func WithProfileFile(path string) Option {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// WithFallbackProfileFS supplies application-owned fallback profile
|
||||||
|
// definitions from fsys under root.
|
||||||
|
//
|
||||||
|
// Profile lookup checks, in order, profiles supplied by WithProfiles; the
|
||||||
|
// ordinary configured source selected by WithProfileFile, WithProfileFS, or
|
||||||
|
// Config.ProfileDir; this fallback source; and Promptkit's embedded built-in
|
||||||
|
// profiles. Each source supplies a complete profile definition; profile fields
|
||||||
|
// are not merged between sources. Only an absent profile ID proceeds to the
|
||||||
|
// next source. A matching read, parse, duplicate, validation, or credential
|
||||||
|
// format failure stops resolution.
|
||||||
|
//
|
||||||
|
// Files use the ordinary strict profile YAML and api_key_env credential rules.
|
||||||
|
// Loading and validation are lazy: NewEngine validates this option's arguments
|
||||||
|
// but does not read profile files. fsys must be non-nil and root must be
|
||||||
|
// nonblank; otherwise NewEngine returns an error matching ErrInvalidConfig.
|
||||||
|
// Repeating this option replaces the earlier valid fallback source.
|
||||||
|
//
|
||||||
|
// This option controls profile-definition lookup, not provider or generation
|
||||||
|
// failover.
|
||||||
|
func WithFallbackProfileFS(fsys fs.FS, root string) Option {
|
||||||
|
return optionFunc(func(options *engineOptions) error {
|
||||||
|
if fsys == nil {
|
||||||
|
return ErrInvalidConfig
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(root) == "" {
|
||||||
|
return ErrInvalidConfig
|
||||||
|
}
|
||||||
|
options.fallbackProfiles = profile.NewFSRepository(fsys, root)
|
||||||
|
options.fallbackProfileSource = true
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
// WithProfiles configures in-memory profiles that take precedence over
|
// WithProfiles configures in-memory profiles that take precedence over
|
||||||
// configured profile files and built-in profiles.
|
// ordinary configured, application fallback, and built-in profiles.
|
||||||
|
//
|
||||||
|
// NewEngine validates and copies every profile. IDs must be unique within one
|
||||||
|
// call. An invalid profile, duplicate ID, or unsupported ExtraParams value
|
||||||
|
// makes construction fail with ErrInvalidConfig. Repeating WithProfiles
|
||||||
|
// replaces the complete earlier in-memory set rather than merging it.
|
||||||
func WithProfiles(profiles ...Profile) Option {
|
func WithProfiles(profiles ...Profile) Option {
|
||||||
return optionFunc(func(options *engineOptions) error {
|
return optionFunc(func(options *engineOptions) error {
|
||||||
repo, err := newMemoryProfileRepository(profiles)
|
repo, err := newMemoryProfileRepository(profiles)
|
||||||
@@ -189,7 +322,9 @@ func WithProfiles(profiles ...Profile) Option {
|
|||||||
// WithSchemaFS loads JSON Schema documents from fsys under root.
|
// WithSchemaFS loads JSON Schema documents from fsys under root.
|
||||||
//
|
//
|
||||||
// Prompt schema_path values resolve within this source when schema validation
|
// Prompt schema_path values resolve within this source when schema validation
|
||||||
// or structured output is requested.
|
// or structured output is requested. fsys must be non-nil and root must be
|
||||||
|
// non-empty; otherwise NewEngine fails with ErrInvalidConfig. This option
|
||||||
|
// replaces Config.SchemaDir and earlier schema-source options.
|
||||||
func WithSchemaFS(fsys fs.FS, root string) Option {
|
func WithSchemaFS(fsys fs.FS, root string) Option {
|
||||||
return optionFunc(func(options *engineOptions) error {
|
return optionFunc(func(options *engineOptions) error {
|
||||||
if fsys == nil {
|
if fsys == nil {
|
||||||
@@ -206,7 +341,9 @@ func WithSchemaFS(fsys fs.FS, root string) Option {
|
|||||||
|
|
||||||
// WithSchemaFile loads JSON Schema documents from the single schema file at path.
|
// WithSchemaFile loads JSON Schema documents from the single schema file at path.
|
||||||
//
|
//
|
||||||
// Prompt schema_path values refer to the file's base name.
|
// Prompt schema_path values refer to the file's base name. path must name an
|
||||||
|
// existing non-directory file when NewEngine applies the option. This option
|
||||||
|
// replaces Config.SchemaDir and earlier schema-source options.
|
||||||
func WithSchemaFile(path string) Option {
|
func WithSchemaFile(path string) Option {
|
||||||
return optionFunc(func(options *engineOptions) error {
|
return optionFunc(func(options *engineOptions) error {
|
||||||
fsys, root, err := fileSource(path)
|
fsys, root, err := fileSource(path)
|
||||||
@@ -220,6 +357,17 @@ func WithSchemaFile(path string) Option {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// NewEngine constructs an Engine from configuration and options.
|
// NewEngine constructs an Engine from configuration and options.
|
||||||
|
//
|
||||||
|
// Options are applied in order according to [Option]. PromptDir is required
|
||||||
|
// unless a prompt-source option is present. Construction validates option
|
||||||
|
// arguments, in-memory profiles, and backend registrations but defers reading
|
||||||
|
// and validating prompt, file-backed profile, and schema contents until Prepare
|
||||||
|
// or Run needs them.
|
||||||
|
//
|
||||||
|
// NewEngine returns an error matching ErrInvalidConfig for invalid
|
||||||
|
// configuration, options, or backend-capacity policies. Each constructed
|
||||||
|
// Engine has independent backend-capacity pools. Construction does not perform
|
||||||
|
// model requests or require credentials.
|
||||||
func NewEngine(cfg Config, opts ...Option) (*Engine, error) {
|
func NewEngine(cfg Config, opts ...Option) (*Engine, error) {
|
||||||
var options engineOptions
|
var options engineOptions
|
||||||
for _, opt := range opts {
|
for _, opt := range opts {
|
||||||
@@ -239,12 +387,16 @@ func NewEngine(cfg Config, opts ...Option) (*Engine, error) {
|
|||||||
promptDefs = promptdef.NewFilesystemRepository(cfg.PromptDir)
|
promptDefs = promptdef.NewFilesystemRepository(cfg.PromptDir)
|
||||||
}
|
}
|
||||||
|
|
||||||
profiles := builtin.NewRepositoryWithDirectory(cfg.ProfileDir)
|
profiles := newProfileRepository(cfg.ProfileDir, options)
|
||||||
if options.profileSource {
|
|
||||||
profiles = builtin.NewRepositoryWithPrimary(options.profiles)
|
backendRegistry, err := backend.NewRegistry(options.backends)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("%w: failed to construct backend registry: %v", ErrInvalidConfig, err)
|
||||||
}
|
}
|
||||||
if options.memorySource {
|
|
||||||
profiles = profile.NewOverlayRepository(options.memoryProfiles, profiles)
|
capacityManager, err := capacity.NewManager(backendRegistry.CapacityPolicies())
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("%w: failed to construct backend capacity manager: %v", ErrInvalidConfig, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
validator := options.validator
|
validator := options.validator
|
||||||
@@ -267,6 +419,7 @@ func NewEngine(cfg Config, opts ...Option) (*Engine, error) {
|
|||||||
return nil, fmt.Errorf("%w: %v", ErrInvalidConfig, err)
|
return nil, fmt.Errorf("%w: %v", ErrInvalidConfig, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
llmClient = capacity.NewClient(capacityManager, llmClient)
|
||||||
|
|
||||||
artifacts := options.artifactReader
|
artifacts := options.artifactReader
|
||||||
if !options.artifactSource {
|
if !options.artifactSource {
|
||||||
@@ -277,35 +430,159 @@ func NewEngine(cfg Config, opts ...Option) (*Engine, error) {
|
|||||||
runner: usecase.NewRunner(
|
runner: usecase.NewRunner(
|
||||||
promptDefs,
|
promptDefs,
|
||||||
profiles,
|
profiles,
|
||||||
|
backendRegistry,
|
||||||
artifacts,
|
artifacts,
|
||||||
prompt.NewGoRenderer(),
|
prompt.NewGoRenderer(),
|
||||||
llmClient,
|
llmClient,
|
||||||
validator,
|
validator,
|
||||||
|
capacityManager,
|
||||||
),
|
),
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func newProfileRepository(profileDir string, options engineOptions) profile.Repository {
|
||||||
|
repository := builtin.NewRepository()
|
||||||
|
|
||||||
|
if options.fallbackProfileSource {
|
||||||
|
repository = profile.NewOverlayRepository(options.fallbackProfiles, repository)
|
||||||
|
}
|
||||||
|
|
||||||
|
if options.profileSource {
|
||||||
|
repository = profile.NewOverlayRepository(options.profiles, repository)
|
||||||
|
} else if strings.TrimSpace(profileDir) != "" {
|
||||||
|
repository = profile.NewOverlayRepository(profile.NewFilesystemRepository(profileDir), repository)
|
||||||
|
}
|
||||||
|
|
||||||
|
if options.memorySource {
|
||||||
|
repository = profile.NewOverlayRepository(options.memoryProfiles, repository)
|
||||||
|
}
|
||||||
|
|
||||||
|
return repository
|
||||||
|
}
|
||||||
|
|
||||||
func fileSource(name string) (fs.FS, string, error) {
|
func fileSource(name string) (fs.FS, string, error) {
|
||||||
cleanName := strings.TrimSpace(name)
|
if strings.TrimSpace(name) == "" {
|
||||||
if cleanName == "" {
|
|
||||||
return nil, "", ErrInvalidConfig
|
return nil, "", ErrInvalidConfig
|
||||||
}
|
}
|
||||||
dir := filepath.Dir(cleanName)
|
dir := filepath.Dir(name)
|
||||||
base := filepath.Base(cleanName)
|
base := filepath.Base(name)
|
||||||
if base == "." || base == string(filepath.Separator) || strings.TrimSpace(base) == "" {
|
if base == "." || base == string(filepath.Separator) {
|
||||||
return nil, "", ErrInvalidConfig
|
return nil, "", ErrInvalidConfig
|
||||||
}
|
}
|
||||||
info, err := os.Stat(cleanName)
|
info, err := os.Stat(name)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, "", fmt.Errorf("%w: failed to access source file %q: %v", ErrInvalidConfig, cleanName, err)
|
return nil, "", fmt.Errorf("%w: failed to access source file %q: %v", ErrInvalidConfig, name, err)
|
||||||
}
|
}
|
||||||
if info.IsDir() {
|
if info.IsDir() {
|
||||||
return nil, "", fmt.Errorf("%w: source path %q must be a file", ErrInvalidConfig, cleanName)
|
return nil, "", fmt.Errorf("%w: source path %q must be a file", ErrInvalidConfig, name)
|
||||||
}
|
}
|
||||||
return os.DirFS(dir), filepath.ToSlash(base), nil
|
return os.DirFS(dir), filepath.ToSlash(base), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Prepare resolves a prompt request without calling an LLM.
|
// InspectPrompt resolves one explicit prompt definition without selecting a
|
||||||
|
// profile or starting execution work.
|
||||||
|
//
|
||||||
|
// InspectPrompt requires a nonblank promptID. It passes nonblank promptID and
|
||||||
|
// promptVersion values unchanged to the engine's ordinary, case-sensitive
|
||||||
|
// prompt selection. An empty version succeeds only when that source has one
|
||||||
|
// selected ID; a nonempty version selects one exact ID/version pair. The
|
||||||
|
// configured prompt source is used without merging, fallback, or enumeration.
|
||||||
|
//
|
||||||
|
// A successful result proves that the selected definition and any referenced
|
||||||
|
// message content files were structurally loaded. Inputs are returned in
|
||||||
|
// definition order. DefaultProfileID is declared metadata only and is not
|
||||||
|
// resolved. OutputContract is the normalized declared contract, with a JSON
|
||||||
|
// Schema path when declared but without loading or compiling that schema.
|
||||||
|
// PromptHash is the same opaque equality value as PreparedRun.PromptHash for
|
||||||
|
// the selected definition and observed source state; its spelling, length,
|
||||||
|
// encoding, algorithm, and security properties are not contracts.
|
||||||
|
//
|
||||||
|
// This method does not return prompt bodies, templates, source paths, schemas,
|
||||||
|
// rendered messages, or execution settings. It does not resolve a profile or
|
||||||
|
// credential, read artifacts or schemas, render, validate, admit capacity,
|
||||||
|
// contact a provider, or generate model output. The returned PromptInspection
|
||||||
|
// and its input slice are caller-owned. Filesystem-backed inspection is a
|
||||||
|
// point-in-time lookup and does not freeze a definition for later execution.
|
||||||
|
//
|
||||||
|
// A nil Engine returns an error matching ErrInvalidConfig. A blank prompt ID
|
||||||
|
// matches ErrInvalidRequest. An absent exact ID or version matches
|
||||||
|
// ErrPromptNotFound and not ErrPromptLoad. Malformed, unreadable, duplicate,
|
||||||
|
// ambiguous, referenced-content, or hashing failures match ErrPromptLoad.
|
||||||
|
// Cancellation during lookup matches ErrPromptLoad while preserving the
|
||||||
|
// context error. InspectPrompt returns no partial result on error.
|
||||||
|
func (e *Engine) InspectPrompt(
|
||||||
|
ctx context.Context,
|
||||||
|
promptID string,
|
||||||
|
promptVersion string,
|
||||||
|
) (*PromptInspection, error) {
|
||||||
|
if e == nil || e.runner == nil {
|
||||||
|
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
|
||||||
|
}
|
||||||
|
|
||||||
|
inspection, err := e.runner.InspectPrompt(ctx, promptID, promptVersion)
|
||||||
|
if err != nil {
|
||||||
|
return nil, mapPublicError(err)
|
||||||
|
}
|
||||||
|
return fromDomainPromptInspection(inspection), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// InspectProfile resolves one explicit profile without selecting a prompt or
|
||||||
|
// starting execution work.
|
||||||
|
//
|
||||||
|
// InspectProfile trims surrounding whitespace from profileID and looks up the
|
||||||
|
// resulting nonblank ID exactly and case-sensitively through the engine's
|
||||||
|
// in-memory, ordinary configured-source, application fallback, and built-in
|
||||||
|
// profile precedence. It applies the framework timeout baseline, selected
|
||||||
|
// backend, and then selected profile to EffectiveModelParams without a request
|
||||||
|
// override. BackendID is empty for an endpoint-only profile.
|
||||||
|
//
|
||||||
|
// APIKeyEnv in the returned target is an environment-variable name, never its
|
||||||
|
// value. APIKeyRequired instead reports a direct credential requirement and is
|
||||||
|
// mutually exclusive with a nonblank APIKeyEnv. InspectProfile neither derives
|
||||||
|
// an ID from a prompt default_profile nor checks credential availability, so an
|
||||||
|
// absent or blank named environment variable is not an error.
|
||||||
|
//
|
||||||
|
// The returned ProfileInspection and all nested mutable values are
|
||||||
|
// caller-owned. Filesystem-backed inspection is a point-in-time lookup and
|
||||||
|
// does not freeze the profile for a later execution. This method does not load
|
||||||
|
// a prompt, render, read artifacts or schemas, admit backend capacity, contact
|
||||||
|
// a provider, or generate model output.
|
||||||
|
//
|
||||||
|
// A nil Engine returns an error matching ErrInvalidConfig. A blank profile ID
|
||||||
|
// matches ErrInvalidRequest. An absent exact ID matches ErrProfileNotFound and
|
||||||
|
// not ErrProfileLoad. Malformed or unreadable profile data, an unknown backend,
|
||||||
|
// or an invalid resolved target matches ErrProfileLoad. Cancellation during
|
||||||
|
// profile loading matches ErrProfileLoad while preserving the context error.
|
||||||
|
// InspectProfile returns no partial result on error.
|
||||||
|
func (e *Engine) InspectProfile(ctx context.Context, profileID string) (*ProfileInspection, error) {
|
||||||
|
if e == nil || e.runner == nil {
|
||||||
|
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
|
||||||
|
}
|
||||||
|
|
||||||
|
inspection, err := e.runner.InspectProfile(ctx, profileID)
|
||||||
|
if err != nil {
|
||||||
|
return nil, mapPublicError(err)
|
||||||
|
}
|
||||||
|
return fromDomainProfileInspection(inspection), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Prepare resolves and renders a prompt request without calling an LLM.
|
||||||
|
//
|
||||||
|
// Prepare selects the prompt and profile, resolves any selected backend and
|
||||||
|
// effective execution settings, resolves the output contract, loads and hashes
|
||||||
|
// inputs, loads structured-output schema metadata when required, and renders
|
||||||
|
// the session ID and messages. The returned PreparedRun is owned by the caller
|
||||||
|
// and never contains a resolved API-key value, model output, or validation
|
||||||
|
// result.
|
||||||
|
//
|
||||||
|
// A nil Engine returns an error matching ErrInvalidConfig. Request and
|
||||||
|
// preparation failures may match ErrInvalidRequest, ErrPromptNotFound,
|
||||||
|
// ErrPromptLoad, ErrProfileNotFound, ErrProfileLoad, ErrProfileRequired,
|
||||||
|
// ErrAPIKeyEnvMissing, ErrArtifactLoad, ErrPromptRender, or ErrValidation as
|
||||||
|
// applicable. Cancellation is passed to the active collaborator and is
|
||||||
|
// reported in the applicable operation category; no general errors.Is
|
||||||
|
// relationship to ctx.Err is promised. Prepare returns no partial result on
|
||||||
|
// error.
|
||||||
func (e *Engine) Prepare(ctx context.Context, req RunRequest) (*PreparedRun, error) {
|
func (e *Engine) Prepare(ctx context.Context, req RunRequest) (*PreparedRun, error) {
|
||||||
if e == nil || e.runner == nil {
|
if e == nil || e.runner == nil {
|
||||||
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
|
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
|
||||||
@@ -323,7 +600,57 @@ func (e *Engine) Prepare(ctx context.Context, req RunRequest) (*PreparedRun, err
|
|||||||
return fromDomainPreparedRun(prepared), nil
|
return fromDomainPreparedRun(prepared), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Run executes a prompt request and returns the generated artifact and metadata.
|
// PrepareExecution completely prepares a prompt request without calling the
|
||||||
|
// configured LLMClient or reserving backend admission capacity.
|
||||||
|
//
|
||||||
|
// The returned opaque handle is bound to this Engine and permits one
|
||||||
|
// [Engine.RunPrepared] invocation. Preparation freezes the selected sources,
|
||||||
|
// rendered messages, effective settings, inputs, provider structured-output
|
||||||
|
// metadata, and validation resources needed by that invocation. The handle
|
||||||
|
// retains a direct RunRequest.APIKey only in private execution state;
|
||||||
|
// [PreparedExecution.Details] is credential-redacted.
|
||||||
|
//
|
||||||
|
// The context governs preparation only. Cancellation after this method
|
||||||
|
// returns does not invalidate the handle or propagate to RunPrepared.
|
||||||
|
// PrepareExecution returns the same error categories as [Engine.Prepare] and
|
||||||
|
// returns no handle on error. A nil Engine returns an error matching
|
||||||
|
// ErrInvalidConfig.
|
||||||
|
func (e *Engine) PrepareExecution(ctx context.Context, req RunRequest) (*PreparedExecution, error) {
|
||||||
|
if e == nil || e.runner == nil {
|
||||||
|
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
|
||||||
|
}
|
||||||
|
|
||||||
|
domainReq, err := toDomainRunRequest(req)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("%w: %v", ErrInvalidRequest, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
prepared, err := e.runner.PrepareExecution(ctx, domainReq)
|
||||||
|
if err != nil {
|
||||||
|
return nil, mapPublicError(err)
|
||||||
|
}
|
||||||
|
return &PreparedExecution{internal: prepared}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Run prepares a request, invokes the configured LLMClient, and validates the
|
||||||
|
// generated output.
|
||||||
|
//
|
||||||
|
// A content-validation failure is a successful run whose
|
||||||
|
// RunResult.Validation has Status ValidationFailed. An inability to perform
|
||||||
|
// validation returns an error matching ErrValidation and no partial result.
|
||||||
|
// The public Engine does not perform output repair, so validation is
|
||||||
|
// single-pass even when OutputContract.RepairAttempts is positive.
|
||||||
|
//
|
||||||
|
// Run can return every error category documented by [Engine.Prepare], plus
|
||||||
|
// ErrCapacityExceeded and ErrLLMGenerate. An engine admission rejection is
|
||||||
|
// discoverable as [CapacityError] and still matches ErrCapacityExceeded. It
|
||||||
|
// occurs before artifacts, schemas, rendering, or model generation because the
|
||||||
|
// selected backend's admission capacity is full; it does not match
|
||||||
|
// ErrInvalidRequest or ErrLLMGenerate. Errors from injected clients remain
|
||||||
|
// available through errors.Is. Cancellation while waiting for model-generation
|
||||||
|
// capacity matches both ErrLLMGenerate and the context error. Cancellation
|
||||||
|
// otherwise follows the active collaborator's documented behavior. A nil
|
||||||
|
// Engine returns ErrInvalidConfig. Run returns no partial result on error.
|
||||||
func (e *Engine) Run(ctx context.Context, req RunRequest) (*RunResult, error) {
|
func (e *Engine) Run(ctx context.Context, req RunRequest) (*RunResult, error) {
|
||||||
if e == nil || e.runner == nil {
|
if e == nil || e.runner == nil {
|
||||||
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
|
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
|
||||||
@@ -340,3 +667,42 @@ func (e *Engine) Run(ctx context.Context, req RunRequest) (*RunResult, error) {
|
|||||||
}
|
}
|
||||||
return fromDomainRunResult(result), nil
|
return fromDomainRunResult(result), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// RunPrepared atomically claims and executes a handle created by
|
||||||
|
// [Engine.PrepareExecution].
|
||||||
|
//
|
||||||
|
// A valid owning-Engine invocation consumes the handle's one attempt before
|
||||||
|
// credential revalidation, backend admission, generation, or validation.
|
||||||
|
// Cancellation, capacity rejection, generation failure, operational
|
||||||
|
// validation failure, and success all leave the handle unusable. A nil,
|
||||||
|
// zero-value, foreign-Engine, discarded, claimed, or used handle returns an
|
||||||
|
// error matching ErrInvalidRequest; a nil Engine returns ErrInvalidConfig and
|
||||||
|
// does not claim the handle.
|
||||||
|
//
|
||||||
|
// The supplied context governs this execution attempt independently of the
|
||||||
|
// preparation context. It covers credential revalidation, admission,
|
||||||
|
// generation, validation, and any internal repair. Result timing begins after
|
||||||
|
// the claim and excludes preparation and consumer-held delay.
|
||||||
|
//
|
||||||
|
// RunPrepared can return ErrInvalidRequest, ErrAPIKeyEnvMissing,
|
||||||
|
// ErrCapacityExceeded, ErrLLMGenerate, or ErrValidation as applicable while
|
||||||
|
// preserving documented collaborator and context identities. An engine
|
||||||
|
// admission rejection is discoverable as [CapacityError] and still matches
|
||||||
|
// ErrCapacityExceeded. A completed content-validation rejection is returned
|
||||||
|
// in RunResult, not as an operational error. An operational error returns no
|
||||||
|
// partial RunResult.
|
||||||
|
func (e *Engine) RunPrepared(ctx context.Context, prepared *PreparedExecution) (*RunResult, error) {
|
||||||
|
if e == nil || e.runner == nil {
|
||||||
|
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
|
||||||
|
}
|
||||||
|
|
||||||
|
var internal *usecase.PreparedExecution
|
||||||
|
if prepared != nil {
|
||||||
|
internal = prepared.internal
|
||||||
|
}
|
||||||
|
result, err := e.runner.RunPrepared(ctx, internal)
|
||||||
|
if err != nil {
|
||||||
|
return nil, mapPublicError(err)
|
||||||
|
}
|
||||||
|
return fromDomainRunResult(result), nil
|
||||||
|
}
|
||||||
|
|||||||
1328
engine_test.go
1328
engine_test.go
File diff suppressed because it is too large
Load Diff
@@ -3,7 +3,9 @@ package promptkit
|
|||||||
import (
|
import (
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/capacity"
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/profile"
|
"gitea.maximumdirect.net/eric/promptkit/internal/profile"
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/promptdef"
|
"gitea.maximumdirect.net/eric/promptkit/internal/promptdef"
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/usecase"
|
"gitea.maximumdirect.net/eric/promptkit/internal/usecase"
|
||||||
@@ -13,6 +15,11 @@ func mapPublicError(err error) error {
|
|||||||
if err == nil {
|
if err == nil {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
var internalCapacityError *usecase.CapacityError
|
||||||
|
if errors.As(err, &internalCapacityError) && internalCapacityError != nil &&
|
||||||
|
strings.TrimSpace(internalCapacityError.BackendID) != "" {
|
||||||
|
return &CapacityError{BackendID: internalCapacityError.BackendID}
|
||||||
|
}
|
||||||
publicErr := publicErrorFor(err)
|
publicErr := publicErrorFor(err)
|
||||||
if publicErr == nil {
|
if publicErr == nil {
|
||||||
return err
|
return err
|
||||||
@@ -38,6 +45,8 @@ func publicErrorFor(err error) error {
|
|||||||
return ErrProfileLoad
|
return ErrProfileLoad
|
||||||
case errors.Is(err, usecase.ErrAPIKeyEnvMissing):
|
case errors.Is(err, usecase.ErrAPIKeyEnvMissing):
|
||||||
return errors.Join(ErrInvalidRequest, ErrAPIKeyEnvMissing)
|
return errors.Join(ErrInvalidRequest, ErrAPIKeyEnvMissing)
|
||||||
|
case errors.Is(err, capacity.ErrCapacityExceeded):
|
||||||
|
return ErrCapacityExceeded
|
||||||
case errors.Is(err, usecase.ErrArtifactLoad):
|
case errors.Is(err, usecase.ErrArtifactLoad):
|
||||||
return ErrArtifactLoad
|
return ErrArtifactLoad
|
||||||
case errors.Is(err, usecase.ErrPromptRender):
|
case errors.Is(err, usecase.ErrPromptRender):
|
||||||
|
|||||||
50
errors_internal_test.go
Normal file
50
errors_internal_test.go
Normal file
@@ -0,0 +1,50 @@
|
|||||||
|
package promptkit
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/usecase"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestMapPublicErrorPreservesGenerationCancellation(t *testing.T) {
|
||||||
|
internalErr := fmt.Errorf("%w: %w", usecase.ErrLLMGenerate, context.Canceled)
|
||||||
|
|
||||||
|
err := mapPublicError(internalErr)
|
||||||
|
if !errors.Is(err, ErrLLMGenerate) {
|
||||||
|
t.Fatalf("mapped error=%v, want ErrLLMGenerate", err)
|
||||||
|
}
|
||||||
|
if !errors.Is(err, context.Canceled) {
|
||||||
|
t.Fatalf("mapped error=%v, want context.Canceled", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMapPublicErrorTranslatesCapacityError(t *testing.T) {
|
||||||
|
internalErr := &usecase.CapacityError{BackendID: "limited"}
|
||||||
|
|
||||||
|
err := mapPublicError(internalErr)
|
||||||
|
var publicErr *CapacityError
|
||||||
|
if !errors.As(err, &publicErr) || publicErr == nil {
|
||||||
|
t.Fatalf("mapped error=%v, want public CapacityError", err)
|
||||||
|
}
|
||||||
|
if publicErr.BackendID != "limited" {
|
||||||
|
t.Fatalf("mapped backend ID=%q, want limited", publicErr.BackendID)
|
||||||
|
}
|
||||||
|
if !errors.Is(err, ErrCapacityExceeded) {
|
||||||
|
t.Fatalf("mapped error=%v, want ErrCapacityExceeded", err)
|
||||||
|
}
|
||||||
|
if errors.Is(err, ErrInvalidRequest) || errors.Is(err, ErrLLMGenerate) {
|
||||||
|
t.Fatalf("mapped capacity error has an unrelated category: %v", err)
|
||||||
|
}
|
||||||
|
var leakedInternalErr *usecase.CapacityError
|
||||||
|
if errors.As(err, &leakedInternalErr) {
|
||||||
|
t.Fatalf("mapped error exposes internal CapacityError: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
internalErr.BackendID = "changed"
|
||||||
|
if publicErr.BackendID != "limited" {
|
||||||
|
t.Fatalf("mapped backend ID changed with source error: %q", publicErr.BackendID)
|
||||||
|
}
|
||||||
|
}
|
||||||
81
examples/go-library/run/main.go
Normal file
81
examples/go-library/run/main.go
Normal file
@@ -0,0 +1,81 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
type deterministicClient struct{}
|
||||||
|
|
||||||
|
func (deterministicClient) Generate(
|
||||||
|
ctx context.Context,
|
||||||
|
_ promptkit.GenerateRequest,
|
||||||
|
) (*promptkit.GenerateResponse, error) {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return &promptkit.GenerateResponse{
|
||||||
|
Content: "Ada finished the migration review.",
|
||||||
|
Usage: promptkit.TokenUsage{
|
||||||
|
PromptTokens: 12,
|
||||||
|
CompletionTokens: 6,
|
||||||
|
TotalTokens: 18,
|
||||||
|
},
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type summary struct {
|
||||||
|
Output string `json:"output"`
|
||||||
|
ValidationStatus promptkit.ValidationStatus `json:"validation_status"`
|
||||||
|
IsValid bool `json:"is_valid"`
|
||||||
|
Model string `json:"model"`
|
||||||
|
TotalTokens int `json:"total_tokens"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
engine, err := promptkit.NewEngine(
|
||||||
|
promptkit.Config{},
|
||||||
|
promptkit.WithPromptFile("examples/go-library/run/prompt.yaml"),
|
||||||
|
promptkit.WithProfiles(promptkit.Profile{
|
||||||
|
ID: "offline-example",
|
||||||
|
Endpoint: "https://example.invalid/v1",
|
||||||
|
Model: "offline-model",
|
||||||
|
}),
|
||||||
|
promptkit.WithLLMClient(deterministicClient{}),
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
exit(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
result, err := engine.Run(context.Background(), promptkit.RunRequest{
|
||||||
|
PromptID: "example.run",
|
||||||
|
Inputs: map[string]promptkit.ArtifactRef{
|
||||||
|
"note": promptkit.Inline("Ada finished the migration review."),
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
exit(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
encoder := json.NewEncoder(os.Stdout)
|
||||||
|
encoder.SetIndent("", " ")
|
||||||
|
if err := encoder.Encode(summary{
|
||||||
|
Output: result.RawOutput,
|
||||||
|
ValidationStatus: result.Validation.Status,
|
||||||
|
IsValid: result.Validation.IsValid,
|
||||||
|
Model: result.ModelName,
|
||||||
|
TotalTokens: result.Usage.TotalTokens,
|
||||||
|
}); err != nil {
|
||||||
|
exit(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func exit(err error) {
|
||||||
|
fmt.Fprintln(os.Stderr, err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
16
examples/go-library/run/prompt.yaml
Normal file
16
examples/go-library/run/prompt.yaml
Normal file
@@ -0,0 +1,16 @@
|
|||||||
|
id: example.run
|
||||||
|
version: "1.0.0"
|
||||||
|
default_profile: offline-example
|
||||||
|
description: Run a prompt with a deterministic injected model client.
|
||||||
|
inputs:
|
||||||
|
- name: note
|
||||||
|
required: true
|
||||||
|
content_type: text/plain
|
||||||
|
messages:
|
||||||
|
- role: system
|
||||||
|
content: Summarize the note in one sentence.
|
||||||
|
- role: user
|
||||||
|
content: '{{input "note"}}'
|
||||||
|
output:
|
||||||
|
format: text
|
||||||
|
validation_mode: basic
|
||||||
@@ -2,19 +2,23 @@ package promptkit
|
|||||||
|
|
||||||
import "fmt"
|
import "fmt"
|
||||||
|
|
||||||
// String returns a concise request summary without exposing direct API keys.
|
// String returns a concise request summary without exposing the direct API key
|
||||||
|
// or input and variable contents. Reflection-based formatting does not carry
|
||||||
|
// this guarantee.
|
||||||
func (r RunRequest) String() string {
|
func (r RunRequest) String() string {
|
||||||
return r.redactedString()
|
return r.redactedString()
|
||||||
}
|
}
|
||||||
|
|
||||||
// GoString returns a concise request summary without exposing direct API keys.
|
// GoString returns a concise request summary without exposing the direct API
|
||||||
|
// key or input and variable contents. Reflection-based formatting does not
|
||||||
|
// carry this guarantee.
|
||||||
func (r RunRequest) GoString() string {
|
func (r RunRequest) GoString() string {
|
||||||
return r.redactedString()
|
return r.redactedString()
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r RunRequest) redactedString() string {
|
func (r RunRequest) redactedString() string {
|
||||||
return fmt.Sprintf(
|
return fmt.Sprintf(
|
||||||
"promptkit.RunRequest{PromptID:%q PromptVersion:%q ProfileID:%q APIKeySet:%t Inputs:%d Vars:%d ExecutionSet:%t ValidationSet:%t Metadata:%d}",
|
"promptkit.RunRequest{PromptID:%q PromptVersion:%q ProfileID:%q APIKeySet:%t Inputs:%d Vars:%d ExecutionSet:%t ValidationSet:%t}",
|
||||||
r.PromptID,
|
r.PromptID,
|
||||||
r.PromptVersion,
|
r.PromptVersion,
|
||||||
r.ProfileID,
|
r.ProfileID,
|
||||||
@@ -23,18 +27,19 @@ func (r RunRequest) redactedString() string {
|
|||||||
len(r.Vars),
|
len(r.Vars),
|
||||||
r.Execution != nil,
|
r.Execution != nil,
|
||||||
r.Validation != nil,
|
r.Validation != nil,
|
||||||
len(r.Metadata),
|
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// String returns a concise request summary without exposing direct API keys or
|
// String returns a concise request summary without exposing direct API keys or
|
||||||
// rendered prompt content.
|
// rendered prompt content. Reflection-based formatting does not carry this
|
||||||
|
// guarantee.
|
||||||
func (r GenerateRequest) String() string {
|
func (r GenerateRequest) String() string {
|
||||||
return r.redactedString()
|
return r.redactedString()
|
||||||
}
|
}
|
||||||
|
|
||||||
// GoString returns a concise request summary without exposing direct API keys or
|
// GoString returns a concise request summary without exposing direct API keys or
|
||||||
// rendered prompt content.
|
// rendered prompt content. Reflection-based formatting does not carry this
|
||||||
|
// guarantee.
|
||||||
func (r GenerateRequest) GoString() string {
|
func (r GenerateRequest) GoString() string {
|
||||||
return r.redactedString()
|
return r.redactedString()
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -16,10 +16,12 @@ import (
|
|||||||
|
|
||||||
var (
|
var (
|
||||||
ErrUnsupportedRefType = errors.New("unsupported artifact reference type")
|
ErrUnsupportedRefType = errors.New("unsupported artifact reference type")
|
||||||
ErrMissingInlineBody = errors.New("missing body for inline artifact")
|
|
||||||
ErrMissingFilePath = errors.New("missing file path for file artifact")
|
ErrMissingFilePath = errors.New("missing file path for file artifact")
|
||||||
|
ErrUnsupportedFile = errors.New("file artifact path is not a regular file")
|
||||||
)
|
)
|
||||||
|
|
||||||
|
const fileReadChunkSize = 64 * 1024
|
||||||
|
|
||||||
// Reader resolves artifact references into actual artifacts.
|
// Reader resolves artifact references into actual artifacts.
|
||||||
type Reader interface {
|
type Reader interface {
|
||||||
Read(ctx context.Context, ref domain.ArtifactRef) (*domain.Artifact, error)
|
Read(ctx context.Context, ref domain.ArtifactRef) (*domain.Artifact, error)
|
||||||
@@ -34,7 +36,7 @@ type CompositeReader struct {
|
|||||||
func NewCompositeReader() Reader {
|
func NewCompositeReader() Reader {
|
||||||
return &CompositeReader{
|
return &CompositeReader{
|
||||||
inlineReader: &inlineReader{},
|
inlineReader: &inlineReader{},
|
||||||
fileReader: &fileReader{},
|
fileReader: &fileReader{open: openArtifactFile},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -64,10 +66,6 @@ func (r *inlineReader) Read(ctx context.Context, ref domain.ArtifactRef) (*domai
|
|||||||
default:
|
default:
|
||||||
}
|
}
|
||||||
|
|
||||||
if ref.Body == "" {
|
|
||||||
return nil, ErrMissingInlineBody
|
|
||||||
}
|
|
||||||
|
|
||||||
body := []byte(ref.Body)
|
body := []byte(ref.Body)
|
||||||
return &domain.Artifact{
|
return &domain.Artifact{
|
||||||
ContentType: defaults.ContentTypeTextPlain,
|
ContentType: defaults.ContentTypeTextPlain,
|
||||||
@@ -78,7 +76,15 @@ func (r *inlineReader) Read(ctx context.Context, ref domain.ArtifactRef) (*domai
|
|||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
type fileReader struct{}
|
type artifactFile interface {
|
||||||
|
Read([]byte) (int, error)
|
||||||
|
Stat() (os.FileInfo, error)
|
||||||
|
Close() error
|
||||||
|
}
|
||||||
|
|
||||||
|
type fileReader struct {
|
||||||
|
open func(string) (artifactFile, error)
|
||||||
|
}
|
||||||
|
|
||||||
func (r *fileReader) Read(ctx context.Context, ref domain.ArtifactRef) (*domain.Artifact, error) {
|
func (r *fileReader) Read(ctx context.Context, ref domain.ArtifactRef) (*domain.Artifact, error) {
|
||||||
select {
|
select {
|
||||||
@@ -91,25 +97,71 @@ func (r *fileReader) Read(ctx context.Context, ref domain.ArtifactRef) (*domain.
|
|||||||
return nil, ErrMissingFilePath
|
return nil, ErrMissingFilePath
|
||||||
}
|
}
|
||||||
|
|
||||||
return readFileArtifact(ref.URI)
|
return readFileArtifact(ctx, ref.URI, r.open)
|
||||||
}
|
}
|
||||||
|
|
||||||
func readFileArtifact(path string) (*domain.Artifact, error) {
|
func openArtifactFile(path string) (artifactFile, error) {
|
||||||
file, err := os.Open(path)
|
return os.Open(path)
|
||||||
|
}
|
||||||
|
|
||||||
|
func readFileArtifact(ctx context.Context, path string, open func(string) (artifactFile, error)) (*domain.Artifact, error) {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
info, err := os.Stat(path)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to read file %s: %w", path, err)
|
||||||
|
}
|
||||||
|
if !info.Mode().IsRegular() {
|
||||||
|
return nil, fmt.Errorf("%w: %s", ErrUnsupportedFile, path)
|
||||||
|
}
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
file, err := open(path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("failed to read file %s: %w", path, err)
|
return nil, fmt.Errorf("failed to read file %s: %w", path, err)
|
||||||
}
|
}
|
||||||
defer file.Close()
|
defer file.Close()
|
||||||
|
|
||||||
data, err := io.ReadAll(file)
|
openedInfo, err := file.Stat()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("failed to read file %s: %w", path, err)
|
return nil, fmt.Errorf("failed to inspect opened file %s: %w", path, err)
|
||||||
|
}
|
||||||
|
if !openedInfo.Mode().IsRegular() {
|
||||||
|
return nil, fmt.Errorf("%w: %s", ErrUnsupportedFile, path)
|
||||||
|
}
|
||||||
|
|
||||||
|
data := make([]byte, 0)
|
||||||
|
chunk := make([]byte, fileReadChunkSize)
|
||||||
|
for {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
n, readErr := file.Read(chunk)
|
||||||
|
if n > 0 {
|
||||||
|
data = append(data, chunk[:n]...)
|
||||||
|
}
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if errors.Is(readErr, io.EOF) {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if readErr != nil {
|
||||||
|
return nil, fmt.Errorf("failed to read file %s: %w", path, readErr)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
contentType := mime.TypeByExtension(filepath.Ext(path))
|
contentType := mime.TypeByExtension(filepath.Ext(path))
|
||||||
if contentType == "" {
|
if contentType == "" {
|
||||||
contentType = defaults.ContentTypeTextPlain
|
contentType = defaults.ContentTypeTextPlain
|
||||||
}
|
}
|
||||||
|
hash := fmt.Sprintf("%x", sha256.Sum256(data))
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
return &domain.Artifact{
|
return &domain.Artifact{
|
||||||
Name: filepath.Base(path),
|
Name: filepath.Base(path),
|
||||||
@@ -117,6 +169,6 @@ func readFileArtifact(path string) (*domain.Artifact, error) {
|
|||||||
Body: data,
|
Body: data,
|
||||||
URI: path,
|
URI: path,
|
||||||
Size: int64(len(data)),
|
Size: int64(len(data)),
|
||||||
Hash: fmt.Sprintf("%x", sha256.Sum256(data)),
|
Hash: hash,
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|||||||
43
internal/artifact/reader_fifo_linux_test.go
Normal file
43
internal/artifact/reader_fifo_linux_test.go
Normal file
@@ -0,0 +1,43 @@
|
|||||||
|
//go:build linux
|
||||||
|
|
||||||
|
package artifact
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"path/filepath"
|
||||||
|
"syscall"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestFileReaderRejectsFIFOBeforeOpen(t *testing.T) {
|
||||||
|
path := filepath.Join(t.TempDir(), "artifact.fifo")
|
||||||
|
if err := syscall.Mkfifo(path, 0o600); err != nil {
|
||||||
|
t.Fatalf("create fifo: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
type result struct {
|
||||||
|
artifact *domain.Artifact
|
||||||
|
err error
|
||||||
|
}
|
||||||
|
done := make(chan result, 1)
|
||||||
|
go func() {
|
||||||
|
artifact, err := NewCompositeReader().Read(context.Background(), domain.ArtifactRef{
|
||||||
|
Type: domain.ArtifactRefFile,
|
||||||
|
URI: path,
|
||||||
|
})
|
||||||
|
done <- result{artifact: artifact, err: err}
|
||||||
|
}()
|
||||||
|
|
||||||
|
select {
|
||||||
|
case got := <-done:
|
||||||
|
if got.artifact != nil || !errors.Is(got.err, ErrUnsupportedFile) {
|
||||||
|
t.Fatalf("artifact=%#v err=%v, want nil/ErrUnsupportedFile", got.artifact, got.err)
|
||||||
|
}
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("FIFO read blocked instead of rejecting the non-regular file")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,6 +1,7 @@
|
|||||||
package artifact
|
package artifact
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
"os"
|
"os"
|
||||||
@@ -11,54 +12,97 @@ import (
|
|||||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestCompositeReader_Read(t *testing.T) {
|
func TestCompositeReaderRejectsUnsupportedReferences(t *testing.T) {
|
||||||
|
_, err := NewCompositeReader().Read(context.Background(), domain.ArtifactRef{
|
||||||
|
Type: domain.ArtifactRefType("unsupported"),
|
||||||
|
URI: "unsupported://bucket/key",
|
||||||
|
})
|
||||||
|
if !errors.Is(err, ErrUnsupportedRefType) {
|
||||||
|
t.Fatalf("expected ErrUnsupportedRefType, got %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCompositeReaderSourceParityAndOpaqueHashes(t *testing.T) {
|
||||||
reader := NewCompositeReader()
|
reader := NewCompositeReader()
|
||||||
ctx := context.Background()
|
hashes := make(map[string]string)
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
content string
|
||||||
|
}{
|
||||||
|
{name: "empty", content: ""},
|
||||||
|
{name: "ordinary", content: "same content"},
|
||||||
|
{name: "changed", content: "changed content"},
|
||||||
|
}
|
||||||
|
|
||||||
t.Run("inline artifact", func(t *testing.T) {
|
for _, tc := range tests {
|
||||||
ref := domain.ArtifactRef{
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
Type: domain.ArtifactRefInline,
|
filePath := filepath.Join(t.TempDir(), "artifact.txt")
|
||||||
Body: "hello world",
|
if err := os.WriteFile(filePath, []byte(tc.content), 0o600); err != nil {
|
||||||
}
|
t.Fatal(err)
|
||||||
art, err := reader.Read(ctx, ref)
|
}
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("unexpected error: %v", err)
|
|
||||||
}
|
|
||||||
if string(art.Body) != "hello world" {
|
|
||||||
t.Errorf("expected 'hello world', got %s", string(art.Body))
|
|
||||||
}
|
|
||||||
if art.ContentType != "text/plain" {
|
|
||||||
t.Errorf("expected text/plain content type, got %q", art.ContentType)
|
|
||||||
}
|
|
||||||
if art.Hash != "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9" {
|
|
||||||
t.Errorf("unexpected hash: %s", art.Hash)
|
|
||||||
}
|
|
||||||
if art.Size != int64(len(ref.Body)) {
|
|
||||||
t.Errorf("expected size %d, got %d", len(ref.Body), art.Size)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("inline artifact missing body", func(t *testing.T) {
|
sources := []struct {
|
||||||
ref := domain.ArtifactRef{
|
name string
|
||||||
Type: domain.ArtifactRefInline,
|
ref domain.ArtifactRef
|
||||||
Body: "",
|
wantURI string
|
||||||
}
|
}{
|
||||||
_, err := reader.Read(ctx, ref)
|
{
|
||||||
if !errors.Is(err, ErrMissingInlineBody) {
|
name: "inline",
|
||||||
t.Errorf("expected ErrMissingInlineBody, got %v", err)
|
ref: domain.ArtifactRef{Type: domain.ArtifactRefInline, Body: tc.content},
|
||||||
}
|
},
|
||||||
})
|
{
|
||||||
|
name: "inline with uri",
|
||||||
|
ref: domain.ArtifactRef{Type: domain.ArtifactRefInline, URI: "memory://input", Body: tc.content},
|
||||||
|
wantURI: "memory://input",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "file",
|
||||||
|
ref: domain.ArtifactRef{Type: domain.ArtifactRefFile, URI: filePath},
|
||||||
|
wantURI: filePath,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
t.Run("unsupported ref type", func(t *testing.T) {
|
var sourceHash string
|
||||||
ref := domain.ArtifactRef{
|
for _, source := range sources {
|
||||||
Type: domain.ArtifactRefType("unsupported"),
|
t.Run(source.name, func(t *testing.T) {
|
||||||
URI: "unsupported://bucket/key",
|
first, err := reader.Read(context.Background(), source.ref)
|
||||||
}
|
if err != nil {
|
||||||
_, err := reader.Read(ctx, ref)
|
t.Fatalf("first read: %v", err)
|
||||||
if !errors.Is(err, ErrUnsupportedRefType) {
|
}
|
||||||
t.Error("expected error for unsupported type")
|
second, err := reader.Read(context.Background(), source.ref)
|
||||||
}
|
if err != nil {
|
||||||
})
|
t.Fatalf("second read: %v", err)
|
||||||
|
}
|
||||||
|
if string(first.Body) != tc.content || first.Size != int64(len(tc.content)) {
|
||||||
|
t.Fatalf("body=%q size=%d, want %q/%d", first.Body, first.Size, tc.content, len(tc.content))
|
||||||
|
}
|
||||||
|
if first.URI != source.wantURI {
|
||||||
|
t.Fatalf("URI = %q, want %q", first.URI, source.wantURI)
|
||||||
|
}
|
||||||
|
if first.Hash == "" || first.Hash != second.Hash {
|
||||||
|
t.Fatalf("hashes are not non-empty and stable: %q/%q", first.Hash, second.Hash)
|
||||||
|
}
|
||||||
|
if sourceHash == "" {
|
||||||
|
sourceHash = first.Hash
|
||||||
|
} else if first.Hash != sourceHash {
|
||||||
|
t.Fatalf("equal content hashes differ: %q/%q", sourceHash, first.Hash)
|
||||||
|
}
|
||||||
|
if source.ref.Type == domain.ArtifactRefFile {
|
||||||
|
if first.Name != filepath.Base(filePath) || !strings.HasPrefix(first.ContentType, "text/plain") {
|
||||||
|
t.Fatalf("unexpected file metadata: %+v", first)
|
||||||
|
}
|
||||||
|
} else if first.ContentType != "text/plain" {
|
||||||
|
t.Fatalf("inline content type = %q", first.ContentType)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
hashes[tc.name] = sourceHash
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
if hashes["empty"] == hashes["ordinary"] || hashes["ordinary"] == hashes["changed"] {
|
||||||
|
t.Fatalf("changed content did not change opaque hash: %#v", hashes)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestCompositeReaderCopiesInlineData(t *testing.T) {
|
func TestCompositeReaderCopiesInlineData(t *testing.T) {
|
||||||
@@ -87,94 +131,154 @@ func TestCompositeReaderCopiesInlineData(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestCompositeReaderHonorsCancellation(t *testing.T) {
|
func TestCompositeReaderHonorsPreCancellation(t *testing.T) {
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
filePath := filepath.Join(t.TempDir(), "artifact.txt")
|
||||||
cancel()
|
if err := os.WriteFile(filePath, []byte("ignored"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
ref domain.ArtifactRef
|
||||||
|
}{
|
||||||
|
{name: "inline", ref: domain.ArtifactRef{Type: domain.ArtifactRefInline, Body: "ignored"}},
|
||||||
|
{name: "file", ref: domain.ArtifactRef{Type: domain.ArtifactRefFile, URI: filePath}},
|
||||||
|
}
|
||||||
|
|
||||||
_, err := NewCompositeReader().Read(ctx, domain.ArtifactRef{
|
for _, tc := range tests {
|
||||||
Type: domain.ArtifactRefInline,
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
Body: "ignored",
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
})
|
cancel()
|
||||||
if !errors.Is(err, context.Canceled) {
|
|
||||||
t.Fatalf("expected context cancellation, got %v", err)
|
artifact, err := NewCompositeReader().Read(ctx, tc.ref)
|
||||||
|
if artifact != nil || !errors.Is(err, context.Canceled) {
|
||||||
|
t.Fatalf("artifact=%#v err=%v, want nil/context.Canceled", artifact, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestFileReader_Read(t *testing.T) {
|
func TestFileReaderFailuresAndMetadata(t *testing.T) {
|
||||||
content := []byte("test file content")
|
|
||||||
filePath := filepath.Join(t.TempDir(), "artifact.txt")
|
|
||||||
if err := os.WriteFile(filePath, content, 0o600); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
reader := NewCompositeReader()
|
reader := NewCompositeReader()
|
||||||
ctx := context.Background()
|
|
||||||
|
|
||||||
t.Run("file artifact loading", func(t *testing.T) {
|
|
||||||
ref := domain.ArtifactRef{
|
|
||||||
Type: domain.ArtifactRefFile,
|
|
||||||
URI: filePath,
|
|
||||||
}
|
|
||||||
art, err := reader.Read(ctx, ref)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("unexpected error: %v", err)
|
|
||||||
}
|
|
||||||
if string(art.Body) != string(content) {
|
|
||||||
t.Errorf("expected %s, got %s", string(content), string(art.Body))
|
|
||||||
}
|
|
||||||
if art.Name != filepath.Base(filePath) {
|
|
||||||
t.Errorf("expected name %q, got %q", filepath.Base(filePath), art.Name)
|
|
||||||
}
|
|
||||||
if !strings.HasPrefix(art.ContentType, "text/plain") {
|
|
||||||
t.Errorf("expected text content type, got %q", art.ContentType)
|
|
||||||
}
|
|
||||||
if art.URI != filePath {
|
|
||||||
t.Errorf("expected URI %q, got %q", filePath, art.URI)
|
|
||||||
}
|
|
||||||
if art.Size != int64(len(content)) {
|
|
||||||
t.Errorf("expected size %d, got %d", len(content), art.Size)
|
|
||||||
}
|
|
||||||
if art.Hash != "60f5237ed4049f0382661ef009d2bc42e48c3ceb3edb6600f7024e7ab3b838f3" {
|
|
||||||
t.Errorf("unexpected hash: %s", art.Hash)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("missing file path", func(t *testing.T) {
|
t.Run("missing file path", func(t *testing.T) {
|
||||||
ref := domain.ArtifactRef{
|
_, err := reader.Read(context.Background(), domain.ArtifactRef{Type: domain.ArtifactRefFile})
|
||||||
Type: domain.ArtifactRefFile,
|
|
||||||
URI: "",
|
|
||||||
}
|
|
||||||
_, err := reader.Read(ctx, ref)
|
|
||||||
if !errors.Is(err, ErrMissingFilePath) {
|
if !errors.Is(err, ErrMissingFilePath) {
|
||||||
t.Errorf("expected ErrMissingFilePath, got %v", err)
|
t.Fatalf("expected ErrMissingFilePath, got %v", err)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
t.Run("missing file", func(t *testing.T) {
|
t.Run("missing file", func(t *testing.T) {
|
||||||
ref := domain.ArtifactRef{
|
_, err := reader.Read(context.Background(), domain.ArtifactRef{
|
||||||
Type: domain.ArtifactRefFile,
|
Type: domain.ArtifactRefFile,
|
||||||
URI: filepath.Join(t.TempDir(), "missing.txt"),
|
URI: filepath.Join(t.TempDir(), "missing.txt"),
|
||||||
}
|
})
|
||||||
if _, err := reader.Read(ctx, ref); err == nil {
|
if err == nil {
|
||||||
t.Fatal("expected missing file error")
|
t.Fatal("expected missing file error")
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
|
t.Run("directory rejected before open", func(t *testing.T) {
|
||||||
|
artifact, err := reader.Read(context.Background(), domain.ArtifactRef{
|
||||||
|
Type: domain.ArtifactRefFile,
|
||||||
|
URI: t.TempDir(),
|
||||||
|
})
|
||||||
|
if artifact != nil || !errors.Is(err, ErrUnsupportedFile) {
|
||||||
|
t.Fatalf("artifact=%#v err=%v, want nil/ErrUnsupportedFile", artifact, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("non-regular opened target rejected", func(t *testing.T) {
|
||||||
|
filePath := filepath.Join(t.TempDir(), "artifact.txt")
|
||||||
|
if err := os.WriteFile(filePath, []byte("content"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
directoryInfo, err := os.Stat(t.TempDir())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
fileReader := &fileReader{open: func(path string) (artifactFile, error) {
|
||||||
|
file, err := os.Open(path)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return &reportedInfoFile{artifactFile: file, info: directoryInfo}, nil
|
||||||
|
}}
|
||||||
|
|
||||||
|
artifact, err := fileReader.Read(context.Background(), domain.ArtifactRef{
|
||||||
|
Type: domain.ArtifactRefFile,
|
||||||
|
URI: filePath,
|
||||||
|
})
|
||||||
|
if artifact != nil || !errors.Is(err, ErrUnsupportedFile) {
|
||||||
|
t.Fatalf("artifact=%#v err=%v, want nil/ErrUnsupportedFile", artifact, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
t.Run("unknown extension uses text fallback", func(t *testing.T) {
|
t.Run("unknown extension uses text fallback", func(t *testing.T) {
|
||||||
path := filepath.Join(t.TempDir(), "artifact.unknownextension")
|
filePath := filepath.Join(t.TempDir(), "artifact.unknownextension")
|
||||||
if err := os.WriteFile(path, content, 0o600); err != nil {
|
if err := os.WriteFile(filePath, []byte("content"), 0o600); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
art, err := reader.Read(ctx, domain.ArtifactRef{
|
artifact, err := reader.Read(context.Background(), domain.ArtifactRef{
|
||||||
Type: domain.ArtifactRefFile,
|
Type: domain.ArtifactRefFile,
|
||||||
URI: path,
|
URI: filePath,
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("unexpected error: %v", err)
|
t.Fatalf("read artifact: %v", err)
|
||||||
}
|
}
|
||||||
if art.ContentType != "text/plain" {
|
if artifact.ContentType != "text/plain" {
|
||||||
t.Errorf("expected text/plain fallback, got %q", art.ContentType)
|
t.Fatalf("content type = %q", artifact.ContentType)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestFileReaderCancelsAfterReadProgress(t *testing.T) {
|
||||||
|
filePath := filepath.Join(t.TempDir(), "artifact.bin")
|
||||||
|
content := bytes.Repeat([]byte("x"), fileReadChunkSize*2)
|
||||||
|
if err := os.WriteFile(filePath, content, 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
var opened *cancelAfterProgressFile
|
||||||
|
reader := &fileReader{open: func(path string) (artifactFile, error) {
|
||||||
|
file, err := os.Open(path)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
opened = &cancelAfterProgressFile{artifactFile: file, cancel: cancel}
|
||||||
|
return opened, nil
|
||||||
|
}}
|
||||||
|
|
||||||
|
artifact, err := reader.Read(ctx, domain.ArtifactRef{Type: domain.ArtifactRefFile, URI: filePath})
|
||||||
|
if artifact != nil || !errors.Is(err, context.Canceled) {
|
||||||
|
t.Fatalf("artifact=%#v err=%v, want nil/context.Canceled", artifact, err)
|
||||||
|
}
|
||||||
|
if opened == nil || opened.reads != 1 {
|
||||||
|
t.Fatalf("read count = %v, want one progressing read", opened)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type reportedInfoFile struct {
|
||||||
|
artifactFile
|
||||||
|
info os.FileInfo
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *reportedInfoFile) Stat() (os.FileInfo, error) {
|
||||||
|
return f.info, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type cancelAfterProgressFile struct {
|
||||||
|
artifactFile
|
||||||
|
cancel context.CancelFunc
|
||||||
|
reads int
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *cancelAfterProgressFile) Read(buffer []byte) (int, error) {
|
||||||
|
n, err := f.artifactFile.Read(buffer)
|
||||||
|
if n > 0 {
|
||||||
|
f.reads++
|
||||||
|
f.cancel()
|
||||||
|
}
|
||||||
|
return n, err
|
||||||
|
}
|
||||||
|
|||||||
184
internal/backend/registry.go
Normal file
184
internal/backend/registry.go
Normal file
@@ -0,0 +1,184 @@
|
|||||||
|
// Package backend owns validated, immutable OpenAI-compatible backend
|
||||||
|
// definitions.
|
||||||
|
package backend
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"regexp"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/jsonvalue"
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/llm"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
// OpenRouterID is the reserved ID of Promptkit's built-in OpenRouter
|
||||||
|
// backend.
|
||||||
|
OpenRouterID = "openrouter"
|
||||||
|
|
||||||
|
openRouterEndpoint = "https://openrouter.ai/api/v1"
|
||||||
|
openRouterAPIKeyEnv = "OPENROUTER_API_KEY"
|
||||||
|
|
||||||
|
openRouterConcurrencyLimit = 16
|
||||||
|
defaultQueueCapacity = 1024
|
||||||
|
)
|
||||||
|
|
||||||
|
// ErrBackendNotFound identifies a registry lookup for an unknown backend ID.
|
||||||
|
var ErrBackendNotFound = errors.New("backend not found")
|
||||||
|
|
||||||
|
var environmentVariableName = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`)
|
||||||
|
|
||||||
|
// Registry is an immutable collection of validated backend definitions.
|
||||||
|
type Registry struct {
|
||||||
|
backends map[string]domain.Backend
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewRegistry constructs a registry containing the built-in OpenRouter
|
||||||
|
// definition followed by the supplied additions. Every ID must be unique.
|
||||||
|
func NewRegistry(additions []domain.Backend) (*Registry, error) {
|
||||||
|
registry := &Registry{
|
||||||
|
backends: make(map[string]domain.Backend, len(additions)+1),
|
||||||
|
}
|
||||||
|
|
||||||
|
definitions := make([]domain.Backend, 0, len(additions)+1)
|
||||||
|
definitions = append(definitions, domain.Backend{
|
||||||
|
ID: OpenRouterID,
|
||||||
|
Endpoint: openRouterEndpoint,
|
||||||
|
APIKeyEnv: openRouterAPIKeyEnv,
|
||||||
|
ConcurrencyLimit: openRouterConcurrencyLimit,
|
||||||
|
})
|
||||||
|
definitions = append(definitions, additions...)
|
||||||
|
|
||||||
|
for _, definition := range definitions {
|
||||||
|
definition.ID = strings.TrimSpace(definition.ID)
|
||||||
|
if definition.ID == "" {
|
||||||
|
return nil, errors.New("backend ID must not be blank")
|
||||||
|
}
|
||||||
|
if _, exists := registry.backends[definition.ID]; exists {
|
||||||
|
return nil, fmt.Errorf("backend ID %q is already registered", definition.ID)
|
||||||
|
}
|
||||||
|
|
||||||
|
normalized, err := normalizeBackend(definition)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
registry.backends[normalized.ID] = normalized
|
||||||
|
}
|
||||||
|
|
||||||
|
return registry, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetBackend returns a defensive copy of the backend registered with id.
|
||||||
|
func (r *Registry) GetBackend(id string) (domain.Backend, error) {
|
||||||
|
if r == nil {
|
||||||
|
return domain.Backend{}, fmt.Errorf("%w: %q", ErrBackendNotFound, id)
|
||||||
|
}
|
||||||
|
definition, ok := r.backends[id]
|
||||||
|
if !ok {
|
||||||
|
return domain.Backend{}, fmt.Errorf("%w: %q", ErrBackendNotFound, id)
|
||||||
|
}
|
||||||
|
extraParams, err := jsonvalue.CopyMap(definition.ExtraParams)
|
||||||
|
if err != nil {
|
||||||
|
return domain.Backend{}, fmt.Errorf("copy backend %q: %w", id, err)
|
||||||
|
}
|
||||||
|
definition.ExtraParams = extraParams
|
||||||
|
return definition, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// CapacityPolicies returns a copy of the normalized policies for limited
|
||||||
|
// backends.
|
||||||
|
func (r *Registry) CapacityPolicies() map[string]domain.BackendCapacityPolicy {
|
||||||
|
policies := make(map[string]domain.BackendCapacityPolicy)
|
||||||
|
if r == nil {
|
||||||
|
return policies
|
||||||
|
}
|
||||||
|
for id, definition := range r.backends {
|
||||||
|
if definition.ConcurrencyLimit == 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
policies[id] = domain.BackendCapacityPolicy{
|
||||||
|
ConcurrencyLimit: definition.ConcurrencyLimit,
|
||||||
|
QueueCapacity: definition.QueueCapacity,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return policies
|
||||||
|
}
|
||||||
|
|
||||||
|
func normalizeBackend(definition domain.Backend) (domain.Backend, error) {
|
||||||
|
endpoint, err := domain.NormalizeOpenAICompatibleBaseEndpoint(definition.Endpoint)
|
||||||
|
if err != nil {
|
||||||
|
return domain.Backend{}, fmt.Errorf("backend %q endpoint: %w", definition.ID, err)
|
||||||
|
}
|
||||||
|
definition.Endpoint = endpoint
|
||||||
|
|
||||||
|
definition.APIKeyEnv = strings.TrimSpace(definition.APIKeyEnv)
|
||||||
|
if definition.APIKeyEnv != "" && !environmentVariableName.MatchString(definition.APIKeyEnv) {
|
||||||
|
return domain.Backend{}, fmt.Errorf(
|
||||||
|
"backend %q api key environment variable %q is invalid",
|
||||||
|
definition.ID,
|
||||||
|
definition.APIKeyEnv,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
if definition.ConcurrencyLimit < 0 {
|
||||||
|
return domain.Backend{}, fmt.Errorf(
|
||||||
|
"backend %q concurrency limit must not be negative",
|
||||||
|
definition.ID,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
if definition.QueueCapacity < 0 {
|
||||||
|
return domain.Backend{}, fmt.Errorf(
|
||||||
|
"backend %q queue capacity must not be negative",
|
||||||
|
definition.ID,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
if definition.ConcurrencyLimit == 0 {
|
||||||
|
if definition.QueueCapacitySet {
|
||||||
|
return domain.Backend{}, fmt.Errorf(
|
||||||
|
"backend %q queue capacity requires a positive concurrency limit",
|
||||||
|
definition.ID,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
definition.QueueCapacity = 0
|
||||||
|
} else {
|
||||||
|
if !definition.QueueCapacitySet {
|
||||||
|
definition.QueueCapacity = defaultQueueCapacity
|
||||||
|
definition.QueueCapacitySet = true
|
||||||
|
}
|
||||||
|
maxInt := int(^uint(0) >> 1)
|
||||||
|
if definition.QueueCapacity > maxInt-definition.ConcurrencyLimit {
|
||||||
|
return domain.Backend{}, fmt.Errorf(
|
||||||
|
"backend %q total capacity overflows int",
|
||||||
|
definition.ID,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
keys := make([]string, 0, len(definition.ExtraParams))
|
||||||
|
for key := range definition.ExtraParams {
|
||||||
|
keys = append(keys, key)
|
||||||
|
}
|
||||||
|
sort.Strings(keys)
|
||||||
|
for _, key := range keys {
|
||||||
|
if key == "" {
|
||||||
|
return domain.Backend{}, fmt.Errorf("backend %q extra parameter key must not be empty", definition.ID)
|
||||||
|
}
|
||||||
|
if llm.IsReservedOpenAIChatRequestField(key) {
|
||||||
|
return domain.Backend{}, fmt.Errorf(
|
||||||
|
"backend %q extra parameter %q collides with a reserved request field",
|
||||||
|
definition.ID,
|
||||||
|
key,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
extraParams, err := jsonvalue.CopyMap(definition.ExtraParams)
|
||||||
|
if err != nil {
|
||||||
|
return domain.Backend{}, fmt.Errorf("backend %q extra parameters: %w", definition.ID, err)
|
||||||
|
}
|
||||||
|
definition.ExtraParams = extraParams
|
||||||
|
return definition, nil
|
||||||
|
}
|
||||||
368
internal/backend/registry_test.go
Normal file
368
internal/backend/registry_test.go
Normal file
@@ -0,0 +1,368 @@
|
|||||||
|
package backend_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/backend"
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||||
|
)
|
||||||
|
|
||||||
|
const validEndpoint = "https://backend.example/v1"
|
||||||
|
|
||||||
|
func TestRegistryIncludesExactOpenRouterDefinition(t *testing.T) {
|
||||||
|
registry, err := backend.NewRegistry(nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("construct registry: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
definition, err := registry.GetBackend(backend.OpenRouterID)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("look up OpenRouter: %v", err)
|
||||||
|
}
|
||||||
|
if definition.ID != "openrouter" ||
|
||||||
|
definition.Endpoint != "https://openrouter.ai/api/v1" ||
|
||||||
|
definition.APIKeyEnv != "OPENROUTER_API_KEY" ||
|
||||||
|
definition.ConcurrencyLimit != 16 ||
|
||||||
|
definition.QueueCapacity != 1024 ||
|
||||||
|
!definition.QueueCapacitySet ||
|
||||||
|
definition.ExtraParams != nil {
|
||||||
|
t.Fatalf("unexpected OpenRouter definition: %#v", definition)
|
||||||
|
}
|
||||||
|
policies := registry.CapacityPolicies()
|
||||||
|
if len(policies) != 1 ||
|
||||||
|
policies["openrouter"] != (domain.BackendCapacityPolicy{
|
||||||
|
ConcurrencyLimit: 16,
|
||||||
|
QueueCapacity: 1024,
|
||||||
|
}) {
|
||||||
|
t.Fatalf("unexpected OpenRouter capacity policies: %#v", policies)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRegistryNormalizesUniqueAdditionsAndIsolatesMutations(t *testing.T) {
|
||||||
|
nested := map[string]int{"limit": 2}
|
||||||
|
extraParams := map[string]any{
|
||||||
|
"count": int64(7),
|
||||||
|
"nested": nested,
|
||||||
|
}
|
||||||
|
registry, err := backend.NewRegistry([]domain.Backend{
|
||||||
|
{
|
||||||
|
ID: " custom ",
|
||||||
|
Endpoint: " https://custom.example/openai/v1 ",
|
||||||
|
APIKeyEnv: " CUSTOM_API_KEY ",
|
||||||
|
ExtraParams: extraParams,
|
||||||
|
ConcurrencyLimit: 3,
|
||||||
|
QueueCapacity: 2,
|
||||||
|
QueueCapacitySet: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
ID: "Custom",
|
||||||
|
Endpoint: validEndpoint,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("construct registry: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
nested["limit"] = 99
|
||||||
|
extraParams["added"] = true
|
||||||
|
|
||||||
|
got, err := registry.GetBackend("custom")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("look up custom backend: %v", err)
|
||||||
|
}
|
||||||
|
if got.ID != "custom" ||
|
||||||
|
got.Endpoint != "https://custom.example/openai/v1" ||
|
||||||
|
got.APIKeyEnv != "CUSTOM_API_KEY" ||
|
||||||
|
got.ConcurrencyLimit != 3 ||
|
||||||
|
got.QueueCapacity != 2 ||
|
||||||
|
!got.QueueCapacitySet {
|
||||||
|
t.Fatalf("unexpected normalized definition: %#v", got)
|
||||||
|
}
|
||||||
|
if count, ok := got.ExtraParams["count"].(int64); !ok || count != 7 {
|
||||||
|
t.Fatalf("integer type or value changed: %#v", got.ExtraParams["count"])
|
||||||
|
}
|
||||||
|
gotNested, ok := got.ExtraParams["nested"].(map[string]int)
|
||||||
|
if !ok || gotNested["limit"] != 2 {
|
||||||
|
t.Fatalf("container type or value changed: %#v", got.ExtraParams["nested"])
|
||||||
|
}
|
||||||
|
if _, exists := got.ExtraParams["added"]; exists {
|
||||||
|
t.Fatalf("registry retained caller map: %#v", got.ExtraParams)
|
||||||
|
}
|
||||||
|
|
||||||
|
gotNested["limit"] = 100
|
||||||
|
got.ExtraParams["added"] = true
|
||||||
|
again, err := registry.GetBackend("custom")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("look up custom backend again: %v", err)
|
||||||
|
}
|
||||||
|
if again.ExtraParams["nested"].(map[string]int)["limit"] != 2 {
|
||||||
|
t.Fatalf("lookup exposed registry nested map: %#v", again.ExtraParams)
|
||||||
|
}
|
||||||
|
if _, exists := again.ExtraParams["added"]; exists {
|
||||||
|
t.Fatalf("lookup exposed registry map: %#v", again.ExtraParams)
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, err := registry.GetBackend("Custom"); err != nil {
|
||||||
|
t.Fatalf("backend IDs should be case-sensitive: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
policies := registry.CapacityPolicies()
|
||||||
|
if len(policies) != 2 {
|
||||||
|
t.Fatalf("unexpected capacity policy count: %#v", policies)
|
||||||
|
}
|
||||||
|
policies["custom"] = domain.BackendCapacityPolicy{}
|
||||||
|
delete(policies, backend.OpenRouterID)
|
||||||
|
againPolicies := registry.CapacityPolicies()
|
||||||
|
if againPolicies["custom"] != (domain.BackendCapacityPolicy{
|
||||||
|
ConcurrencyLimit: 3,
|
||||||
|
QueueCapacity: 2,
|
||||||
|
}) {
|
||||||
|
t.Fatalf("capacity policy map mutated registry state: %#v", againPolicies)
|
||||||
|
}
|
||||||
|
if _, ok := againPolicies[backend.OpenRouterID]; !ok {
|
||||||
|
t.Fatalf("capacity policy deletion mutated registry state: %#v", againPolicies)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNewRegistryNormalizesCapacityPolicy(t *testing.T) {
|
||||||
|
maxInt := int(^uint(0) >> 1)
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
definition domain.Backend
|
||||||
|
want domain.BackendCapacityPolicy
|
||||||
|
wantSet bool
|
||||||
|
wantError bool
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "unlimited when omitted",
|
||||||
|
definition: domain.Backend{},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "default queue",
|
||||||
|
definition: domain.Backend{
|
||||||
|
ConcurrencyLimit: 2,
|
||||||
|
},
|
||||||
|
want: domain.BackendCapacityPolicy{
|
||||||
|
ConcurrencyLimit: 2,
|
||||||
|
QueueCapacity: 1024,
|
||||||
|
},
|
||||||
|
wantSet: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "explicit zero queue",
|
||||||
|
definition: domain.Backend{
|
||||||
|
ConcurrencyLimit: 2,
|
||||||
|
QueueCapacitySet: true,
|
||||||
|
},
|
||||||
|
want: domain.BackendCapacityPolicy{
|
||||||
|
ConcurrencyLimit: 2,
|
||||||
|
},
|
||||||
|
wantSet: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "negative concurrency limit",
|
||||||
|
definition: domain.Backend{
|
||||||
|
ConcurrencyLimit: -1,
|
||||||
|
},
|
||||||
|
wantError: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "negative queue capacity",
|
||||||
|
definition: domain.Backend{
|
||||||
|
ConcurrencyLimit: 1,
|
||||||
|
QueueCapacity: -1,
|
||||||
|
QueueCapacitySet: true,
|
||||||
|
},
|
||||||
|
wantError: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "queue without limit",
|
||||||
|
definition: domain.Backend{
|
||||||
|
QueueCapacitySet: true,
|
||||||
|
},
|
||||||
|
wantError: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "total overflow",
|
||||||
|
definition: domain.Backend{
|
||||||
|
ConcurrencyLimit: maxInt,
|
||||||
|
QueueCapacity: 1,
|
||||||
|
QueueCapacitySet: true,
|
||||||
|
},
|
||||||
|
wantError: true,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
tc.definition.ID = "custom"
|
||||||
|
tc.definition.Endpoint = validEndpoint
|
||||||
|
registry, err := backend.NewRegistry([]domain.Backend{tc.definition})
|
||||||
|
if tc.wantError {
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected invalid capacity policy error")
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("construct registry: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
definition, err := registry.GetBackend("custom")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("look up custom backend: %v", err)
|
||||||
|
}
|
||||||
|
if definition.ConcurrencyLimit != tc.want.ConcurrencyLimit ||
|
||||||
|
definition.QueueCapacity != tc.want.QueueCapacity ||
|
||||||
|
definition.QueueCapacitySet != tc.wantSet {
|
||||||
|
t.Fatalf("normalized capacity=(%d, %d, %t), want (%d, %d, %t)",
|
||||||
|
definition.ConcurrencyLimit,
|
||||||
|
definition.QueueCapacity,
|
||||||
|
definition.QueueCapacitySet,
|
||||||
|
tc.want.ConcurrencyLimit,
|
||||||
|
tc.want.QueueCapacity,
|
||||||
|
tc.wantSet,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
policies := registry.CapacityPolicies()
|
||||||
|
got, ok := policies["custom"]
|
||||||
|
if ok != tc.wantSet || got != tc.want {
|
||||||
|
t.Fatalf("capacity policy=(%#v, %t), want (%#v, %t)", got, ok, tc.want, tc.wantSet)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNewRegistryRejectsDuplicateIDs(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
additions []domain.Backend
|
||||||
|
wantID string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "built-in collision after normalization",
|
||||||
|
additions: []domain.Backend{{
|
||||||
|
ID: " openrouter ",
|
||||||
|
}},
|
||||||
|
wantID: "openrouter",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "consumer collision after normalization",
|
||||||
|
additions: []domain.Backend{
|
||||||
|
{ID: "custom", Endpoint: validEndpoint},
|
||||||
|
{ID: " custom ", Endpoint: "https://other.example/v1"},
|
||||||
|
},
|
||||||
|
wantID: "custom",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
_, err := backend.NewRegistry(tc.additions)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected duplicate ID error")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), tc.wantID) {
|
||||||
|
t.Fatalf("expected error to identify %q, got %v", tc.wantID, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNewRegistryValidatesIDs(t *testing.T) {
|
||||||
|
for _, id := range []string{"", " \t\n "} {
|
||||||
|
t.Run(id, func(t *testing.T) {
|
||||||
|
_, err := backend.NewRegistry([]domain.Backend{{
|
||||||
|
ID: id,
|
||||||
|
Endpoint: validEndpoint,
|
||||||
|
}})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected blank ID error")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNewRegistryValidatesEndpoints(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
endpoint string
|
||||||
|
}{
|
||||||
|
{name: "blank", endpoint: ""},
|
||||||
|
{name: "relative", endpoint: "/v1"},
|
||||||
|
{name: "missing host", endpoint: "https:///v1"},
|
||||||
|
{name: "unsupported scheme", endpoint: "ftp://backend.example/v1"},
|
||||||
|
{name: "user information", endpoint: "https://user@backend.example/v1"},
|
||||||
|
{name: "query", endpoint: "https://backend.example/v1?mode=chat"},
|
||||||
|
{name: "empty query", endpoint: "https://backend.example/v1?"},
|
||||||
|
{name: "fragment", endpoint: "https://backend.example/v1#chat"},
|
||||||
|
{name: "empty fragment", endpoint: "https://backend.example/v1#"},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
_, err := backend.NewRegistry([]domain.Backend{{
|
||||||
|
ID: "custom",
|
||||||
|
Endpoint: tc.endpoint,
|
||||||
|
}})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected invalid endpoint error")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNewRegistryValidatesEnvironmentVariableNames(t *testing.T) {
|
||||||
|
for _, name := range []string{"1API_KEY", "API-KEY", "API KEY", "ÅPI_KEY"} {
|
||||||
|
t.Run(name, func(t *testing.T) {
|
||||||
|
_, err := backend.NewRegistry([]domain.Backend{{
|
||||||
|
ID: "custom",
|
||||||
|
Endpoint: validEndpoint,
|
||||||
|
APIKeyEnv: name,
|
||||||
|
}})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected invalid environment-variable name error")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNewRegistryRejectsInvalidAndReservedExtraParameters(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
extraParams map[string]any
|
||||||
|
}{
|
||||||
|
{name: "unsupported value", extraParams: map[string]any{"value": make(chan int)}},
|
||||||
|
{name: "reserved key", extraParams: map[string]any{"model": "override"}},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
_, err := backend.NewRegistry([]domain.Backend{{
|
||||||
|
ID: "custom",
|
||||||
|
Endpoint: validEndpoint,
|
||||||
|
ExtraParams: tc.extraParams,
|
||||||
|
}})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected invalid extra parameters error")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRegistryLookupReportsNotFound(t *testing.T) {
|
||||||
|
registry, err := backend.NewRegistry(nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("construct registry: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = registry.GetBackend("missing")
|
||||||
|
if !errors.Is(err, backend.ErrBackendNotFound) {
|
||||||
|
t.Fatalf("expected ErrBackendNotFound, got %v", err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "missing") {
|
||||||
|
t.Fatalf("expected error to identify backend, got %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
40
internal/capacity/client.go
Normal file
40
internal/capacity/client.go
Normal file
@@ -0,0 +1,40 @@
|
|||||||
|
package capacity
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/llm"
|
||||||
|
)
|
||||||
|
|
||||||
|
type client struct {
|
||||||
|
manager *Manager
|
||||||
|
next llm.Client
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewClient wraps next with configured active-generation limits. A nil manager
|
||||||
|
// leaves next unchanged.
|
||||||
|
func NewClient(manager *Manager, next llm.Client) llm.Client {
|
||||||
|
if manager == nil {
|
||||||
|
return next
|
||||||
|
}
|
||||||
|
return &client{
|
||||||
|
manager: manager,
|
||||||
|
next: next,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *client) Generate(
|
||||||
|
ctx context.Context,
|
||||||
|
req domain.GenerateRequest,
|
||||||
|
) (*domain.GenerateResponse, error) {
|
||||||
|
pool := c.manager.getPool(req.Target.BackendID)
|
||||||
|
if pool == nil {
|
||||||
|
return c.next.Generate(ctx, req)
|
||||||
|
}
|
||||||
|
if err := pool.acquire(ctx); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer pool.releaseActive()
|
||||||
|
return c.next.Generate(ctx, req)
|
||||||
|
}
|
||||||
517
internal/capacity/client_test.go
Normal file
517
internal/capacity/client_test.go
Normal file
@@ -0,0 +1,517 @@
|
|||||||
|
package capacity
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"reflect"
|
||||||
|
"runtime"
|
||||||
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/llm"
|
||||||
|
)
|
||||||
|
|
||||||
|
type generateResult struct {
|
||||||
|
response *domain.GenerateResponse
|
||||||
|
err error
|
||||||
|
}
|
||||||
|
|
||||||
|
type clientFunc func(
|
||||||
|
context.Context,
|
||||||
|
domain.GenerateRequest,
|
||||||
|
) (*domain.GenerateResponse, error)
|
||||||
|
|
||||||
|
func (f clientFunc) Generate(
|
||||||
|
ctx context.Context,
|
||||||
|
req domain.GenerateRequest,
|
||||||
|
) (*domain.GenerateResponse, error) {
|
||||||
|
return f(ctx, req)
|
||||||
|
}
|
||||||
|
|
||||||
|
type blockingClient struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
active int
|
||||||
|
peak int
|
||||||
|
calls map[string]int
|
||||||
|
started chan string
|
||||||
|
releases map[string]chan struct{}
|
||||||
|
}
|
||||||
|
|
||||||
|
func newBlockingClient(releases map[string]chan struct{}) *blockingClient {
|
||||||
|
return &blockingClient{
|
||||||
|
calls: make(map[string]int),
|
||||||
|
started: make(chan string, 64),
|
||||||
|
releases: releases,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *blockingClient) Generate(
|
||||||
|
ctx context.Context,
|
||||||
|
req domain.GenerateRequest,
|
||||||
|
) (*domain.GenerateResponse, error) {
|
||||||
|
id := req.Prompt.SessionID
|
||||||
|
c.mu.Lock()
|
||||||
|
c.active++
|
||||||
|
if c.active > c.peak {
|
||||||
|
c.peak = c.active
|
||||||
|
}
|
||||||
|
c.calls[id]++
|
||||||
|
c.mu.Unlock()
|
||||||
|
defer func() {
|
||||||
|
c.mu.Lock()
|
||||||
|
c.active--
|
||||||
|
c.mu.Unlock()
|
||||||
|
}()
|
||||||
|
|
||||||
|
c.started <- id
|
||||||
|
if release := c.releases[id]; release != nil {
|
||||||
|
select {
|
||||||
|
case <-release:
|
||||||
|
case <-ctx.Done():
|
||||||
|
return nil, ctx.Err()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return &domain.GenerateResponse{Content: id}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *blockingClient) callCount(id string) int {
|
||||||
|
c.mu.Lock()
|
||||||
|
defer c.mu.Unlock()
|
||||||
|
return c.calls[id]
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *blockingClient) peakConcurrency() int {
|
||||||
|
c.mu.Lock()
|
||||||
|
defer c.mu.Unlock()
|
||||||
|
return c.peak
|
||||||
|
}
|
||||||
|
|
||||||
|
func generateAsync(
|
||||||
|
client llm.Client,
|
||||||
|
ctx context.Context,
|
||||||
|
backendID string,
|
||||||
|
id string,
|
||||||
|
) <-chan generateResult {
|
||||||
|
result := make(chan generateResult, 1)
|
||||||
|
go func() {
|
||||||
|
response, err := client.Generate(ctx, domain.GenerateRequest{
|
||||||
|
Prompt: domain.RenderedPrompt{SessionID: id},
|
||||||
|
Target: domain.ExecutionTarget{BackendID: backendID},
|
||||||
|
})
|
||||||
|
result <- generateResult{response: response, err: err}
|
||||||
|
}()
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
|
func waitForWaiterCount(t *testing.T, manager *Manager, backendID string, want int) {
|
||||||
|
t.Helper()
|
||||||
|
pool := manager.pools[backendID]
|
||||||
|
deadline := time.Now().Add(2 * time.Second)
|
||||||
|
for {
|
||||||
|
pool.mu.Lock()
|
||||||
|
got := pool.waiters.Len()
|
||||||
|
pool.mu.Unlock()
|
||||||
|
if got == want {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if time.Now().After(deadline) {
|
||||||
|
t.Fatalf("waiter count=%d, want %d", got, want)
|
||||||
|
}
|
||||||
|
runtime.Gosched()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func receiveStarted(t *testing.T, started <-chan string) string {
|
||||||
|
t.Helper()
|
||||||
|
select {
|
||||||
|
case id := <-started:
|
||||||
|
return id
|
||||||
|
case <-time.After(2 * time.Second):
|
||||||
|
t.Fatal("timed out waiting for wrapped client invocation")
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func receiveResult(t *testing.T, result <-chan generateResult) generateResult {
|
||||||
|
t.Helper()
|
||||||
|
select {
|
||||||
|
case got := <-result:
|
||||||
|
return got
|
||||||
|
case <-time.After(2 * time.Second):
|
||||||
|
t.Fatal("timed out waiting for generation result")
|
||||||
|
return generateResult{}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func newTestManager(t *testing.T, policies map[string]domain.BackendCapacityPolicy) *Manager {
|
||||||
|
t.Helper()
|
||||||
|
manager, err := NewManager(policies)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("construct manager: %v", err)
|
||||||
|
}
|
||||||
|
return manager
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestClientLimitsPeakConcurrencyAndServesWaitersFIFO(t *testing.T) {
|
||||||
|
manager := newTestManager(t, map[string]domain.BackendCapacityPolicy{
|
||||||
|
"limited": {ConcurrencyLimit: 1},
|
||||||
|
})
|
||||||
|
firstRelease := make(chan struct{})
|
||||||
|
secondRelease := make(chan struct{})
|
||||||
|
thirdRelease := make(chan struct{})
|
||||||
|
next := newBlockingClient(map[string]chan struct{}{
|
||||||
|
"first": firstRelease,
|
||||||
|
"second": secondRelease,
|
||||||
|
"third": thirdRelease,
|
||||||
|
})
|
||||||
|
client := NewClient(manager, next)
|
||||||
|
|
||||||
|
first := generateAsync(client, context.Background(), "limited", "first")
|
||||||
|
if got := receiveStarted(t, next.started); got != "first" {
|
||||||
|
t.Fatalf("first invocation=%q, want first", got)
|
||||||
|
}
|
||||||
|
second := generateAsync(client, context.Background(), "limited", "second")
|
||||||
|
waitForWaiterCount(t, manager, "limited", 1)
|
||||||
|
third := generateAsync(client, context.Background(), "limited", "third")
|
||||||
|
waitForWaiterCount(t, manager, "limited", 2)
|
||||||
|
|
||||||
|
close(firstRelease)
|
||||||
|
if got := receiveResult(t, first); got.err != nil {
|
||||||
|
t.Fatalf("first generation: %v", got.err)
|
||||||
|
}
|
||||||
|
if got := receiveStarted(t, next.started); got != "second" {
|
||||||
|
t.Fatalf("second invocation=%q, want second", got)
|
||||||
|
}
|
||||||
|
close(secondRelease)
|
||||||
|
if got := receiveResult(t, second); got.err != nil {
|
||||||
|
t.Fatalf("second generation: %v", got.err)
|
||||||
|
}
|
||||||
|
if got := receiveStarted(t, next.started); got != "third" {
|
||||||
|
t.Fatalf("third invocation=%q, want third", got)
|
||||||
|
}
|
||||||
|
close(thirdRelease)
|
||||||
|
if got := receiveResult(t, third); got.err != nil {
|
||||||
|
t.Fatalf("third generation: %v", got.err)
|
||||||
|
}
|
||||||
|
if peak := next.peakConcurrency(); peak != 1 {
|
||||||
|
t.Fatalf("peak concurrency=%d, want 1", peak)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestClientPeakConcurrencyDoesNotExceedConfiguredLimit(t *testing.T) {
|
||||||
|
const limit = 2
|
||||||
|
manager := newTestManager(t, map[string]domain.BackendCapacityPolicy{
|
||||||
|
"limited": {ConcurrencyLimit: limit},
|
||||||
|
})
|
||||||
|
gate := make(chan struct{})
|
||||||
|
releases := make(map[string]chan struct{})
|
||||||
|
for i := range 5 {
|
||||||
|
releases[string(rune('a'+i))] = gate
|
||||||
|
}
|
||||||
|
next := newBlockingClient(releases)
|
||||||
|
client := NewClient(manager, next)
|
||||||
|
|
||||||
|
results := make([]<-chan generateResult, 0, len(releases))
|
||||||
|
for id := range releases {
|
||||||
|
results = append(results, generateAsync(client, context.Background(), "limited", id))
|
||||||
|
}
|
||||||
|
for range limit {
|
||||||
|
receiveStarted(t, next.started)
|
||||||
|
}
|
||||||
|
waitForWaiterCount(t, manager, "limited", len(releases)-limit)
|
||||||
|
|
||||||
|
close(gate)
|
||||||
|
for _, result := range results {
|
||||||
|
if got := receiveResult(t, result); got.err != nil {
|
||||||
|
t.Fatalf("generation: %v", got.err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if peak := next.peakConcurrency(); peak != limit {
|
||||||
|
t.Fatalf("peak concurrency=%d, want %d", peak, limit)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestClientRemovesCanceledWaiters(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
cancelID string
|
||||||
|
wantOrder []string
|
||||||
|
}{
|
||||||
|
{name: "first waiter", cancelID: "one", wantOrder: []string{"two", "three"}},
|
||||||
|
{name: "middle waiter", cancelID: "two", wantOrder: []string{"one", "three"}},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
manager := newTestManager(t, map[string]domain.BackendCapacityPolicy{
|
||||||
|
"limited": {ConcurrencyLimit: 1},
|
||||||
|
})
|
||||||
|
holderRelease := make(chan struct{})
|
||||||
|
releases := map[string]chan struct{}{
|
||||||
|
"holder": holderRelease,
|
||||||
|
"one": make(chan struct{}),
|
||||||
|
"two": make(chan struct{}),
|
||||||
|
"three": make(chan struct{}),
|
||||||
|
}
|
||||||
|
next := newBlockingClient(releases)
|
||||||
|
client := NewClient(manager, next)
|
||||||
|
|
||||||
|
holder := generateAsync(client, context.Background(), "limited", "holder")
|
||||||
|
if got := receiveStarted(t, next.started); got != "holder" {
|
||||||
|
t.Fatalf("initial invocation=%q, want holder", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
contexts := make(map[string]context.Context)
|
||||||
|
cancels := make(map[string]context.CancelFunc)
|
||||||
|
results := make(map[string]<-chan generateResult)
|
||||||
|
for _, id := range []string{"one", "two", "three"} {
|
||||||
|
contexts[id], cancels[id] = context.WithCancel(context.Background())
|
||||||
|
results[id] = generateAsync(client, contexts[id], "limited", id)
|
||||||
|
waitForWaiterCount(t, manager, "limited", len(results))
|
||||||
|
}
|
||||||
|
|
||||||
|
cancels[tc.cancelID]()
|
||||||
|
if got := receiveResult(t, results[tc.cancelID]); !errors.Is(got.err, context.Canceled) {
|
||||||
|
t.Fatalf("canceled waiter error=%v, want context.Canceled", got.err)
|
||||||
|
}
|
||||||
|
waitForWaiterCount(t, manager, "limited", 2)
|
||||||
|
|
||||||
|
close(holderRelease)
|
||||||
|
if got := receiveResult(t, holder); got.err != nil {
|
||||||
|
t.Fatalf("holder generation: %v", got.err)
|
||||||
|
}
|
||||||
|
for _, id := range tc.wantOrder {
|
||||||
|
if got := receiveStarted(t, next.started); got != id {
|
||||||
|
t.Fatalf("next invocation=%q, want %q", got, id)
|
||||||
|
}
|
||||||
|
close(releases[id])
|
||||||
|
if got := receiveResult(t, results[id]); got.err != nil {
|
||||||
|
t.Fatalf("%s generation: %v", id, got.err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if calls := next.callCount(tc.cancelID); calls != 0 {
|
||||||
|
t.Fatalf("canceled waiter invoked wrapped client %d times", calls)
|
||||||
|
}
|
||||||
|
for _, cancel := range cancels {
|
||||||
|
cancel()
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestClientGrantCancellationRaceDoesNotLeakPermit(t *testing.T) {
|
||||||
|
const iterations = 200
|
||||||
|
for i := range iterations {
|
||||||
|
manager := newTestManager(t, map[string]domain.BackendCapacityPolicy{
|
||||||
|
"limited": {ConcurrencyLimit: 1},
|
||||||
|
})
|
||||||
|
holderRelease := make(chan struct{})
|
||||||
|
var waiterCalls atomic.Int64
|
||||||
|
next := clientFunc(func(
|
||||||
|
_ context.Context,
|
||||||
|
req domain.GenerateRequest,
|
||||||
|
) (*domain.GenerateResponse, error) {
|
||||||
|
if req.Prompt.SessionID == "holder" {
|
||||||
|
<-holderRelease
|
||||||
|
} else if req.Prompt.SessionID == "waiter" {
|
||||||
|
waiterCalls.Add(1)
|
||||||
|
}
|
||||||
|
return &domain.GenerateResponse{Content: req.Prompt.SessionID}, nil
|
||||||
|
})
|
||||||
|
client := NewClient(manager, next)
|
||||||
|
|
||||||
|
holder := generateAsync(client, context.Background(), "limited", "holder")
|
||||||
|
waitForActiveCount(t, manager, "limited", 1)
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
waiterResult := generateAsync(client, ctx, "limited", "waiter")
|
||||||
|
waitForWaiterCount(t, manager, "limited", 1)
|
||||||
|
|
||||||
|
start := make(chan struct{})
|
||||||
|
var race sync.WaitGroup
|
||||||
|
race.Add(2)
|
||||||
|
go func() {
|
||||||
|
defer race.Done()
|
||||||
|
<-start
|
||||||
|
cancel()
|
||||||
|
}()
|
||||||
|
go func() {
|
||||||
|
defer race.Done()
|
||||||
|
<-start
|
||||||
|
close(holderRelease)
|
||||||
|
}()
|
||||||
|
close(start)
|
||||||
|
race.Wait()
|
||||||
|
|
||||||
|
if got := receiveResult(t, holder); got.err != nil {
|
||||||
|
t.Fatalf("iteration %d holder generation: %v", i, got.err)
|
||||||
|
}
|
||||||
|
got := receiveResult(t, waiterResult)
|
||||||
|
switch calls := waiterCalls.Load(); {
|
||||||
|
case calls == 0 && errors.Is(got.err, context.Canceled):
|
||||||
|
case calls == 1 && got.err == nil:
|
||||||
|
default:
|
||||||
|
t.Fatalf("iteration %d waiter calls=%d error=%v", i, calls, got.err)
|
||||||
|
}
|
||||||
|
|
||||||
|
probe := generateAsync(client, context.Background(), "limited", "probe")
|
||||||
|
if got := receiveResult(t, probe); got.err != nil {
|
||||||
|
t.Fatalf("iteration %d probe generation: %v", i, got.err)
|
||||||
|
}
|
||||||
|
waitForActiveCount(t, manager, "limited", 0)
|
||||||
|
waitForWaiterCount(t, manager, "limited", 0)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func waitForActiveCount(t *testing.T, manager *Manager, backendID string, want int) {
|
||||||
|
t.Helper()
|
||||||
|
pool := manager.pools[backendID]
|
||||||
|
deadline := time.Now().Add(2 * time.Second)
|
||||||
|
for {
|
||||||
|
pool.mu.Lock()
|
||||||
|
got := pool.active
|
||||||
|
pool.mu.Unlock()
|
||||||
|
if got == want {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if time.Now().After(deadline) {
|
||||||
|
t.Fatalf("active count=%d, want %d", got, want)
|
||||||
|
}
|
||||||
|
runtime.Gosched()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestClientUsesIndependentPoolsAndUnlimitedFastPaths(t *testing.T) {
|
||||||
|
manager := newTestManager(t, map[string]domain.BackendCapacityPolicy{
|
||||||
|
"alpha": {ConcurrencyLimit: 1},
|
||||||
|
"beta": {ConcurrencyLimit: 1},
|
||||||
|
})
|
||||||
|
alphaRelease := make(chan struct{})
|
||||||
|
betaRelease := make(chan struct{})
|
||||||
|
next := newBlockingClient(map[string]chan struct{}{
|
||||||
|
"alpha": alphaRelease,
|
||||||
|
"beta": betaRelease,
|
||||||
|
})
|
||||||
|
client := NewClient(manager, next)
|
||||||
|
|
||||||
|
alpha := generateAsync(client, context.Background(), "alpha", "alpha")
|
||||||
|
beta := generateAsync(client, context.Background(), "beta", "beta")
|
||||||
|
started := map[string]bool{
|
||||||
|
receiveStarted(t, next.started): true,
|
||||||
|
receiveStarted(t, next.started): true,
|
||||||
|
}
|
||||||
|
if !started["alpha"] || !started["beta"] {
|
||||||
|
t.Fatalf("independent pools did not both start: %#v", started)
|
||||||
|
}
|
||||||
|
close(alphaRelease)
|
||||||
|
close(betaRelease)
|
||||||
|
if got := receiveResult(t, alpha); got.err != nil {
|
||||||
|
t.Fatalf("alpha generation: %v", got.err)
|
||||||
|
}
|
||||||
|
if got := receiveResult(t, beta); got.err != nil {
|
||||||
|
t.Fatalf("beta generation: %v", got.err)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, backendID := range []string{"", "unknown"} {
|
||||||
|
response, err := client.Generate(context.Background(), domain.GenerateRequest{
|
||||||
|
Prompt: domain.RenderedPrompt{SessionID: backendID},
|
||||||
|
Target: domain.ExecutionTarget{BackendID: backendID},
|
||||||
|
})
|
||||||
|
if err != nil || response == nil {
|
||||||
|
t.Fatalf("unlimited backend %q response=(%#v, %v)", backendID, response, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if got := NewClient(nil, next); got != next {
|
||||||
|
t.Fatal("nil manager did not return the wrapped client unchanged")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestClientPreservesRequestsResponsesAndErrors(t *testing.T) {
|
||||||
|
manager := newTestManager(t, map[string]domain.BackendCapacityPolicy{
|
||||||
|
"limited": {ConcurrencyLimit: 1},
|
||||||
|
})
|
||||||
|
request := domain.GenerateRequest{
|
||||||
|
Prompt: domain.RenderedPrompt{
|
||||||
|
SessionID: "session",
|
||||||
|
Messages: []domain.RenderedMessage{
|
||||||
|
{Role: "user", Content: "content"},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
Target: domain.ExecutionTarget{
|
||||||
|
BackendID: "limited",
|
||||||
|
Model: "model",
|
||||||
|
ExtraParams: map[string]any{"key": "value"},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
response := &domain.GenerateResponse{
|
||||||
|
Content: "output",
|
||||||
|
Usage: domain.TokenUsage{TotalTokens: 7},
|
||||||
|
}
|
||||||
|
collaboratorErr := errors.New("collaborator failure")
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
response *domain.GenerateResponse
|
||||||
|
err error
|
||||||
|
}{
|
||||||
|
{name: "successful response", response: response},
|
||||||
|
{name: "nil response"},
|
||||||
|
{name: "collaborator error", response: response, err: collaboratorErr},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
var captured domain.GenerateRequest
|
||||||
|
next := clientFunc(func(
|
||||||
|
_ context.Context,
|
||||||
|
req domain.GenerateRequest,
|
||||||
|
) (*domain.GenerateResponse, error) {
|
||||||
|
captured = req
|
||||||
|
return tc.response, tc.err
|
||||||
|
})
|
||||||
|
gotResponse, gotErr := NewClient(manager, next).Generate(context.Background(), request)
|
||||||
|
if !reflect.DeepEqual(captured, request) {
|
||||||
|
t.Fatalf("request changed: %#v", captured)
|
||||||
|
}
|
||||||
|
if gotResponse != tc.response || gotErr != tc.err {
|
||||||
|
t.Fatalf("response=(%p, %v), want (%p, %v)",
|
||||||
|
gotResponse, gotErr, tc.response, tc.err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestClientReleasesPermitDuringPanicUnwinding(t *testing.T) {
|
||||||
|
manager := newTestManager(t, map[string]domain.BackendCapacityPolicy{
|
||||||
|
"limited": {ConcurrencyLimit: 1},
|
||||||
|
})
|
||||||
|
var calls atomic.Int64
|
||||||
|
next := clientFunc(func(
|
||||||
|
_ context.Context,
|
||||||
|
_ domain.GenerateRequest,
|
||||||
|
) (*domain.GenerateResponse, error) {
|
||||||
|
if calls.Add(1) == 1 {
|
||||||
|
panic("test panic")
|
||||||
|
}
|
||||||
|
return &domain.GenerateResponse{Content: "recovered"}, nil
|
||||||
|
})
|
||||||
|
client := NewClient(manager, next)
|
||||||
|
request := domain.GenerateRequest{
|
||||||
|
Target: domain.ExecutionTarget{BackendID: "limited"},
|
||||||
|
}
|
||||||
|
|
||||||
|
func() {
|
||||||
|
defer func() {
|
||||||
|
if recover() == nil {
|
||||||
|
t.Fatal("expected wrapped client panic")
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
_, _ = client.Generate(context.Background(), request)
|
||||||
|
}()
|
||||||
|
|
||||||
|
response, err := client.Generate(context.Background(), request)
|
||||||
|
if err != nil || response == nil || response.Content != "recovered" {
|
||||||
|
t.Fatalf("generation after panic=(%#v, %v)", response, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
160
internal/capacity/manager.go
Normal file
160
internal/capacity/manager.go
Normal file
@@ -0,0 +1,160 @@
|
|||||||
|
// Package capacity coordinates engine-local run admission and model-generation
|
||||||
|
// concurrency for configured backends.
|
||||||
|
package capacity
|
||||||
|
|
||||||
|
import (
|
||||||
|
"container/list"
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ErrCapacityExceeded identifies an admission rejected because a backend's
|
||||||
|
// configured run capacity is full.
|
||||||
|
var ErrCapacityExceeded = errors.New("backend capacity exceeded")
|
||||||
|
|
||||||
|
// Manager owns independent backend capacity pools with immutable limits.
|
||||||
|
type Manager struct {
|
||||||
|
pools map[string]*pool
|
||||||
|
}
|
||||||
|
|
||||||
|
type pool struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
concurrencyLimit int
|
||||||
|
totalCapacity int
|
||||||
|
admitted int
|
||||||
|
active int
|
||||||
|
waiters list.List
|
||||||
|
}
|
||||||
|
|
||||||
|
type waiter struct {
|
||||||
|
ready chan struct{}
|
||||||
|
element *list.Element
|
||||||
|
granted bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewManager constructs independent pools from normalized backend policies.
|
||||||
|
func NewManager(policies map[string]domain.BackendCapacityPolicy) (*Manager, error) {
|
||||||
|
manager := &Manager{
|
||||||
|
pools: make(map[string]*pool, len(policies)),
|
||||||
|
}
|
||||||
|
maxInt := int(^uint(0) >> 1)
|
||||||
|
for id, policy := range policies {
|
||||||
|
if strings.TrimSpace(id) == "" {
|
||||||
|
return nil, errors.New("backend capacity policy ID must not be blank")
|
||||||
|
}
|
||||||
|
if policy.ConcurrencyLimit <= 0 {
|
||||||
|
return nil, fmt.Errorf(
|
||||||
|
"backend %q concurrency limit must be positive",
|
||||||
|
id,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
if policy.QueueCapacity < 0 {
|
||||||
|
return nil, fmt.Errorf(
|
||||||
|
"backend %q queue capacity must not be negative",
|
||||||
|
id,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
if policy.QueueCapacity > maxInt-policy.ConcurrencyLimit {
|
||||||
|
return nil, fmt.Errorf("backend %q total capacity overflows int", id)
|
||||||
|
}
|
||||||
|
manager.pools[id] = &pool{
|
||||||
|
concurrencyLimit: policy.ConcurrencyLimit,
|
||||||
|
totalCapacity: policy.ConcurrencyLimit + policy.QueueCapacity,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return manager, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Admit immediately reserves one configured backend run slot. Backends without
|
||||||
|
// a configured pool are unlimited.
|
||||||
|
func (m *Manager) Admit(ctx context.Context, backendID string) (func(), error) {
|
||||||
|
pool := m.getPool(backendID)
|
||||||
|
if pool == nil {
|
||||||
|
return releaseNothing, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
pool.mu.Lock()
|
||||||
|
defer pool.mu.Unlock()
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if pool.admitted >= pool.totalCapacity {
|
||||||
|
return nil, ErrCapacityExceeded
|
||||||
|
}
|
||||||
|
pool.admitted++
|
||||||
|
|
||||||
|
var once sync.Once
|
||||||
|
return func() {
|
||||||
|
once.Do(func() {
|
||||||
|
pool.mu.Lock()
|
||||||
|
pool.admitted--
|
||||||
|
pool.mu.Unlock()
|
||||||
|
})
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func releaseNothing() {}
|
||||||
|
|
||||||
|
func (m *Manager) getPool(backendID string) *pool {
|
||||||
|
if m == nil || backendID == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return m.pools[backendID]
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *pool) acquire(ctx context.Context) error {
|
||||||
|
p.mu.Lock()
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
p.mu.Unlock()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if p.active < p.concurrencyLimit && p.waiters.Len() == 0 {
|
||||||
|
p.active++
|
||||||
|
p.mu.Unlock()
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
waiter := &waiter{ready: make(chan struct{})}
|
||||||
|
waiter.element = p.waiters.PushBack(waiter)
|
||||||
|
p.mu.Unlock()
|
||||||
|
|
||||||
|
select {
|
||||||
|
case <-waiter.ready:
|
||||||
|
return nil
|
||||||
|
case <-ctx.Done():
|
||||||
|
p.mu.Lock()
|
||||||
|
if !waiter.granted {
|
||||||
|
p.waiters.Remove(waiter.element)
|
||||||
|
waiter.element = nil
|
||||||
|
p.mu.Unlock()
|
||||||
|
return ctx.Err()
|
||||||
|
}
|
||||||
|
p.mu.Unlock()
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *pool) releaseActive() {
|
||||||
|
var ready chan struct{}
|
||||||
|
|
||||||
|
p.mu.Lock()
|
||||||
|
if element := p.waiters.Front(); element != nil {
|
||||||
|
waiter := element.Value.(*waiter)
|
||||||
|
p.waiters.Remove(element)
|
||||||
|
waiter.element = nil
|
||||||
|
waiter.granted = true
|
||||||
|
ready = waiter.ready
|
||||||
|
} else {
|
||||||
|
p.active--
|
||||||
|
}
|
||||||
|
p.mu.Unlock()
|
||||||
|
|
||||||
|
if ready != nil {
|
||||||
|
close(ready)
|
||||||
|
}
|
||||||
|
}
|
||||||
163
internal/capacity/manager_test.go
Normal file
163
internal/capacity/manager_test.go
Normal file
@@ -0,0 +1,163 @@
|
|||||||
|
package capacity
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestNewManagerRejectsInvalidPolicies(t *testing.T) {
|
||||||
|
maxInt := int(^uint(0) >> 1)
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
id string
|
||||||
|
policy domain.BackendCapacityPolicy
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "blank ID",
|
||||||
|
id: " \t ",
|
||||||
|
policy: domain.BackendCapacityPolicy{ConcurrencyLimit: 1},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "zero concurrency",
|
||||||
|
id: "backend",
|
||||||
|
policy: domain.BackendCapacityPolicy{},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "negative concurrency",
|
||||||
|
id: "backend",
|
||||||
|
policy: domain.BackendCapacityPolicy{ConcurrencyLimit: -1},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "negative queue",
|
||||||
|
id: "backend",
|
||||||
|
policy: domain.BackendCapacityPolicy{
|
||||||
|
ConcurrencyLimit: 1,
|
||||||
|
QueueCapacity: -1,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "total overflow",
|
||||||
|
id: "backend",
|
||||||
|
policy: domain.BackendCapacityPolicy{
|
||||||
|
ConcurrencyLimit: maxInt,
|
||||||
|
QueueCapacity: 1,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
_, err := NewManager(map[string]domain.BackendCapacityPolicy{
|
||||||
|
tc.id: tc.policy,
|
||||||
|
})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected invalid policy error")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestManagerAdmissionIsBoundedAndReleaseIsIdempotent(t *testing.T) {
|
||||||
|
policies := map[string]domain.BackendCapacityPolicy{
|
||||||
|
"limited": {
|
||||||
|
ConcurrencyLimit: 2,
|
||||||
|
QueueCapacity: 1,
|
||||||
|
},
|
||||||
|
"independent": {
|
||||||
|
ConcurrencyLimit: 1,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
manager, err := NewManager(policies)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("construct manager: %v", err)
|
||||||
|
}
|
||||||
|
policies["limited"] = domain.BackendCapacityPolicy{
|
||||||
|
ConcurrencyLimit: 100,
|
||||||
|
QueueCapacity: 100,
|
||||||
|
}
|
||||||
|
|
||||||
|
releases := make([]func(), 0, 3)
|
||||||
|
for range 3 {
|
||||||
|
release, err := manager.Admit(context.Background(), "limited")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("admit within configured capacity: %v", err)
|
||||||
|
}
|
||||||
|
releases = append(releases, release)
|
||||||
|
}
|
||||||
|
if release, err := manager.Admit(context.Background(), "limited"); release != nil ||
|
||||||
|
!errors.Is(err, ErrCapacityExceeded) {
|
||||||
|
t.Fatalf("admission beyond capacity=(release=%t, err=%v), want ErrCapacityExceeded",
|
||||||
|
release != nil, err)
|
||||||
|
}
|
||||||
|
independentRelease, err := manager.Admit(context.Background(), "independent")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("admit independent backend while first is full: %v", err)
|
||||||
|
}
|
||||||
|
independentRelease()
|
||||||
|
|
||||||
|
releases[0]()
|
||||||
|
releases[0]()
|
||||||
|
replacement, err := manager.Admit(context.Background(), "limited")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("admit after release: %v", err)
|
||||||
|
}
|
||||||
|
replacement()
|
||||||
|
releases[1]()
|
||||||
|
releases[2]()
|
||||||
|
|
||||||
|
pool := manager.pools["limited"]
|
||||||
|
pool.mu.Lock()
|
||||||
|
admitted := pool.admitted
|
||||||
|
pool.mu.Unlock()
|
||||||
|
if admitted != 0 {
|
||||||
|
t.Fatalf("admitted runs after releases=%d, want 0", admitted)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestManagerAdmissionHonorsContextAndUnlimitedBackends(t *testing.T) {
|
||||||
|
manager, err := NewManager(map[string]domain.BackendCapacityPolicy{
|
||||||
|
"limited": {ConcurrencyLimit: 1},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("construct manager: %v", err)
|
||||||
|
}
|
||||||
|
release, err := manager.Admit(context.Background(), "limited")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("fill limited pool: %v", err)
|
||||||
|
}
|
||||||
|
defer release()
|
||||||
|
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
cancel()
|
||||||
|
if release, err := manager.Admit(ctx, "limited"); release != nil ||
|
||||||
|
!errors.Is(err, context.Canceled) {
|
||||||
|
t.Fatalf("canceled limited admission=(release=%t, err=%v), want context cancellation",
|
||||||
|
release != nil, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var nilManager *Manager
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
manager *Manager
|
||||||
|
backendID string
|
||||||
|
}{
|
||||||
|
{name: "nil manager", manager: nilManager, backendID: "limited"},
|
||||||
|
{name: "blank ID", manager: manager},
|
||||||
|
{name: "unknown ID", manager: manager, backendID: "unknown"},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
release, err := tc.manager.Admit(ctx, tc.backendID)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("unlimited admission: %v", err)
|
||||||
|
}
|
||||||
|
if release == nil {
|
||||||
|
t.Fatal("unlimited admission returned nil release")
|
||||||
|
}
|
||||||
|
release()
|
||||||
|
release()
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -14,21 +14,12 @@ const (
|
|||||||
ContentTypeApplicationJSON = "application/json"
|
ContentTypeApplicationJSON = "application/json"
|
||||||
OpenAIChatCompletionsPath = "/chat/completions"
|
OpenAIChatCompletionsPath = "/chat/completions"
|
||||||
|
|
||||||
ExecutionDefaultTemperature = 0.0
|
|
||||||
ExecutionDefaultMaxTokens = 0
|
|
||||||
ExecutionDefaultTopP = 1.0
|
|
||||||
ExecutionDefaultTimeoutSeconds = 600
|
ExecutionDefaultTimeoutSeconds = 600
|
||||||
)
|
LLMRequestTimeoutDefault = 10 * time.Minute
|
||||||
|
|
||||||
var (
|
|
||||||
LLMRequestTimeoutDefault = 10 * time.Minute
|
|
||||||
)
|
)
|
||||||
|
|
||||||
func ExecutionTargetDefault() domain.ExecutionTarget {
|
func ExecutionTargetDefault() domain.ExecutionTarget {
|
||||||
return domain.ExecutionTarget{
|
return domain.ExecutionTarget{
|
||||||
Temperature: ExecutionDefaultTemperature,
|
|
||||||
MaxTokens: ExecutionDefaultMaxTokens,
|
|
||||||
TopP: ExecutionDefaultTopP,
|
|
||||||
TimeoutSeconds: ExecutionDefaultTimeoutSeconds,
|
TimeoutSeconds: ExecutionDefaultTimeoutSeconds,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -63,12 +63,12 @@ type RunRequest struct {
|
|||||||
PromptID string
|
PromptID string
|
||||||
PromptVersion string
|
PromptVersion string
|
||||||
ProfileID string
|
ProfileID string
|
||||||
|
SessionID string
|
||||||
APIKey string `json:"-" yaml:"-"`
|
APIKey string `json:"-" yaml:"-"`
|
||||||
Inputs map[string]ArtifactRef
|
Inputs map[string]ArtifactRef
|
||||||
Vars map[string]string
|
Vars map[string]string
|
||||||
Execution *ExecutionTargetOverride
|
Execution *ExecutionTargetOverride
|
||||||
Validation *OutputContract
|
Validation *OutputContract
|
||||||
Metadata map[string]string
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// RunResult represents the complete result of a prompt execution run.
|
// RunResult represents the complete result of a prompt execution run.
|
||||||
@@ -80,8 +80,10 @@ type RunResult struct {
|
|||||||
PromptID string
|
PromptID string
|
||||||
PromptVersion string
|
PromptVersion string
|
||||||
PromptHash string
|
PromptHash string
|
||||||
|
SessionID string
|
||||||
RenderedPromptHash string
|
RenderedPromptHash string
|
||||||
SelectedProfileID string
|
SelectedProfileID string
|
||||||
|
SelectedBackendID string
|
||||||
ModelName string
|
ModelName string
|
||||||
Endpoint string
|
Endpoint string
|
||||||
EffectiveModelParams ExecutionTarget
|
EffectiveModelParams ExecutionTarget
|
||||||
@@ -95,21 +97,22 @@ type RunResult struct {
|
|||||||
// PreparedRun contains pre-LLM execution state from the prepare/render phase.
|
// PreparedRun contains pre-LLM execution state from the prepare/render phase.
|
||||||
// It must never include resolved API key values, model output, or validation data.
|
// It must never include resolved API key values, model output, or validation data.
|
||||||
type PreparedRun struct {
|
type PreparedRun struct {
|
||||||
PromptID string `json:"prompt_id"`
|
PromptID string
|
||||||
PromptVersion string `json:"prompt_version,omitempty"`
|
PromptVersion string
|
||||||
PromptHash string `json:"prompt_hash,omitempty"`
|
PromptHash string
|
||||||
SelectedProfileID string `json:"selected_profile_id"`
|
SelectedProfileID string
|
||||||
EffectiveModelParams ExecutionTarget `json:"effective_model_params"`
|
SelectedBackendID string
|
||||||
TargetPresence ExecutionTargetPresence `json:"-"`
|
EffectiveModelParams ExecutionTarget
|
||||||
OutputContract OutputContract `json:"output_contract"`
|
TargetPresence ExecutionTargetPresence
|
||||||
StructuredOutput *StructuredOutputSpec `json:"structured_output,omitempty"`
|
OutputContract OutputContract
|
||||||
InputHashes map[string]string `json:"input_hashes,omitempty"`
|
StructuredOutput *StructuredOutputSpec
|
||||||
SessionID string `json:"session_id,omitempty"`
|
InputHashes map[string]string
|
||||||
RenderedPromptHash string `json:"rendered_prompt_hash"`
|
SessionID string
|
||||||
Messages []RenderedMessage `json:"messages"`
|
RenderedPromptHash string
|
||||||
StartTime time.Time `json:"start_time,omitempty"`
|
Messages []RenderedMessage
|
||||||
EndTime time.Time `json:"end_time,omitempty"`
|
StartTime time.Time
|
||||||
DurationMS int64 `json:"duration_ms,omitempty"`
|
EndTime time.Time
|
||||||
|
DurationMS int64
|
||||||
}
|
}
|
||||||
|
|
||||||
// ArtifactRef represents a reference to an input artifact.
|
// ArtifactRef represents a reference to an input artifact.
|
||||||
@@ -142,6 +145,16 @@ type PromptDefinition struct {
|
|||||||
Validation OutputContract `yaml:"validation"`
|
Validation OutputContract `yaml:"validation"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// PromptInspection is the resolved result of exact prompt inspection.
|
||||||
|
type PromptInspection struct {
|
||||||
|
PromptID string
|
||||||
|
PromptVersion string
|
||||||
|
PromptHash string
|
||||||
|
DefaultProfileID string
|
||||||
|
Inputs []PromptInput
|
||||||
|
OutputContract OutputContract
|
||||||
|
}
|
||||||
|
|
||||||
// PromptInput describes one named input expected by a prompt definition.
|
// PromptInput describes one named input expected by a prompt definition.
|
||||||
type PromptInput struct {
|
type PromptInput struct {
|
||||||
Name string `yaml:"name"`
|
Name string `yaml:"name"`
|
||||||
@@ -158,9 +171,28 @@ type PromptMessageTemplate struct {
|
|||||||
CacheControl *CacheControl `yaml:"cache_control,omitempty" json:"cache_control,omitempty"`
|
CacheControl *CacheControl `yaml:"cache_control,omitempty" json:"cache_control,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Backend describes reusable OpenAI-compatible connection defaults.
|
||||||
|
type Backend struct {
|
||||||
|
ID string
|
||||||
|
Endpoint string
|
||||||
|
APIKeyEnv string
|
||||||
|
ExtraParams map[string]any
|
||||||
|
ConcurrencyLimit int
|
||||||
|
QueueCapacity int
|
||||||
|
QueueCapacitySet bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// BackendCapacityPolicy describes normalized run and generation capacity for
|
||||||
|
// one limited backend.
|
||||||
|
type BackendCapacityPolicy struct {
|
||||||
|
ConcurrencyLimit int
|
||||||
|
QueueCapacity int
|
||||||
|
}
|
||||||
|
|
||||||
// ExecutionProfile describes how and where to execute a model.
|
// ExecutionProfile describes how and where to execute a model.
|
||||||
type ExecutionProfile struct {
|
type ExecutionProfile struct {
|
||||||
ID string `yaml:"id"`
|
ID string `yaml:"id"`
|
||||||
|
BackendID string `yaml:"backend"`
|
||||||
Endpoint string `yaml:"endpoint"`
|
Endpoint string `yaml:"endpoint"`
|
||||||
Model string `yaml:"model"`
|
Model string `yaml:"model"`
|
||||||
Temperature float64 `yaml:"temperature"`
|
Temperature float64 `yaml:"temperature"`
|
||||||
@@ -183,7 +215,7 @@ type ExecutionTargetOverride struct {
|
|||||||
TopP *float64 `json:"top_p,omitempty"`
|
TopP *float64 `json:"top_p,omitempty"`
|
||||||
TimeoutSeconds *int `json:"timeout_seconds,omitempty"`
|
TimeoutSeconds *int `json:"timeout_seconds,omitempty"`
|
||||||
ServiceTier string `json:"service_tier,omitempty"`
|
ServiceTier string `json:"service_tier,omitempty"`
|
||||||
ReasoningEffort string `json:"reasoning_effort,omitempty"`
|
ReasoningEffort *string `json:"reasoning_effort,omitempty"`
|
||||||
APIKeyEnv string `json:"api_key_env,omitempty"`
|
APIKeyEnv string `json:"api_key_env,omitempty"`
|
||||||
ExtraParams map[string]any `json:"extra_params,omitempty"`
|
ExtraParams map[string]any `json:"extra_params,omitempty"`
|
||||||
}
|
}
|
||||||
@@ -199,6 +231,7 @@ type ExecutionTargetPresence struct {
|
|||||||
|
|
||||||
// ExecutionTarget represents effective model runtime settings for a run.
|
// ExecutionTarget represents effective model runtime settings for a run.
|
||||||
type ExecutionTarget struct {
|
type ExecutionTarget struct {
|
||||||
|
BackendID string `yaml:"backend" json:"backend_id,omitempty"`
|
||||||
Endpoint string `yaml:"endpoint" json:"endpoint"`
|
Endpoint string `yaml:"endpoint" json:"endpoint"`
|
||||||
Model string `yaml:"model" json:"model"`
|
Model string `yaml:"model" json:"model"`
|
||||||
Temperature float64 `yaml:"temperature" json:"temperature"`
|
Temperature float64 `yaml:"temperature" json:"temperature"`
|
||||||
@@ -213,6 +246,13 @@ type ExecutionTarget struct {
|
|||||||
ExtraParams map[string]any `yaml:"extra_params" json:"extra_params"`
|
ExtraParams map[string]any `yaml:"extra_params" json:"extra_params"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ProfileInspection is the resolved result of exact profile inspection.
|
||||||
|
type ProfileInspection struct {
|
||||||
|
ProfileID string
|
||||||
|
EffectiveModelParams ExecutionTarget
|
||||||
|
APIKeyRequired bool
|
||||||
|
}
|
||||||
|
|
||||||
// OutputContract defines the requirements for the output artifact.
|
// OutputContract defines the requirements for the output artifact.
|
||||||
type OutputContract struct {
|
type OutputContract struct {
|
||||||
Format OutputFormat `yaml:"format"`
|
Format OutputFormat `yaml:"format"`
|
||||||
|
|||||||
39
internal/domain/endpoint.go
Normal file
39
internal/domain/endpoint.go
Normal file
@@ -0,0 +1,39 @@
|
|||||||
|
package domain
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"net/url"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// NormalizeOpenAICompatibleBaseEndpoint trims and validates a source-neutral
|
||||||
|
// OpenAI-compatible provider base endpoint.
|
||||||
|
func NormalizeOpenAICompatibleBaseEndpoint(endpoint string) (string, error) {
|
||||||
|
endpoint = strings.TrimSpace(endpoint)
|
||||||
|
if endpoint == "" {
|
||||||
|
return "", errors.New("endpoint must not be blank")
|
||||||
|
}
|
||||||
|
if strings.Contains(endpoint, "#") {
|
||||||
|
return "", errors.New("endpoint must not contain a fragment")
|
||||||
|
}
|
||||||
|
|
||||||
|
parsed, err := url.Parse(endpoint)
|
||||||
|
if err != nil {
|
||||||
|
return "", errors.New("endpoint must be a valid URL")
|
||||||
|
}
|
||||||
|
parsed.Scheme = strings.ToLower(parsed.Scheme)
|
||||||
|
if parsed.Scheme != "http" && parsed.Scheme != "https" {
|
||||||
|
return "", errors.New("endpoint must use http or https")
|
||||||
|
}
|
||||||
|
if !parsed.IsAbs() || parsed.Hostname() == "" {
|
||||||
|
return "", errors.New("endpoint must be absolute and include a host")
|
||||||
|
}
|
||||||
|
if parsed.User != nil {
|
||||||
|
return "", errors.New("endpoint must not contain user information")
|
||||||
|
}
|
||||||
|
if parsed.RawQuery != "" || parsed.ForceQuery {
|
||||||
|
return "", errors.New("endpoint must not contain a query string")
|
||||||
|
}
|
||||||
|
|
||||||
|
return parsed.String(), nil
|
||||||
|
}
|
||||||
47
internal/domain/endpoint_test.go
Normal file
47
internal/domain/endpoint_test.go
Normal file
@@ -0,0 +1,47 @@
|
|||||||
|
package domain
|
||||||
|
|
||||||
|
import "testing"
|
||||||
|
|
||||||
|
func TestNormalizeOpenAICompatibleBaseEndpoint(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
endpoint string
|
||||||
|
want string
|
||||||
|
wantErr bool
|
||||||
|
}{
|
||||||
|
{name: "http host", endpoint: "http://provider.example", want: "http://provider.example"},
|
||||||
|
{name: "https nested path and whitespace", endpoint: " HTTPS://provider.example/api/openai/v1 ", want: "https://provider.example/api/openai/v1"},
|
||||||
|
{name: "IPv4 host and port", endpoint: "http://127.0.0.1:8080/v1", want: "http://127.0.0.1:8080/v1"},
|
||||||
|
{name: "IPv6 host and port", endpoint: "https://[::1]:8443/v1", want: "https://[::1]:8443/v1"},
|
||||||
|
{name: "repeated trailing slashes", endpoint: "https://provider.example/v1///", want: "https://provider.example/v1///"},
|
||||||
|
{name: "blank", endpoint: " \t\n ", wantErr: true},
|
||||||
|
{name: "relative path", endpoint: "/api/v1", wantErr: true},
|
||||||
|
{name: "scheme relative", endpoint: "//provider.example/v1", wantErr: true},
|
||||||
|
{name: "missing host", endpoint: "https:///v1", wantErr: true},
|
||||||
|
{name: "unsupported scheme", endpoint: "ftp://provider.example/v1", wantErr: true},
|
||||||
|
{name: "user information", endpoint: "https://user:secret@provider.example/v1", wantErr: true},
|
||||||
|
{name: "query", endpoint: "https://provider.example/v1?mode=chat", wantErr: true},
|
||||||
|
{name: "empty query", endpoint: "https://provider.example/v1?", wantErr: true},
|
||||||
|
{name: "fragment", endpoint: "https://provider.example/v1#chat", wantErr: true},
|
||||||
|
{name: "empty fragment", endpoint: "https://provider.example/v1#", wantErr: true},
|
||||||
|
{name: "malformed URL", endpoint: "https://provider.example/%zz", wantErr: true},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
got, err := NormalizeOpenAICompatibleBaseEndpoint(tc.endpoint)
|
||||||
|
if tc.wantErr {
|
||||||
|
if err == nil {
|
||||||
|
t.Fatalf("expected endpoint error, got %q", got)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("normalize endpoint: %v", err)
|
||||||
|
}
|
||||||
|
if got != tc.want {
|
||||||
|
t.Fatalf("normalized endpoint = %q, want %q", got, tc.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
34
internal/domain/execution_settings.go
Normal file
34
internal/domain/execution_settings.go
Normal file
@@ -0,0 +1,34 @@
|
|||||||
|
package domain
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"math"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
const maxExecutionTimeoutSeconds int64 = math.MaxInt64 / int64(time.Second)
|
||||||
|
|
||||||
|
// ValidateExecutionTargetSettings validates source-neutral execution-setting
|
||||||
|
// invariants on a resolved target.
|
||||||
|
func ValidateExecutionTargetSettings(target ExecutionTarget) error {
|
||||||
|
if !isFinite(target.Temperature) || target.Temperature < 0 || target.Temperature > 2 {
|
||||||
|
return errors.New("temperature must be finite and between 0 and 2")
|
||||||
|
}
|
||||||
|
if target.MaxTokens < 0 {
|
||||||
|
return errors.New("max_tokens must be greater than or equal to 0")
|
||||||
|
}
|
||||||
|
if !isFinite(target.TopP) || target.TopP < 0 || target.TopP > 1 {
|
||||||
|
return errors.New("top_p must be finite and between 0 and 1")
|
||||||
|
}
|
||||||
|
if target.TimeoutSeconds < 0 {
|
||||||
|
return errors.New("timeout_seconds must be greater than or equal to 0")
|
||||||
|
}
|
||||||
|
if int64(target.TimeoutSeconds) > maxExecutionTimeoutSeconds {
|
||||||
|
return errors.New("timeout_seconds exceeds the maximum supported duration")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func isFinite(value float64) bool {
|
||||||
|
return !math.IsNaN(value) && !math.IsInf(value, 0)
|
||||||
|
}
|
||||||
73
internal/domain/execution_settings_test.go
Normal file
73
internal/domain/execution_settings_test.go
Normal file
@@ -0,0 +1,73 @@
|
|||||||
|
package domain
|
||||||
|
|
||||||
|
import (
|
||||||
|
"math"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestValidateExecutionTargetSettings(t *testing.T) {
|
||||||
|
valid := ExecutionTarget{
|
||||||
|
Temperature: 1,
|
||||||
|
MaxTokens: 1,
|
||||||
|
TopP: 0.5,
|
||||||
|
TimeoutSeconds: 1,
|
||||||
|
}
|
||||||
|
|
||||||
|
type testCase struct {
|
||||||
|
name string
|
||||||
|
change func(*ExecutionTarget)
|
||||||
|
wantErr string
|
||||||
|
}
|
||||||
|
tests := []testCase{
|
||||||
|
{name: "temperature lower boundary", change: func(v *ExecutionTarget) { v.Temperature = 0 }},
|
||||||
|
{name: "temperature finite lower neighbor", change: func(v *ExecutionTarget) { v.Temperature = math.Nextafter(0, 1) }},
|
||||||
|
{name: "temperature finite upper neighbor", change: func(v *ExecutionTarget) { v.Temperature = math.Nextafter(2, 0) }},
|
||||||
|
{name: "temperature upper boundary", change: func(v *ExecutionTarget) { v.Temperature = 2 }},
|
||||||
|
{name: "temperature below lower boundary", change: func(v *ExecutionTarget) { v.Temperature = math.Nextafter(0, math.Inf(-1)) }, wantErr: "temperature"},
|
||||||
|
{name: "temperature above upper boundary", change: func(v *ExecutionTarget) { v.Temperature = math.Nextafter(2, math.Inf(1)) }, wantErr: "temperature"},
|
||||||
|
{name: "temperature NaN", change: func(v *ExecutionTarget) { v.Temperature = math.NaN() }, wantErr: "temperature"},
|
||||||
|
{name: "temperature positive infinity", change: func(v *ExecutionTarget) { v.Temperature = math.Inf(1) }, wantErr: "temperature"},
|
||||||
|
{name: "temperature negative infinity", change: func(v *ExecutionTarget) { v.Temperature = math.Inf(-1) }, wantErr: "temperature"},
|
||||||
|
{name: "max tokens lower boundary", change: func(v *ExecutionTarget) { v.MaxTokens = 0 }},
|
||||||
|
{name: "max tokens finite neighbor", change: func(v *ExecutionTarget) { v.MaxTokens = 1 }},
|
||||||
|
{name: "max tokens below lower boundary", change: func(v *ExecutionTarget) { v.MaxTokens = -1 }, wantErr: "max_tokens"},
|
||||||
|
{name: "top p lower boundary", change: func(v *ExecutionTarget) { v.TopP = 0 }},
|
||||||
|
{name: "top p finite lower neighbor", change: func(v *ExecutionTarget) { v.TopP = math.Nextafter(0, 1) }},
|
||||||
|
{name: "top p finite upper neighbor", change: func(v *ExecutionTarget) { v.TopP = math.Nextafter(1, 0) }},
|
||||||
|
{name: "top p upper boundary", change: func(v *ExecutionTarget) { v.TopP = 1 }},
|
||||||
|
{name: "top p below lower boundary", change: func(v *ExecutionTarget) { v.TopP = math.Nextafter(0, math.Inf(-1)) }, wantErr: "top_p"},
|
||||||
|
{name: "top p above upper boundary", change: func(v *ExecutionTarget) { v.TopP = math.Nextafter(1, math.Inf(1)) }, wantErr: "top_p"},
|
||||||
|
{name: "top p NaN", change: func(v *ExecutionTarget) { v.TopP = math.NaN() }, wantErr: "top_p"},
|
||||||
|
{name: "top p positive infinity", change: func(v *ExecutionTarget) { v.TopP = math.Inf(1) }, wantErr: "top_p"},
|
||||||
|
{name: "top p negative infinity", change: func(v *ExecutionTarget) { v.TopP = math.Inf(-1) }, wantErr: "top_p"},
|
||||||
|
{name: "timeout lower boundary", change: func(v *ExecutionTarget) { v.TimeoutSeconds = 0 }},
|
||||||
|
{name: "timeout finite neighbor", change: func(v *ExecutionTarget) { v.TimeoutSeconds = 1 }},
|
||||||
|
{name: "timeout below lower boundary", change: func(v *ExecutionTarget) { v.TimeoutSeconds = -1 }, wantErr: "timeout_seconds"},
|
||||||
|
}
|
||||||
|
if strconv.IntSize == 64 {
|
||||||
|
durationLimit := maxExecutionTimeoutSeconds
|
||||||
|
tests = append(tests,
|
||||||
|
testCase{name: "timeout duration boundary", change: func(v *ExecutionTarget) { v.TimeoutSeconds = int(durationLimit) }},
|
||||||
|
testCase{name: "timeout above duration boundary", change: func(v *ExecutionTarget) { v.TimeoutSeconds = int(durationLimit) + 1 }, wantErr: "timeout_seconds"},
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
target := valid
|
||||||
|
tt.change(&target)
|
||||||
|
err := ValidateExecutionTargetSettings(target)
|
||||||
|
if tt.wantErr == "" {
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("validate execution settings: %v", err)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err == nil || !strings.Contains(err.Error(), tt.wantErr) {
|
||||||
|
t.Fatalf("error = %v, want diagnostic containing %q", err, tt.wantErr)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
30
internal/domain/output_contract.go
Normal file
30
internal/domain/output_contract.go
Normal file
@@ -0,0 +1,30 @@
|
|||||||
|
package domain
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ValidateOutputContract validates source-neutral output-contract invariants.
|
||||||
|
func ValidateOutputContract(contract OutputContract) error {
|
||||||
|
switch contract.Format {
|
||||||
|
case FormatText, FormatMarkdown, FormatJSON:
|
||||||
|
default:
|
||||||
|
return fmt.Errorf("invalid output format: %q", contract.Format)
|
||||||
|
}
|
||||||
|
|
||||||
|
switch contract.ValidationMode {
|
||||||
|
case ValidationNone, ValidationBasic, ValidationJSON, ValidationJSONSchema:
|
||||||
|
default:
|
||||||
|
return fmt.Errorf("invalid validation mode: %q", contract.ValidationMode)
|
||||||
|
}
|
||||||
|
|
||||||
|
if contract.ValidationMode == ValidationJSONSchema && strings.TrimSpace(contract.SchemaPath) == "" {
|
||||||
|
return errors.New("schema_path is required when validation_mode is json_schema")
|
||||||
|
}
|
||||||
|
if contract.RepairAttempts < 0 {
|
||||||
|
return errors.New("repair_attempts must be greater than or equal to 0")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
68
internal/domain/output_contract_test.go
Normal file
68
internal/domain/output_contract_test.go
Normal file
@@ -0,0 +1,68 @@
|
|||||||
|
package domain
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestValidateOutputContract(t *testing.T) {
|
||||||
|
valid := OutputContract{
|
||||||
|
Format: FormatText,
|
||||||
|
ValidationMode: ValidationNone,
|
||||||
|
}
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
change func(*OutputContract)
|
||||||
|
wantErr string
|
||||||
|
}{
|
||||||
|
{name: "text format", change: func(c *OutputContract) { c.Format = FormatText }},
|
||||||
|
{name: "markdown format", change: func(c *OutputContract) { c.Format = FormatMarkdown }},
|
||||||
|
{name: "json format", change: func(c *OutputContract) { c.Format = FormatJSON }},
|
||||||
|
{name: "empty format", change: func(c *OutputContract) { c.Format = "" }, wantErr: "format"},
|
||||||
|
{name: "unsupported format", change: func(c *OutputContract) { c.Format = OutputFormat("binary") }, wantErr: "format"},
|
||||||
|
{name: "none validation", change: func(c *OutputContract) { c.ValidationMode = ValidationNone }},
|
||||||
|
{name: "basic validation", change: func(c *OutputContract) { c.ValidationMode = ValidationBasic }},
|
||||||
|
{name: "json validation", change: func(c *OutputContract) { c.ValidationMode = ValidationJSON }},
|
||||||
|
{name: "json schema validation", change: func(c *OutputContract) {
|
||||||
|
c.ValidationMode = ValidationJSONSchema
|
||||||
|
c.SchemaPath = "schema.json"
|
||||||
|
}},
|
||||||
|
{name: "empty validation mode", change: func(c *OutputContract) { c.ValidationMode = "" }, wantErr: "validation mode"},
|
||||||
|
{name: "unsupported validation mode", change: func(c *OutputContract) { c.ValidationMode = ValidationMode("unknown") }, wantErr: "validation mode"},
|
||||||
|
{name: "negative repair attempts", change: func(c *OutputContract) { c.RepairAttempts = -1 }, wantErr: "repair_attempts"},
|
||||||
|
{name: "zero repair attempts", change: func(c *OutputContract) { c.RepairAttempts = 0 }},
|
||||||
|
{name: "positive repair attempts", change: func(c *OutputContract) { c.RepairAttempts = 1 }},
|
||||||
|
{name: "json schema empty path", change: func(c *OutputContract) {
|
||||||
|
c.ValidationMode = ValidationJSONSchema
|
||||||
|
c.SchemaPath = ""
|
||||||
|
}, wantErr: "schema_path"},
|
||||||
|
{name: "json schema whitespace path", change: func(c *OutputContract) {
|
||||||
|
c.ValidationMode = ValidationJSONSchema
|
||||||
|
c.SchemaPath = " \t "
|
||||||
|
}, wantErr: "schema_path"},
|
||||||
|
{name: "json schema nonblank path", change: func(c *OutputContract) {
|
||||||
|
c.ValidationMode = ValidationJSONSchema
|
||||||
|
c.SchemaPath = " schema.json "
|
||||||
|
}},
|
||||||
|
{name: "non-schema empty path", change: func(c *OutputContract) { c.SchemaPath = "" }},
|
||||||
|
{name: "non-schema populated path", change: func(c *OutputContract) { c.SchemaPath = "ignored.json" }},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
contract := valid
|
||||||
|
tt.change(&contract)
|
||||||
|
err := ValidateOutputContract(contract)
|
||||||
|
if tt.wantErr == "" {
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("validate output contract: %v", err)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err == nil || !strings.Contains(err.Error(), tt.wantErr) {
|
||||||
|
t.Fatalf("error = %v, want diagnostic containing %q", err, tt.wantErr)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,141 +0,0 @@
|
|||||||
package domain
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/json"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestPreparedRunJSONDoesNotIncludeSecretValues(t *testing.T) {
|
|
||||||
const envName = "PROMPTKIT_TEST_API_KEY"
|
|
||||||
const secret = "super-secret-value"
|
|
||||||
t.Setenv(envName, secret)
|
|
||||||
|
|
||||||
prepared := PreparedRun{
|
|
||||||
PromptID: "prompt.id",
|
|
||||||
PromptVersion: "v1",
|
|
||||||
PromptHash: "prompt-hash",
|
|
||||||
SelectedProfileID: "local-fast",
|
|
||||||
EffectiveModelParams: ExecutionTarget{
|
|
||||||
Endpoint: "http://llm/v1",
|
|
||||||
Model: "gpt-test",
|
|
||||||
APIKeyEnv: envName,
|
|
||||||
APIKey: secret,
|
|
||||||
},
|
|
||||||
InputHashes: map[string]string{"transcript": "hash-1"},
|
|
||||||
RenderedPromptHash: "rendered-hash",
|
|
||||||
Messages: []RenderedMessage{
|
|
||||||
{Role: "system", Content: "You are helpful."},
|
|
||||||
{Role: "user", Content: "Summarize this."},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
b, err := json.Marshal(prepared)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("marshal failed: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
out := string(b)
|
|
||||||
if strings.Contains(out, secret) {
|
|
||||||
t.Fatalf("prepared run JSON unexpectedly contains secret value: %s", out)
|
|
||||||
}
|
|
||||||
if !strings.Contains(out, `"api_key_env":"`+envName+`"`) {
|
|
||||||
t.Fatalf("prepared run JSON should include api_key_env name: %s", out)
|
|
||||||
}
|
|
||||||
|
|
||||||
var top map[string]any
|
|
||||||
if err := json.Unmarshal(b, &top); err != nil {
|
|
||||||
t.Fatalf("unmarshal failed: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, forbidden := range []string{"raw_output", "validation", "artifact"} {
|
|
||||||
if _, ok := top[forbidden]; ok {
|
|
||||||
t.Fatalf("prepared run JSON should not include %q", forbidden)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestPreparedRunJSONIncludesMessageCacheControlOnlyWhenPresent(t *testing.T) {
|
|
||||||
prepared := PreparedRun{
|
|
||||||
PromptID: "prompt.id",
|
|
||||||
SelectedProfileID: "local-fast",
|
|
||||||
EffectiveModelParams: ExecutionTarget{
|
|
||||||
Endpoint: "http://llm/v1",
|
|
||||||
Model: "gpt-test",
|
|
||||||
},
|
|
||||||
RenderedPromptHash: "rendered-hash",
|
|
||||||
Messages: []RenderedMessage{
|
|
||||||
{
|
|
||||||
Role: "system",
|
|
||||||
Content: "You are helpful.",
|
|
||||||
CacheControl: &CacheControl{
|
|
||||||
Type: CacheControlEphemeral,
|
|
||||||
TTL: "1h",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{Role: "user", Content: "Summarize this."},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
b, err := json.Marshal(prepared)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("marshal failed: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
var decoded struct {
|
|
||||||
Messages []map[string]any `json:"messages"`
|
|
||||||
}
|
|
||||||
if err := json.Unmarshal(b, &decoded); err != nil {
|
|
||||||
t.Fatalf("unmarshal failed: %v", err)
|
|
||||||
}
|
|
||||||
if len(decoded.Messages) != 2 {
|
|
||||||
t.Fatalf("expected 2 messages, got %d", len(decoded.Messages))
|
|
||||||
}
|
|
||||||
|
|
||||||
cacheControl, ok := decoded.Messages[0]["cache_control"].(map[string]any)
|
|
||||||
if !ok {
|
|
||||||
t.Fatalf("expected cache_control on first message, got %#v", decoded.Messages[0])
|
|
||||||
}
|
|
||||||
if cacheControl["type"] != string(CacheControlEphemeral) || cacheControl["ttl"] != "1h" {
|
|
||||||
t.Fatalf("unexpected cache_control payload: %#v", cacheControl)
|
|
||||||
}
|
|
||||||
if _, ok := decoded.Messages[1]["cache_control"]; ok {
|
|
||||||
t.Fatalf("expected second message to omit cache_control, got %#v", decoded.Messages[1])
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestPreparedRunJSONIncludesSessionIDOnlyWhenPresent(t *testing.T) {
|
|
||||||
prepared := PreparedRun{
|
|
||||||
PromptID: "prompt.id",
|
|
||||||
SelectedProfileID: "local-fast",
|
|
||||||
EffectiveModelParams: ExecutionTarget{
|
|
||||||
Endpoint: "http://llm/v1",
|
|
||||||
Model: "gpt-test",
|
|
||||||
},
|
|
||||||
SessionID: "session-123",
|
|
||||||
RenderedPromptHash: "rendered-hash",
|
|
||||||
Messages: []RenderedMessage{{Role: "user", Content: "Summarize this."}},
|
|
||||||
}
|
|
||||||
|
|
||||||
b, err := json.Marshal(prepared)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("marshal failed: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
var decoded map[string]any
|
|
||||||
if err := json.Unmarshal(b, &decoded); err != nil {
|
|
||||||
t.Fatalf("unmarshal failed: %v", err)
|
|
||||||
}
|
|
||||||
if decoded["session_id"] != "session-123" {
|
|
||||||
t.Fatalf("expected session_id in prepared run JSON, got %#v", decoded["session_id"])
|
|
||||||
}
|
|
||||||
|
|
||||||
prepared.SessionID = ""
|
|
||||||
b, err = json.Marshal(prepared)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("marshal failed: %v", err)
|
|
||||||
}
|
|
||||||
if strings.Contains(string(b), "session_id") {
|
|
||||||
t.Fatalf("expected empty session_id to be omitted, got %s", b)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
22
internal/domain/session.go
Normal file
22
internal/domain/session.go
Normal file
@@ -0,0 +1,22 @@
|
|||||||
|
package domain
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
"unicode/utf8"
|
||||||
|
)
|
||||||
|
|
||||||
|
// NormalizeSessionID applies the shared session identifier rule.
|
||||||
|
func NormalizeSessionID(raw string) (string, error) {
|
||||||
|
if !utf8.ValidString(raw) {
|
||||||
|
return "", fmt.Errorf("session_id must contain valid UTF-8")
|
||||||
|
}
|
||||||
|
normalized := strings.TrimSpace(raw)
|
||||||
|
if normalized == "" {
|
||||||
|
return "", nil
|
||||||
|
}
|
||||||
|
if length := utf8.RuneCountInString(normalized); length > SessionIDMaxLength {
|
||||||
|
return "", fmt.Errorf("session_id length %d exceeds maximum %d", length, SessionIDMaxLength)
|
||||||
|
}
|
||||||
|
return normalized, nil
|
||||||
|
}
|
||||||
60
internal/domain/session_test.go
Normal file
60
internal/domain/session_test.go
Normal file
@@ -0,0 +1,60 @@
|
|||||||
|
package domain
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestNormalizeSessionID(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
raw string
|
||||||
|
want string
|
||||||
|
wantErrContains string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "trims surrounding Unicode whitespace",
|
||||||
|
raw: "\u2003 session-123 \u2003",
|
||||||
|
want: "session-123",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "blank input is omitted",
|
||||||
|
raw: " \t\u2003 ",
|
||||||
|
want: "",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "maximum Unicode length is accepted",
|
||||||
|
raw: strings.Repeat("界", SessionIDMaxLength),
|
||||||
|
want: strings.Repeat("界", SessionIDMaxLength),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "one Unicode code point over maximum is rejected",
|
||||||
|
raw: strings.Repeat("界", SessionIDMaxLength+1),
|
||||||
|
wantErrContains: "exceeds maximum",
|
||||||
|
},
|
||||||
|
{name: "invalid UTF-8 before valid content", raw: string([]byte{0xff}) + "session", wantErrContains: "valid UTF-8"},
|
||||||
|
{name: "invalid UTF-8 within valid content", raw: "ses" + string([]byte{0xff}) + "sion", wantErrContains: "valid UTF-8"},
|
||||||
|
{name: "invalid UTF-8 after valid content", raw: "session" + string([]byte{0xff}), wantErrContains: "valid UTF-8"},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
got, err := NormalizeSessionID(tt.raw)
|
||||||
|
if tt.wantErrContains != "" {
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected normalization error")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), tt.wantErrContains) {
|
||||||
|
t.Fatalf("expected diagnostic containing %q, got %v", tt.wantErrContains, err)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("normalize session id: %v", err)
|
||||||
|
}
|
||||||
|
if got != tt.want {
|
||||||
|
t.Fatalf("normalized session id = %q, want %q", got, tt.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -36,7 +36,8 @@ func FindYAMLFiles(ctx context.Context, root string) ([]string, error) {
|
|||||||
return files, err
|
return files, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// FindFSYAMLFiles returns sorted paths for .yaml and .yml files under root in fsys.
|
// FindFSYAMLFiles returns root itself when it names a file. For a directory
|
||||||
|
// root, it returns sorted paths for .yaml and .yml files beneath that root.
|
||||||
func FindFSYAMLFiles(ctx context.Context, fsys fs.FS, root string) ([]string, error) {
|
func FindFSYAMLFiles(ctx context.Context, fsys fs.FS, root string) ([]string, error) {
|
||||||
cleanRoot := CleanFSRoot(root)
|
cleanRoot := CleanFSRoot(root)
|
||||||
var files []string
|
var files []string
|
||||||
@@ -52,6 +53,10 @@ func FindFSYAMLFiles(ctx context.Context, fsys fs.FS, root string) ([]string, er
|
|||||||
if d.IsDir() {
|
if d.IsDir() {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
if name == cleanRoot {
|
||||||
|
files = append(files, name)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
if !IsYAMLFile(d.Name()) {
|
if !IsYAMLFile(d.Name()) {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -71,10 +76,10 @@ func RelativePath(root string, filePath string) string {
|
|||||||
return filepath.Clean(rel)
|
return filepath.Clean(rel)
|
||||||
}
|
}
|
||||||
|
|
||||||
// CleanFSRoot normalizes a root path for use with fs.FS.
|
// CleanFSRoot normalizes a root path for use with fs.FS while preserving
|
||||||
|
// nonblank leading and trailing whitespace.
|
||||||
func CleanFSRoot(root string) string {
|
func CleanFSRoot(root string) string {
|
||||||
root = strings.TrimSpace(root)
|
if strings.TrimSpace(root) == "" || root == "." {
|
||||||
if root == "" || root == "." {
|
|
||||||
return "."
|
return "."
|
||||||
}
|
}
|
||||||
return path.Clean(root)
|
return path.Clean(root)
|
||||||
@@ -97,22 +102,21 @@ func DisplayPath(root string, name string) string {
|
|||||||
// ResolveFSPath resolves userPath from baseDir and keeps it inside root.
|
// ResolveFSPath resolves userPath from baseDir and keeps it inside root.
|
||||||
func ResolveFSPath(root string, baseDir string, userPath string) (string, string, error) {
|
func ResolveFSPath(root string, baseDir string, userPath string) (string, string, error) {
|
||||||
cleanRoot := CleanFSRoot(root)
|
cleanRoot := CleanFSRoot(root)
|
||||||
cleanBase := path.Clean(strings.TrimSpace(baseDir))
|
cleanBase := path.Clean(baseDir)
|
||||||
if cleanBase == "" {
|
if strings.TrimSpace(baseDir) == "" {
|
||||||
cleanBase = cleanRoot
|
cleanBase = cleanRoot
|
||||||
}
|
}
|
||||||
if !containsFSPath(cleanRoot, cleanBase) {
|
if !containsFSPath(cleanRoot, cleanBase) {
|
||||||
return "", "", fmt.Errorf("base path %q is outside source root %q", cleanBase, cleanRoot)
|
return "", "", fmt.Errorf("base path %q is outside source root %q", cleanBase, cleanRoot)
|
||||||
}
|
}
|
||||||
|
|
||||||
cleanUserPath := strings.TrimSpace(userPath)
|
if strings.TrimSpace(userPath) == "" {
|
||||||
if cleanUserPath == "" {
|
|
||||||
return "", "", fmt.Errorf("path is required")
|
return "", "", fmt.Errorf("path is required")
|
||||||
}
|
}
|
||||||
cleanUserPath = path.Clean(cleanUserPath)
|
if path.IsAbs(userPath) {
|
||||||
if path.IsAbs(cleanUserPath) {
|
|
||||||
return "", "", fmt.Errorf("path %q must be relative", userPath)
|
return "", "", fmt.Errorf("path %q must be relative", userPath)
|
||||||
}
|
}
|
||||||
|
cleanUserPath := path.Clean(userPath)
|
||||||
|
|
||||||
resolved := path.Clean(path.Join(cleanBase, cleanUserPath))
|
resolved := path.Clean(path.Join(cleanBase, cleanUserPath))
|
||||||
if !containsFSPath(cleanRoot, resolved) {
|
if !containsFSPath(cleanRoot, resolved) {
|
||||||
@@ -130,13 +134,6 @@ func containsFSPath(root string, name string) bool {
|
|||||||
return name == root || strings.HasPrefix(name, strings.TrimSuffix(root, "/")+"/")
|
return name == root || strings.HasPrefix(name, strings.TrimSuffix(root, "/")+"/")
|
||||||
}
|
}
|
||||||
|
|
||||||
// Stem strips .yaml or .yml from a file name.
|
|
||||||
func Stem(name string) string {
|
|
||||||
name = strings.TrimSuffix(name, ".yaml")
|
|
||||||
name = strings.TrimSuffix(name, ".yml")
|
|
||||||
return name
|
|
||||||
}
|
|
||||||
|
|
||||||
func IsYAMLFile(name string) bool {
|
func IsYAMLFile(name string) bool {
|
||||||
return strings.HasSuffix(name, ".yaml") || strings.HasSuffix(name, ".yml")
|
return strings.HasSuffix(name, ".yaml") || strings.HasSuffix(name, ".yml")
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -54,7 +54,7 @@ func TestFindFSYAMLFilesNestedSortedAndFiltered(t *testing.T) {
|
|||||||
"other/ignored.yaml": &fstest.MapFile{Data: []byte("id: ignored")},
|
"other/ignored.yaml": &fstest.MapFile{Data: []byte("id: ignored")},
|
||||||
}
|
}
|
||||||
|
|
||||||
got, err := FindFSYAMLFiles(context.Background(), fsys, " prompts ")
|
got, err := FindFSYAMLFiles(context.Background(), fsys, "prompts")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("expected no error, got %v", err)
|
t.Fatalf("expected no error, got %v", err)
|
||||||
}
|
}
|
||||||
@@ -98,8 +98,11 @@ func TestCleanFSRoot(t *testing.T) {
|
|||||||
want string
|
want string
|
||||||
}{
|
}{
|
||||||
{name: "empty", root: "", want: "."},
|
{name: "empty", root: "", want: "."},
|
||||||
|
{name: "whitespace only", root: " \t ", want: "."},
|
||||||
{name: "dot", root: ".", want: "."},
|
{name: "dot", root: ".", want: "."},
|
||||||
{name: "trimmed", root: " prompts/../profiles ", want: "profiles"},
|
{name: "cleaned", root: "prompts/../profiles", want: "profiles"},
|
||||||
|
{name: "leading whitespace preserved", root: " profiles", want: " profiles"},
|
||||||
|
{name: "trailing whitespace preserved", root: "profiles ", want: "profiles "},
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, tc := range tests {
|
for _, tc := range tests {
|
||||||
@@ -158,6 +161,22 @@ func TestResolveFSPath(t *testing.T) {
|
|||||||
wantPath: "prompts/shared/user.tmpl",
|
wantPath: "prompts/shared/user.tmpl",
|
||||||
wantDisplay: "shared/user.tmpl",
|
wantDisplay: "shared/user.tmpl",
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
name: "leading whitespace preserved",
|
||||||
|
root: "prompts",
|
||||||
|
baseDir: "prompts/nested",
|
||||||
|
userPath: " user.tmpl",
|
||||||
|
wantPath: "prompts/nested/ user.tmpl",
|
||||||
|
wantDisplay: "nested/ user.tmpl",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "trailing whitespace preserved",
|
||||||
|
root: "prompts",
|
||||||
|
baseDir: "prompts/nested",
|
||||||
|
userPath: "user.tmpl ",
|
||||||
|
wantPath: "prompts/nested/user.tmpl ",
|
||||||
|
wantDisplay: "nested/user.tmpl ",
|
||||||
|
},
|
||||||
{
|
{
|
||||||
name: "escape rejected",
|
name: "escape rejected",
|
||||||
root: "prompts",
|
root: "prompts",
|
||||||
@@ -218,26 +237,6 @@ func TestResolveFSPath(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestStemStripsYAMLExtensions(t *testing.T) {
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
in string
|
|
||||||
want string
|
|
||||||
}{
|
|
||||||
{name: "yaml", in: "prompt.yaml", want: "prompt"},
|
|
||||||
{name: "yml", in: "profile.yml", want: "profile"},
|
|
||||||
{name: "other", in: "file.txt", want: "file.txt"},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tc := range tests {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
if got := Stem(tc.in); got != tc.want {
|
|
||||||
t.Fatalf("expected %q, got %q", tc.want, got)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestIsYAMLFile(t *testing.T) {
|
func TestIsYAMLFile(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
|
|||||||
373
internal/jsonvalue/jsonvalue.go
Normal file
373
internal/jsonvalue/jsonvalue.go
Normal file
@@ -0,0 +1,373 @@
|
|||||||
|
// Package jsonvalue validates and defensively copies bounded JSON-compatible
|
||||||
|
// value trees used by configuration, request, and prepared-state boundaries.
|
||||||
|
package jsonvalue
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"math"
|
||||||
|
"reflect"
|
||||||
|
"sort"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
maxContainerDepth = 100
|
||||||
|
maxProducedNodes = 100_000
|
||||||
|
)
|
||||||
|
|
||||||
|
type visit struct {
|
||||||
|
typ reflect.Type
|
||||||
|
ptr uintptr
|
||||||
|
}
|
||||||
|
|
||||||
|
type traversalState struct {
|
||||||
|
active map[visit]struct{}
|
||||||
|
producedNodes int
|
||||||
|
}
|
||||||
|
|
||||||
|
// Copy validates and deeply copies a JSON-compatible value while preserving
|
||||||
|
// compatible concrete map, slice, array, scalar, and number types. It rejects
|
||||||
|
// cycles and values that exceed the package's traversal limits.
|
||||||
|
func Copy(src any) (any, error) {
|
||||||
|
return copyValue(reflect.ValueOf(src), "value", newTraversalState(), true, 0)
|
||||||
|
}
|
||||||
|
|
||||||
|
// CopyMap validates and deeply copies an extra-parameter map while preserving
|
||||||
|
// compatible concrete map, slice, array, scalar, and number types. It rejects
|
||||||
|
// empty object keys, cycles, and values that exceed the package's traversal
|
||||||
|
// limits.
|
||||||
|
func CopyMap(src map[string]any) (map[string]any, error) {
|
||||||
|
if src == nil {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
copied, err := copyValue(reflect.ValueOf(src), "extra_params", newTraversalState(), false, 0)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
out, ok := copied.(map[string]any)
|
||||||
|
if !ok {
|
||||||
|
return nil, fmt.Errorf("extra_params: expected object")
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func copyValue(
|
||||||
|
value reflect.Value,
|
||||||
|
path string,
|
||||||
|
state *traversalState,
|
||||||
|
allowEmptyMapKeys bool,
|
||||||
|
containerDepth int,
|
||||||
|
) (any, error) {
|
||||||
|
resolved, cleanup, isNull, err := state.resolveIndirection(value, path)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer cleanup()
|
||||||
|
if isNull {
|
||||||
|
if err := state.produceNode(path); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
value = resolved
|
||||||
|
if !value.CanInterface() {
|
||||||
|
return nil, fmt.Errorf("%s: value cannot be copied", path)
|
||||||
|
}
|
||||||
|
if number, ok := value.Interface().(json.Number); ok {
|
||||||
|
if !validJSONNumber(number) {
|
||||||
|
return nil, fmt.Errorf("%s: invalid JSON number", path)
|
||||||
|
}
|
||||||
|
if err := state.produceNode(path); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return number, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
switch value.Kind() {
|
||||||
|
case reflect.Bool, reflect.String:
|
||||||
|
if err := state.produceNode(path); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return value.Interface(), nil
|
||||||
|
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
|
||||||
|
if err := state.produceNode(path); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return value.Interface(), nil
|
||||||
|
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
|
||||||
|
if err := state.produceNode(path); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return value.Interface(), nil
|
||||||
|
case reflect.Float32, reflect.Float64:
|
||||||
|
number := value.Float()
|
||||||
|
if math.IsNaN(number) || math.IsInf(number, 0) {
|
||||||
|
return nil, fmt.Errorf("%s: floating-point value must be finite", path)
|
||||||
|
}
|
||||||
|
if err := state.produceNode(path); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return value.Interface(), nil
|
||||||
|
case reflect.Map:
|
||||||
|
if value.IsNil() {
|
||||||
|
if err := state.produceNode(path); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
nextDepth, err := state.enterContainer(path, containerDepth)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return copyMapValue(value, path, state, allowEmptyMapKeys, nextDepth)
|
||||||
|
case reflect.Slice:
|
||||||
|
if value.IsNil() {
|
||||||
|
if err := state.produceNode(path); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
nextDepth, err := state.enterContainer(path, containerDepth)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return copySequenceValue(value, path, state, allowEmptyMapKeys, nextDepth)
|
||||||
|
case reflect.Array:
|
||||||
|
nextDepth, err := state.enterContainer(path, containerDepth)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return copySequenceValue(value, path, state, allowEmptyMapKeys, nextDepth)
|
||||||
|
default:
|
||||||
|
return nil, fmt.Errorf("%s: unsupported JSON value type %s", path, value.Type())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func copyMapValue(
|
||||||
|
value reflect.Value,
|
||||||
|
path string,
|
||||||
|
state *traversalState,
|
||||||
|
allowEmptyMapKeys bool,
|
||||||
|
containerDepth int,
|
||||||
|
) (any, error) {
|
||||||
|
if value.Type().Key().Kind() != reflect.String {
|
||||||
|
return nil, fmt.Errorf("%s: map key type %s is not supported", path, value.Type().Key())
|
||||||
|
}
|
||||||
|
if err := state.produceNode(path); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if err := state.ensureChildCapacity(path, value.Len()); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
current := visit{typ: value.Type(), ptr: value.Pointer()}
|
||||||
|
if _, ok := state.active[current]; ok {
|
||||||
|
return nil, fmt.Errorf("%s: cyclic value is not supported", path)
|
||||||
|
}
|
||||||
|
state.active[current] = struct{}{}
|
||||||
|
defer delete(state.active, current)
|
||||||
|
|
||||||
|
keys := value.MapKeys()
|
||||||
|
sort.Slice(keys, func(i, j int) bool {
|
||||||
|
return keys[i].String() < keys[j].String()
|
||||||
|
})
|
||||||
|
|
||||||
|
type entry struct {
|
||||||
|
key reflect.Value
|
||||||
|
name string
|
||||||
|
value any
|
||||||
|
}
|
||||||
|
entries := make([]entry, 0, len(keys))
|
||||||
|
preserveType := true
|
||||||
|
elementType := value.Type().Elem()
|
||||||
|
for _, key := range keys {
|
||||||
|
name := key.String()
|
||||||
|
if name == "" && !allowEmptyMapKeys {
|
||||||
|
return nil, fmt.Errorf("%s: map key must not be empty", path)
|
||||||
|
}
|
||||||
|
copied, err := copyValue(
|
||||||
|
value.MapIndex(key),
|
||||||
|
path+"."+name,
|
||||||
|
state,
|
||||||
|
allowEmptyMapKeys,
|
||||||
|
containerDepth,
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
entries = append(entries, entry{key: key, name: name, value: copied})
|
||||||
|
if copied == nil {
|
||||||
|
if !canAssignNil(elementType) {
|
||||||
|
preserveType = false
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if !reflect.TypeOf(copied).AssignableTo(elementType) {
|
||||||
|
preserveType = false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if preserveType {
|
||||||
|
out := reflect.MakeMapWithSize(value.Type(), len(entries))
|
||||||
|
for _, entry := range entries {
|
||||||
|
if entry.value == nil {
|
||||||
|
out.SetMapIndex(entry.key, reflect.Zero(elementType))
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out.SetMapIndex(entry.key, reflect.ValueOf(entry.value))
|
||||||
|
}
|
||||||
|
return out.Interface(), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
out := make(map[string]any, len(entries))
|
||||||
|
for _, entry := range entries {
|
||||||
|
out[entry.name] = entry.value
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func copySequenceValue(
|
||||||
|
value reflect.Value,
|
||||||
|
path string,
|
||||||
|
state *traversalState,
|
||||||
|
allowEmptyMapKeys bool,
|
||||||
|
containerDepth int,
|
||||||
|
) (any, error) {
|
||||||
|
if err := state.produceNode(path); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if err := state.ensureChildCapacity(path, value.Len()); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
var current visit
|
||||||
|
if value.Kind() == reflect.Slice {
|
||||||
|
current = visit{typ: value.Type(), ptr: value.Pointer()}
|
||||||
|
if _, ok := state.active[current]; ok {
|
||||||
|
return nil, fmt.Errorf("%s: cyclic value is not supported", path)
|
||||||
|
}
|
||||||
|
state.active[current] = struct{}{}
|
||||||
|
defer delete(state.active, current)
|
||||||
|
}
|
||||||
|
|
||||||
|
values := make([]any, value.Len())
|
||||||
|
preserveType := true
|
||||||
|
elementType := value.Type().Elem()
|
||||||
|
for i := 0; i < value.Len(); i++ {
|
||||||
|
copied, err := copyValue(
|
||||||
|
value.Index(i),
|
||||||
|
fmt.Sprintf("%s[%d]", path, i),
|
||||||
|
state,
|
||||||
|
allowEmptyMapKeys,
|
||||||
|
containerDepth,
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
values[i] = copied
|
||||||
|
if copied == nil {
|
||||||
|
if !canAssignNil(elementType) {
|
||||||
|
preserveType = false
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if !reflect.TypeOf(copied).AssignableTo(elementType) {
|
||||||
|
preserveType = false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if preserveType {
|
||||||
|
out := reflect.New(value.Type()).Elem()
|
||||||
|
if value.Kind() == reflect.Slice {
|
||||||
|
out = reflect.MakeSlice(value.Type(), value.Len(), value.Len())
|
||||||
|
}
|
||||||
|
for i, copied := range values {
|
||||||
|
if copied == nil {
|
||||||
|
out.Index(i).Set(reflect.Zero(elementType))
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out.Index(i).Set(reflect.ValueOf(copied))
|
||||||
|
}
|
||||||
|
return out.Interface(), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
out := make([]any, len(values))
|
||||||
|
copy(out, values)
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func validJSONNumber(number json.Number) bool {
|
||||||
|
var parsed json.Number
|
||||||
|
if err := json.Unmarshal([]byte(number.String()), &parsed); err != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return parsed.String() == number.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
func newTraversalState() *traversalState {
|
||||||
|
return &traversalState{active: make(map[visit]struct{})}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (state *traversalState) produceNode(path string) error {
|
||||||
|
if state.producedNodes >= maxProducedNodes {
|
||||||
|
return fmt.Errorf("%s: JSON value work limit exceeded", path)
|
||||||
|
}
|
||||||
|
state.producedNodes++
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (state *traversalState) enterContainer(path string, depth int) (int, error) {
|
||||||
|
depth++
|
||||||
|
if depth > maxContainerDepth {
|
||||||
|
return 0, fmt.Errorf("%s: JSON container depth limit exceeded", path)
|
||||||
|
}
|
||||||
|
return depth, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (state *traversalState) ensureChildCapacity(path string, count int) error {
|
||||||
|
if count > maxProducedNodes-state.producedNodes {
|
||||||
|
return fmt.Errorf("%s: JSON value work limit exceeded", path)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (state *traversalState) resolveIndirection(
|
||||||
|
value reflect.Value,
|
||||||
|
path string,
|
||||||
|
) (reflect.Value, func(), bool, error) {
|
||||||
|
var visits []visit
|
||||||
|
cleanup := func() {
|
||||||
|
for _, current := range visits {
|
||||||
|
delete(state.active, current)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for value.IsValid() && (value.Kind() == reflect.Interface || value.Kind() == reflect.Pointer) {
|
||||||
|
if value.IsNil() {
|
||||||
|
return reflect.Value{}, cleanup, true, nil
|
||||||
|
}
|
||||||
|
if value.Kind() == reflect.Pointer {
|
||||||
|
current := visit{typ: value.Type(), ptr: value.Pointer()}
|
||||||
|
if _, ok := state.active[current]; ok {
|
||||||
|
cleanup()
|
||||||
|
return reflect.Value{}, nil, false, fmt.Errorf("%s: cyclic value is not supported", path)
|
||||||
|
}
|
||||||
|
state.active[current] = struct{}{}
|
||||||
|
visits = append(visits, current)
|
||||||
|
}
|
||||||
|
value = value.Elem()
|
||||||
|
}
|
||||||
|
if !value.IsValid() {
|
||||||
|
return reflect.Value{}, cleanup, true, nil
|
||||||
|
}
|
||||||
|
return value, cleanup, false, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func canAssignNil(typ reflect.Type) bool {
|
||||||
|
switch typ.Kind() {
|
||||||
|
case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Pointer, reflect.Slice:
|
||||||
|
return true
|
||||||
|
default:
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
332
internal/jsonvalue/jsonvalue_test.go
Normal file
332
internal/jsonvalue/jsonvalue_test.go
Normal file
@@ -0,0 +1,332 @@
|
|||||||
|
package jsonvalue
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"math"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
type (
|
||||||
|
namedBool bool
|
||||||
|
namedString string
|
||||||
|
namedInt64 int64
|
||||||
|
namedUint64 uint64
|
||||||
|
namedFloat32 float32
|
||||||
|
namedFloat64 float64
|
||||||
|
namedKey string
|
||||||
|
namedMap map[namedKey]namedInt64
|
||||||
|
namedSlice []namedString
|
||||||
|
namedArray [1]map[string]int
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestCopyPreservesSupportedScalarAndNumberTypes(t *testing.T) {
|
||||||
|
maxInt := int(^uint(0) >> 1)
|
||||||
|
minInt := -maxInt - 1
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
value any
|
||||||
|
}{
|
||||||
|
{name: "bool", value: true},
|
||||||
|
{name: "named bool", value: namedBool(true)},
|
||||||
|
{name: "string", value: "value"},
|
||||||
|
{name: "named string", value: namedString("value")},
|
||||||
|
{name: "int", value: minInt},
|
||||||
|
{name: "int8", value: int8(-1 << 7)},
|
||||||
|
{name: "int16", value: int16(-1 << 15)},
|
||||||
|
{name: "int32", value: int32(-1 << 31)},
|
||||||
|
{name: "int64", value: int64(-1 << 63)},
|
||||||
|
{name: "named int64", value: namedInt64(1<<63 - 1)},
|
||||||
|
{name: "uint", value: ^uint(0)},
|
||||||
|
{name: "uint8", value: ^uint8(0)},
|
||||||
|
{name: "uint16", value: ^uint16(0)},
|
||||||
|
{name: "uint32", value: ^uint32(0)},
|
||||||
|
{name: "uint64", value: ^uint64(0)},
|
||||||
|
{name: "uintptr", value: ^uintptr(0)},
|
||||||
|
{name: "named uint64", value: namedUint64(^uint64(0))},
|
||||||
|
{name: "float32", value: float32(1.25)},
|
||||||
|
{name: "float64", value: float64(-2.5e100)},
|
||||||
|
{name: "named float32", value: namedFloat32(3.5)},
|
||||||
|
{name: "named float64", value: namedFloat64(-4.5e200)},
|
||||||
|
{name: "JSON number integer", value: json.Number("18446744073709551615")},
|
||||||
|
{name: "JSON number fraction", value: json.Number("-1.25e+2")},
|
||||||
|
{name: "JSON number beyond float64", value: json.Number("1e9999")},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
got, err := Copy(tc.value)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("copy value: %v", err)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(got, tc.value) {
|
||||||
|
t.Fatalf("value or concrete type changed: got %#v (%T), want %#v (%T)", got, got, tc.value, tc.value)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCopyRejectsInvalidNumbers(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
value any
|
||||||
|
}{
|
||||||
|
{name: "float32 NaN", value: float32(math.NaN())},
|
||||||
|
{name: "float64 NaN", value: math.NaN()},
|
||||||
|
{name: "named float NaN", value: namedFloat64(math.NaN())},
|
||||||
|
{name: "positive infinity", value: math.Inf(1)},
|
||||||
|
{name: "negative infinity", value: math.Inf(-1)},
|
||||||
|
{name: "empty JSON number", value: json.Number("")},
|
||||||
|
{name: "leading zero JSON number", value: json.Number("01")},
|
||||||
|
{name: "leading plus JSON number", value: json.Number("+1")},
|
||||||
|
{name: "trailing decimal JSON number", value: json.Number("1.")},
|
||||||
|
{name: "leading decimal JSON number", value: json.Number(".1")},
|
||||||
|
{name: "non-number JSON number", value: json.Number("NaN")},
|
||||||
|
{name: "spaced JSON number", value: json.Number(" 1")},
|
||||||
|
{name: "quoted JSON number", value: json.Number(`"1"`)},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
if _, err := Copy(tc.value); err == nil {
|
||||||
|
t.Fatal("expected validation error")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCopyPreservesCompatibleCollectionsAndNilEmptyDistinctions(t *testing.T) {
|
||||||
|
collections := []struct {
|
||||||
|
name string
|
||||||
|
value any
|
||||||
|
}{
|
||||||
|
{name: "unnamed map", value: map[string]int{"limit": 2}},
|
||||||
|
{name: "named map", value: namedMap{"limit": 2}},
|
||||||
|
{name: "unnamed slice", value: []string{"one", "two"}},
|
||||||
|
{name: "named slice", value: namedSlice{"one", "two"}},
|
||||||
|
{name: "unnamed array", value: [2]int{1, 2}},
|
||||||
|
{name: "named array", value: namedArray{{"limit": 2}}},
|
||||||
|
{name: "empty map", value: map[string]int{}},
|
||||||
|
{name: "empty named map", value: namedMap{}},
|
||||||
|
{name: "empty slice", value: []string{}},
|
||||||
|
{name: "empty named slice", value: namedSlice{}},
|
||||||
|
{name: "empty array", value: [0]string{}},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range collections {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
got, err := Copy(tc.value)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("copy collection: %v", err)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(got, tc.value) || reflect.TypeOf(got) != reflect.TypeOf(tc.value) {
|
||||||
|
t.Fatalf("collection changed: got %#v (%T), want %#v (%T)", got, got, tc.value, tc.value)
|
||||||
|
}
|
||||||
|
kind := reflect.ValueOf(got).Kind()
|
||||||
|
if (kind == reflect.Map || kind == reflect.Slice) && reflect.ValueOf(got).IsNil() {
|
||||||
|
t.Fatal("non-nil collection became nil")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
var nilMap map[string]int
|
||||||
|
var nilSlice []string
|
||||||
|
var nilPointer *namedInt64
|
||||||
|
for _, value := range []any{nil, nilMap, nilSlice, nilPointer} {
|
||||||
|
got, err := Copy(value)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("copy null value: %v", err)
|
||||||
|
}
|
||||||
|
if got != nil {
|
||||||
|
t.Fatalf("null value became %#v (%T)", got, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
gotNil, err := CopyMap(nil)
|
||||||
|
if err != nil || gotNil != nil {
|
||||||
|
t.Fatalf("nil CopyMap result = %#v, %v", gotNil, err)
|
||||||
|
}
|
||||||
|
gotEmpty, err := CopyMap(map[string]any{})
|
||||||
|
if err != nil || gotEmpty == nil || len(gotEmpty) != 0 {
|
||||||
|
t.Fatalf("empty CopyMap result = %#v, %v", gotEmpty, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCopyHandlesIndirectionAndIsolatesNestedMutations(t *testing.T) {
|
||||||
|
integer := namedInt64(7)
|
||||||
|
nestedMap := namedMap{"limit": 2}
|
||||||
|
nestedSlice := namedSlice{"original"}
|
||||||
|
nestedArray := namedArray{{"limit": 3}}
|
||||||
|
shared := []any{map[string]int{"value": 4}}
|
||||||
|
input := map[string]any{
|
||||||
|
"integer": &integer,
|
||||||
|
"map": nestedMap,
|
||||||
|
"slice": nestedSlice,
|
||||||
|
"array": nestedArray,
|
||||||
|
"first": shared,
|
||||||
|
"second": shared,
|
||||||
|
}
|
||||||
|
|
||||||
|
copiedValue, err := Copy(input)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("copy mixed tree: %v", err)
|
||||||
|
}
|
||||||
|
copied := copiedValue.(map[string]any)
|
||||||
|
nestedMap["limit"] = 20
|
||||||
|
nestedSlice[0] = "changed"
|
||||||
|
nestedArray[0]["limit"] = 30
|
||||||
|
shared[0].(map[string]int)["value"] = 40
|
||||||
|
|
||||||
|
if got, ok := copied["integer"].(namedInt64); !ok || got != 7 {
|
||||||
|
t.Fatalf("pointer target changed: %#v", copied["integer"])
|
||||||
|
}
|
||||||
|
if got := copied["map"].(namedMap)["limit"]; got != 2 {
|
||||||
|
t.Fatalf("nested map aliased input: %d", got)
|
||||||
|
}
|
||||||
|
if got := copied["slice"].(namedSlice)[0]; got != "original" {
|
||||||
|
t.Fatalf("nested slice aliased input: %q", got)
|
||||||
|
}
|
||||||
|
if got := copied["array"].(namedArray)[0]["limit"]; got != 3 {
|
||||||
|
t.Fatalf("nested array aliased input: %d", got)
|
||||||
|
}
|
||||||
|
first := copied["first"].([]any)
|
||||||
|
second := copied["second"].([]any)
|
||||||
|
if got := first[0].(map[string]int)["value"]; got != 4 {
|
||||||
|
t.Fatalf("shared child aliased input: %d", got)
|
||||||
|
}
|
||||||
|
first[0].(map[string]int)["value"] = 99
|
||||||
|
if got := second[0].(map[string]int)["value"]; got != 4 {
|
||||||
|
t.Fatalf("repeated acyclic value shared copied output: %d", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCopyAndCopyMapApplyDistinctEmptyKeyRules(t *testing.T) {
|
||||||
|
nested := map[string]any{"": []any{"original"}}
|
||||||
|
copiedValue, err := Copy(nested)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Copy rejected empty schema key: %v", err)
|
||||||
|
}
|
||||||
|
nested[""].([]any)[0] = "changed"
|
||||||
|
if got := copiedValue.(map[string]any)[""].([]any)[0]; got != "original" {
|
||||||
|
t.Fatalf("copied schema value was not isolated: %v", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = CopyMap(map[string]any{"nested": map[string]any{"": true}})
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "extra_params.nested") {
|
||||||
|
t.Fatalf("CopyMap empty-key error = %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCopyRejectsUnsupportedValuesAndActiveCycles(t *testing.T) {
|
||||||
|
cyclicMap := map[string]any{}
|
||||||
|
cyclicMap["self"] = cyclicMap
|
||||||
|
cyclicSlice := []any{nil}
|
||||||
|
cyclicSlice[0] = cyclicSlice
|
||||||
|
var cyclicPointer any
|
||||||
|
cyclicPointer = &cyclicPointer
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
value any
|
||||||
|
wantPath string
|
||||||
|
}{
|
||||||
|
{name: "non-string map key", value: map[int]string{1: "one"}, wantPath: "value"},
|
||||||
|
{name: "unsupported channel", value: make(chan int), wantPath: "value"},
|
||||||
|
{name: "deterministic map path", value: map[string]any{"z": make(chan int), "a": make(chan int)}, wantPath: "value.a"},
|
||||||
|
{name: "cyclic map", value: cyclicMap, wantPath: "value.self"},
|
||||||
|
{name: "cyclic slice", value: cyclicSlice, wantPath: "value[0]"},
|
||||||
|
{name: "cyclic pointer", value: cyclicPointer, wantPath: "value"},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
_, err := Copy(tc.value)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), tc.wantPath) {
|
||||||
|
t.Fatalf("error = %v, want structural path %q", err, tc.wantPath)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCopyEnforcesContainerDepth(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
depth int
|
||||||
|
wantErr bool
|
||||||
|
}{
|
||||||
|
{name: "just below", depth: maxContainerDepth - 1},
|
||||||
|
{name: "at limit", depth: maxContainerDepth},
|
||||||
|
{name: "over limit", depth: maxContainerDepth + 1, wantErr: true},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
_, err := Copy(alternatingContainers(tc.depth))
|
||||||
|
if tc.wantErr {
|
||||||
|
if err == nil || !strings.HasPrefix(err.Error(), "value") || !strings.Contains(err.Error(), "container depth limit") {
|
||||||
|
t.Fatalf("depth error = %v", err)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("copy depth %d: %v", tc.depth, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCopyEnforcesProducedNodeBudgetForRepeatedAcyclicValues(t *testing.T) {
|
||||||
|
shared := []any{true}
|
||||||
|
sharedOccurrences := (maxProducedNodes - 2) / 2
|
||||||
|
justBelow := repeatedValues(shared, sharedOccurrences, 0)
|
||||||
|
atLimit := repeatedValues(shared, sharedOccurrences, 1)
|
||||||
|
overLimit := repeatedValues(shared, sharedOccurrences, 2)
|
||||||
|
|
||||||
|
for name, value := range map[string]any{
|
||||||
|
"just below": justBelow,
|
||||||
|
"at limit": atLimit,
|
||||||
|
} {
|
||||||
|
t.Run(name, func(t *testing.T) {
|
||||||
|
if _, err := Copy(value); err != nil {
|
||||||
|
t.Fatalf("copy value within work budget: %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err := Copy(overLimit)
|
||||||
|
if err == nil || !strings.HasPrefix(err.Error(), "value[") || !strings.Contains(err.Error(), "value work limit") {
|
||||||
|
t.Fatalf("work-budget error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = Copy(make([]any, maxProducedNodes))
|
||||||
|
if err == nil || !strings.HasPrefix(err.Error(), "value:") || !strings.Contains(err.Error(), "value work limit") {
|
||||||
|
t.Fatalf("flat work-budget error = %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func alternatingContainers(depth int) any {
|
||||||
|
var value any = true
|
||||||
|
for level := 0; level < depth; level++ {
|
||||||
|
switch level % 3 {
|
||||||
|
case 0:
|
||||||
|
value = map[string]any{"child": value}
|
||||||
|
case 1:
|
||||||
|
value = []any{value}
|
||||||
|
default:
|
||||||
|
value = [1]any{value}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return value
|
||||||
|
}
|
||||||
|
|
||||||
|
func repeatedValues(shared []any, occurrences, leadingScalars int) []any {
|
||||||
|
values := make([]any, 0, leadingScalars+occurrences)
|
||||||
|
for i := 0; i < leadingScalars; i++ {
|
||||||
|
values = append(values, false)
|
||||||
|
}
|
||||||
|
for i := 0; i < occurrences; i++ {
|
||||||
|
values = append(values, shared)
|
||||||
|
}
|
||||||
|
return values
|
||||||
|
}
|
||||||
@@ -12,7 +12,6 @@ import (
|
|||||||
"os"
|
"os"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
"unicode/utf8"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/defaults"
|
"gitea.maximumdirect.net/eric/promptkit/internal/defaults"
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||||
@@ -26,6 +25,20 @@ var (
|
|||||||
ErrMalformedResponse = errors.New("malformed llm response")
|
ErrMalformedResponse = errors.New("malformed llm response")
|
||||||
)
|
)
|
||||||
|
|
||||||
|
const maxOpenAIChatResponseBytes int64 = 16 << 20
|
||||||
|
|
||||||
|
type requestFailedError struct {
|
||||||
|
cause error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *requestFailedError) Error() string {
|
||||||
|
return ErrRequestFailed.Error()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *requestFailedError) Unwrap() []error {
|
||||||
|
return []error{ErrRequestFailed, e.cause}
|
||||||
|
}
|
||||||
|
|
||||||
type OpenAICompatibleConfig struct {
|
type OpenAICompatibleConfig struct {
|
||||||
BaseURL string
|
BaseURL string
|
||||||
Model string
|
Model string
|
||||||
@@ -40,9 +53,11 @@ type OpenAICompatibleClient struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func NewOpenAICompatibleClient(cfg OpenAICompatibleConfig) (*OpenAICompatibleClient, error) {
|
func NewOpenAICompatibleClient(cfg OpenAICompatibleConfig) (*OpenAICompatibleClient, error) {
|
||||||
baseURL := strings.TrimSpace(cfg.BaseURL)
|
baseURL := ""
|
||||||
if baseURL != "" {
|
if strings.TrimSpace(cfg.BaseURL) != "" {
|
||||||
if _, err := url.ParseRequestURI(baseURL); err != nil {
|
var err error
|
||||||
|
baseURL, err = domain.NormalizeOpenAICompatibleBaseEndpoint(cfg.BaseURL)
|
||||||
|
if err != nil {
|
||||||
return nil, fmt.Errorf("%w: invalid base URL: %v", ErrInvalidConfig, err)
|
return nil, fmt.Errorf("%w: invalid base URL: %v", ErrInvalidConfig, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -64,25 +79,29 @@ func NewOpenAICompatibleClient(cfg OpenAICompatibleConfig) (*OpenAICompatibleCli
|
|||||||
}
|
}
|
||||||
|
|
||||||
return &OpenAICompatibleClient{
|
return &OpenAICompatibleClient{
|
||||||
baseURL: strings.TrimRight(baseURL, "/"),
|
baseURL: baseURL,
|
||||||
defaultModel: cfg.Model,
|
defaultModel: cfg.Model,
|
||||||
httpClient: client,
|
httpClient: client,
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *OpenAICompatibleClient) Generate(ctx context.Context, req domain.GenerateRequest) (*domain.GenerateResponse, error) {
|
func (c *OpenAICompatibleClient) Generate(ctx context.Context, req domain.GenerateRequest) (*domain.GenerateResponse, error) {
|
||||||
if req.Target.TimeoutSeconds < 0 {
|
if err := domain.ValidateExecutionTargetSettings(req.Target); err != nil {
|
||||||
return nil, fmt.Errorf("%w: timeout_seconds must be greater than or equal to 0", ErrInvalidRequest)
|
return nil, fmt.Errorf("%w: %v", ErrInvalidRequest, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
endpoint := strings.TrimSpace(req.Target.Endpoint)
|
selectedEndpoint := req.Target.Endpoint
|
||||||
if endpoint == "" {
|
if strings.TrimSpace(selectedEndpoint) == "" {
|
||||||
endpoint = c.baseURL
|
selectedEndpoint = c.baseURL
|
||||||
}
|
}
|
||||||
if endpoint == "" {
|
endpoint, err := domain.NormalizeOpenAICompatibleBaseEndpoint(selectedEndpoint)
|
||||||
return nil, fmt.Errorf("%w: endpoint is required", ErrInvalidRequest)
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("%w: invalid endpoint: %v", ErrInvalidRequest, err)
|
||||||
|
}
|
||||||
|
endpoint, err = url.JoinPath(endpoint, defaults.OpenAIChatCompletionsPath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("%w: invalid endpoint path: %v", ErrInvalidRequest, err)
|
||||||
}
|
}
|
||||||
endpoint = strings.TrimRight(endpoint, "/") + defaults.OpenAIChatCompletionsPath
|
|
||||||
|
|
||||||
wireReq, err := openAIChatRequestFromGenerateRequest(req, c.defaultModel)
|
wireReq, err := openAIChatRequestFromGenerateRequest(req, c.defaultModel)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -131,7 +150,7 @@ func (c *OpenAICompatibleClient) Generate(ctx context.Context, req domain.Genera
|
|||||||
|
|
||||||
httpResp, err := httpClient.Do(httpReq)
|
httpResp, err := httpClient.Do(httpReq)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("%w: %v", ErrRequestFailed, err)
|
return nil, &requestFailedError{cause: err}
|
||||||
}
|
}
|
||||||
defer httpResp.Body.Close()
|
defer httpResp.Body.Close()
|
||||||
|
|
||||||
@@ -139,10 +158,13 @@ func (c *OpenAICompatibleClient) Generate(ctx context.Context, req domain.Genera
|
|||||||
_, _ = io.Copy(io.Discard, io.LimitReader(httpResp.Body, 4096))
|
_, _ = io.Copy(io.Discard, io.LimitReader(httpResp.Body, 4096))
|
||||||
return nil, fmt.Errorf("%w: status=%d", ErrUnexpectedStatus, httpResp.StatusCode)
|
return nil, fmt.Errorf("%w: status=%d", ErrUnexpectedStatus, httpResp.StatusCode)
|
||||||
}
|
}
|
||||||
|
if httpResp.ContentLength > maxOpenAIChatResponseBytes {
|
||||||
|
return nil, openAIChatResponseTooLargeError()
|
||||||
|
}
|
||||||
|
|
||||||
var wireResp openAIChatResponse
|
wireResp, err := decodeOpenAIChatResponse(httpResp.Body)
|
||||||
if err := json.NewDecoder(httpResp.Body).Decode(&wireResp); err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("%w: failed to decode response: %v", ErrMalformedResponse, err)
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(wireResp.Choices) == 0 {
|
if len(wireResp.Choices) == 0 {
|
||||||
@@ -165,6 +187,46 @@ func (c *OpenAICompatibleClient) Generate(ctx context.Context, req domain.Genera
|
|||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func decodeOpenAIChatResponse(body io.Reader) (openAIChatResponse, error) {
|
||||||
|
limited := &io.LimitedReader{
|
||||||
|
R: body,
|
||||||
|
N: maxOpenAIChatResponseBytes + 1,
|
||||||
|
}
|
||||||
|
decoder := json.NewDecoder(limited)
|
||||||
|
|
||||||
|
var response openAIChatResponse
|
||||||
|
if err := decoder.Decode(&response); err != nil {
|
||||||
|
if limited.N == 0 {
|
||||||
|
return openAIChatResponse{}, openAIChatResponseTooLargeError()
|
||||||
|
}
|
||||||
|
return openAIChatResponse{}, fmt.Errorf("%w: failed to decode response", ErrMalformedResponse)
|
||||||
|
}
|
||||||
|
if limited.N == 0 {
|
||||||
|
return openAIChatResponse{}, openAIChatResponseTooLargeError()
|
||||||
|
}
|
||||||
|
|
||||||
|
var trailing any
|
||||||
|
if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) {
|
||||||
|
if limited.N == 0 {
|
||||||
|
return openAIChatResponse{}, openAIChatResponseTooLargeError()
|
||||||
|
}
|
||||||
|
return openAIChatResponse{}, fmt.Errorf("%w: response contains trailing data", ErrMalformedResponse)
|
||||||
|
}
|
||||||
|
if limited.N == 0 {
|
||||||
|
return openAIChatResponse{}, openAIChatResponseTooLargeError()
|
||||||
|
}
|
||||||
|
|
||||||
|
return response, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func openAIChatResponseTooLargeError() error {
|
||||||
|
return fmt.Errorf(
|
||||||
|
"%w: response exceeds %d-byte limit",
|
||||||
|
ErrMalformedResponse,
|
||||||
|
maxOpenAIChatResponseBytes,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
func openAIChatRequestFromGenerateRequest(req domain.GenerateRequest, defaultModel string) (openAIChatRequest, error) {
|
func openAIChatRequestFromGenerateRequest(req domain.GenerateRequest, defaultModel string) (openAIChatRequest, error) {
|
||||||
model := strings.TrimSpace(req.Target.Model)
|
model := strings.TrimSpace(req.Target.Model)
|
||||||
if model == "" {
|
if model == "" {
|
||||||
@@ -177,12 +239,11 @@ func openAIChatRequestFromGenerateRequest(req domain.GenerateRequest, defaultMod
|
|||||||
wireReq := openAIChatRequest{
|
wireReq := openAIChatRequest{
|
||||||
Model: model,
|
Model: model,
|
||||||
}
|
}
|
||||||
if sessionID := strings.TrimSpace(req.Prompt.SessionID); sessionID != "" {
|
sessionID, err := domain.NormalizeSessionID(req.Prompt.SessionID)
|
||||||
if n := utf8.RuneCountInString(sessionID); n > domain.SessionIDMaxLength {
|
if err != nil {
|
||||||
return openAIChatRequest{}, fmt.Errorf("session_id length %d exceeds maximum %d", n, domain.SessionIDMaxLength)
|
return openAIChatRequest{}, err
|
||||||
}
|
|
||||||
wireReq.SessionID = sessionID
|
|
||||||
}
|
}
|
||||||
|
wireReq.SessionID = sessionID
|
||||||
|
|
||||||
wireReq.Messages = make([]openAIChatRequestMessage, 0, len(req.Prompt.Messages))
|
wireReq.Messages = make([]openAIChatRequestMessage, 0, len(req.Prompt.Messages))
|
||||||
for _, msg := range req.Prompt.Messages {
|
for _, msg := range req.Prompt.Messages {
|
||||||
@@ -262,7 +323,7 @@ func openAIChatRequestPayload(req openAIChatRequest) (map[string]any, error) {
|
|||||||
if key == "" {
|
if key == "" {
|
||||||
return nil, errors.New("extra_params key must not be empty")
|
return nil, errors.New("extra_params key must not be empty")
|
||||||
}
|
}
|
||||||
if _, reserved := reservedOpenAIChatRequestFields[key]; reserved {
|
if IsReservedOpenAIChatRequestField(key) {
|
||||||
return nil, fmt.Errorf("extra_params key %q collides with reserved request field", key)
|
return nil, fmt.Errorf("extra_params key %q collides with reserved request field", key)
|
||||||
}
|
}
|
||||||
if _, err := json.Marshal(value); err != nil {
|
if _, err := json.Marshal(value); err != nil {
|
||||||
@@ -274,16 +335,23 @@ func openAIChatRequestPayload(req openAIChatRequest) (map[string]any, error) {
|
|||||||
return out, nil
|
return out, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
var reservedOpenAIChatRequestFields = map[string]struct{}{
|
// IsReservedOpenAIChatRequestField reports whether name is owned by the
|
||||||
"model": {},
|
// standard OpenAI-compatible chat request rather than extra parameters.
|
||||||
"session_id": {},
|
func IsReservedOpenAIChatRequestField(name string) bool {
|
||||||
"messages": {},
|
switch name {
|
||||||
"temperature": {},
|
case "model",
|
||||||
"max_tokens": {},
|
"session_id",
|
||||||
"top_p": {},
|
"messages",
|
||||||
"service_tier": {},
|
"temperature",
|
||||||
"reasoning_effort": {},
|
"max_tokens",
|
||||||
"response_format": {},
|
"top_p",
|
||||||
|
"service_tier",
|
||||||
|
"reasoning_effort",
|
||||||
|
"response_format":
|
||||||
|
return true
|
||||||
|
default:
|
||||||
|
return false
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
type openAIChatRequestMessage struct {
|
type openAIChatRequestMessage struct {
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -1,9 +1,8 @@
|
|||||||
id: aion-2
|
id: aion-2
|
||||||
endpoint: https://openrouter.ai/api/v1
|
backend: openrouter
|
||||||
model: aion-labs/aion-2.0
|
model: aion-labs/aion-2.0
|
||||||
temperature: 0.72
|
temperature: 0.72
|
||||||
reasoning_effort: high
|
reasoning_effort: high
|
||||||
top_p: 0.95
|
top_p: 0.95
|
||||||
timeout_seconds: 180
|
timeout_seconds: 180
|
||||||
api_key_env: OPENROUTER_API_KEY
|
|
||||||
service_tier: flex
|
service_tier: flex
|
||||||
|
|||||||
@@ -1,7 +1,6 @@
|
|||||||
id: claude-fable-latest
|
id: claude-fable-latest
|
||||||
endpoint: https://openrouter.ai/api/v1
|
backend: openrouter
|
||||||
model: "~anthropic/claude-fable-latest"
|
model: "~anthropic/claude-fable-latest"
|
||||||
reasoning_effort: high
|
reasoning_effort: high
|
||||||
timeout_seconds: 600
|
timeout_seconds: 600
|
||||||
api_key_env: OPENROUTER_API_KEY
|
|
||||||
service_tier: flex
|
service_tier: flex
|
||||||
|
|||||||
@@ -1,7 +1,6 @@
|
|||||||
id: claude-haiku-latest
|
id: claude-haiku-latest
|
||||||
endpoint: https://openrouter.ai/api/v1
|
backend: openrouter
|
||||||
model: "~anthropic/claude-haiku-latest"
|
model: "~anthropic/claude-haiku-latest"
|
||||||
reasoning_effort: medium
|
reasoning_effort: medium
|
||||||
timeout_seconds: 240
|
timeout_seconds: 240
|
||||||
api_key_env: OPENROUTER_API_KEY
|
|
||||||
service_tier: flex
|
service_tier: flex
|
||||||
|
|||||||
@@ -1,7 +1,6 @@
|
|||||||
id: claude-opus-latest
|
id: claude-opus-latest
|
||||||
endpoint: https://openrouter.ai/api/v1
|
backend: openrouter
|
||||||
model: "~anthropic/claude-opus-latest"
|
model: "~anthropic/claude-opus-latest"
|
||||||
reasoning_effort: high
|
reasoning_effort: high
|
||||||
timeout_seconds: 240
|
timeout_seconds: 240
|
||||||
api_key_env: OPENROUTER_API_KEY
|
|
||||||
service_tier: flex
|
service_tier: flex
|
||||||
|
|||||||
@@ -1,7 +1,6 @@
|
|||||||
id: claude-sonnet-latest
|
id: claude-sonnet-latest
|
||||||
endpoint: https://openrouter.ai/api/v1
|
backend: openrouter
|
||||||
model: "~anthropic/claude-sonnet-latest"
|
model: "~anthropic/claude-sonnet-latest"
|
||||||
reasoning_effort: high
|
reasoning_effort: high
|
||||||
timeout_seconds: 240
|
timeout_seconds: 240
|
||||||
api_key_env: OPENROUTER_API_KEY
|
|
||||||
service_tier: flex
|
service_tier: flex
|
||||||
|
|||||||
@@ -1,7 +1,6 @@
|
|||||||
id: deepseek-3-2
|
id: deepseek-3-2
|
||||||
endpoint: https://openrouter.ai/api/v1
|
backend: openrouter
|
||||||
model: deepseek/deepseek-v3.2
|
model: deepseek/deepseek-v3.2
|
||||||
reasoning_effort: high
|
reasoning_effort: high
|
||||||
timeout_seconds: 180
|
timeout_seconds: 180
|
||||||
api_key_env: OPENROUTER_API_KEY
|
|
||||||
service_tier: flex
|
service_tier: flex
|
||||||
|
|||||||
@@ -1,7 +1,6 @@
|
|||||||
id: deepseek-4-flash
|
id: deepseek-4-flash
|
||||||
endpoint: https://openrouter.ai/api/v1
|
backend: openrouter
|
||||||
model: deepseek/deepseek-v4-flash
|
model: deepseek/deepseek-v4-flash
|
||||||
#reasoning_effort: medium
|
#reasoning_effort: medium
|
||||||
timeout_seconds: 180
|
timeout_seconds: 180
|
||||||
api_key_env: OPENROUTER_API_KEY
|
|
||||||
service_tier: flex
|
service_tier: flex
|
||||||
|
|||||||
@@ -1,7 +1,6 @@
|
|||||||
id: deepseek-4-pro
|
id: deepseek-4-pro
|
||||||
endpoint: https://openrouter.ai/api/v1
|
backend: openrouter
|
||||||
model: deepseek/deepseek-v4-pro
|
model: deepseek/deepseek-v4-pro
|
||||||
reasoning_effort: high
|
reasoning_effort: high
|
||||||
timeout_seconds: 180
|
timeout_seconds: 180
|
||||||
api_key_env: OPENROUTER_API_KEY
|
|
||||||
service_tier: flex
|
service_tier: flex
|
||||||
|
|||||||
@@ -1,9 +1,8 @@
|
|||||||
id: gemini-2-flash-lite
|
id: gemini-2-flash-lite
|
||||||
endpoint: https://openrouter.ai/api/v1
|
backend: openrouter
|
||||||
model: "google/gemini-2.5-flash-lite"
|
model: "google/gemini-2.5-flash-lite"
|
||||||
#temperature: 0.15
|
#temperature: 0.15
|
||||||
reasoning_effort: high
|
reasoning_effort: high
|
||||||
#top_p: 0.98
|
#top_p: 0.98
|
||||||
timeout_seconds: 240
|
timeout_seconds: 240
|
||||||
api_key_env: OPENROUTER_API_KEY
|
|
||||||
service_tier: flex
|
service_tier: flex
|
||||||
|
|||||||
@@ -1,9 +1,8 @@
|
|||||||
id: gemini-2-flash
|
id: gemini-2-flash
|
||||||
endpoint: https://openrouter.ai/api/v1
|
backend: openrouter
|
||||||
model: "google/gemini-2.5-flash"
|
model: "google/gemini-2.5-flash"
|
||||||
#temperature: 0.15
|
#temperature: 0.15
|
||||||
reasoning_effort: high
|
reasoning_effort: high
|
||||||
#top_p: 0.98
|
#top_p: 0.98
|
||||||
timeout_seconds: 240
|
timeout_seconds: 240
|
||||||
api_key_env: OPENROUTER_API_KEY
|
|
||||||
service_tier: flex
|
service_tier: flex
|
||||||
|
|||||||
@@ -1,9 +1,8 @@
|
|||||||
id: gemini-2-pro
|
id: gemini-2-pro
|
||||||
endpoint: https://openrouter.ai/api/v1
|
backend: openrouter
|
||||||
model: "google/gemini-2.5-pro"
|
model: "google/gemini-2.5-pro"
|
||||||
#temperature: 0.15
|
#temperature: 0.15
|
||||||
reasoning_effort: high
|
reasoning_effort: high
|
||||||
#top_p: 0.98
|
#top_p: 0.98
|
||||||
timeout_seconds: 240
|
timeout_seconds: 240
|
||||||
api_key_env: OPENROUTER_API_KEY
|
|
||||||
service_tier: flex
|
service_tier: flex
|
||||||
|
|||||||
@@ -1,9 +1,8 @@
|
|||||||
id: gemini-3-flash-lite
|
id: gemini-3-flash-lite
|
||||||
endpoint: https://openrouter.ai/api/v1
|
backend: openrouter
|
||||||
model: "google/gemini-3.1-flash-lite"
|
model: "google/gemini-3.1-flash-lite"
|
||||||
#temperature: 0.15
|
#temperature: 0.15
|
||||||
reasoning_effort: high
|
reasoning_effort: high
|
||||||
#top_p: 0.98
|
#top_p: 0.98
|
||||||
timeout_seconds: 240
|
timeout_seconds: 240
|
||||||
api_key_env: OPENROUTER_API_KEY
|
|
||||||
service_tier: flex
|
service_tier: flex
|
||||||
|
|||||||
@@ -1,9 +1,8 @@
|
|||||||
id: gemini-flash-latest
|
id: gemini-flash-latest
|
||||||
endpoint: https://openrouter.ai/api/v1
|
backend: openrouter
|
||||||
model: "~google/gemini-flash-latest"
|
model: "~google/gemini-flash-latest"
|
||||||
#temperature: 0.15
|
#temperature: 0.15
|
||||||
reasoning_effort: high
|
reasoning_effort: high
|
||||||
#top_p: 0.98
|
#top_p: 0.98
|
||||||
timeout_seconds: 240
|
timeout_seconds: 240
|
||||||
api_key_env: OPENROUTER_API_KEY
|
|
||||||
service_tier: flex
|
service_tier: flex
|
||||||
|
|||||||
@@ -1,9 +1,8 @@
|
|||||||
id: gemini-pro-latest
|
id: gemini-pro-latest
|
||||||
endpoint: https://openrouter.ai/api/v1
|
backend: openrouter
|
||||||
model: "~google/gemini-pro-latest"
|
model: "~google/gemini-pro-latest"
|
||||||
#temperature: 0.15
|
#temperature: 0.15
|
||||||
reasoning_effort: high
|
reasoning_effort: high
|
||||||
#top_p: 0.98
|
#top_p: 0.98
|
||||||
timeout_seconds: 240
|
timeout_seconds: 240
|
||||||
api_key_env: OPENROUTER_API_KEY
|
|
||||||
service_tier: flex
|
service_tier: flex
|
||||||
|
|||||||
@@ -1,9 +1,8 @@
|
|||||||
id: gemma-4-31b
|
id: gemma-4-31b
|
||||||
endpoint: https://openrouter.ai/api/v1
|
backend: openrouter
|
||||||
model: google/gemma-4-31b-it:exacto
|
model: google/gemma-4-31b-it:exacto
|
||||||
temperature: 0.15
|
temperature: 0.15
|
||||||
reasoning_effort: high
|
reasoning_effort: high
|
||||||
top_p: 0.98
|
top_p: 0.98
|
||||||
timeout_seconds: 240
|
timeout_seconds: 240
|
||||||
api_key_env: OPENROUTER_API_KEY
|
|
||||||
service_tier: flex
|
service_tier: flex
|
||||||
|
|||||||
@@ -1,9 +1,8 @@
|
|||||||
id: minimax-m2
|
id: minimax-m2
|
||||||
endpoint: https://openrouter.ai/api/v1
|
backend: openrouter
|
||||||
model: minimax/minimax-m2.5
|
model: minimax/minimax-m2.5
|
||||||
temperature: 0.5
|
temperature: 0.5
|
||||||
reasoning_effort: high
|
reasoning_effort: high
|
||||||
top_p: 0.95
|
top_p: 0.95
|
||||||
timeout_seconds: 180
|
timeout_seconds: 180
|
||||||
api_key_env: OPENROUTER_API_KEY
|
|
||||||
service_tier: flex
|
service_tier: flex
|
||||||
|
|||||||
@@ -1,9 +1,8 @@
|
|||||||
id: minimax-m3
|
id: minimax-m3
|
||||||
endpoint: https://openrouter.ai/api/v1
|
backend: openrouter
|
||||||
model: minimax/minimax-m3
|
model: minimax/minimax-m3
|
||||||
#temperature: 0.5
|
#temperature: 0.5
|
||||||
reasoning_effort: high
|
reasoning_effort: high
|
||||||
#top_p: 0.95
|
#top_p: 0.95
|
||||||
timeout_seconds: 180
|
timeout_seconds: 180
|
||||||
api_key_env: OPENROUTER_API_KEY
|
|
||||||
service_tier: flex
|
service_tier: flex
|
||||||
|
|||||||
@@ -1,7 +1,6 @@
|
|||||||
id: mistral-large-2512
|
id: mistral-large-2512
|
||||||
endpoint: https://openrouter.ai/api/v1
|
backend: openrouter
|
||||||
model: mistralai/mistral-large-2512
|
model: mistralai/mistral-large-2512
|
||||||
temperature: 0.15
|
temperature: 0.15
|
||||||
top_p: 0.98
|
top_p: 0.98
|
||||||
timeout_seconds: 180
|
timeout_seconds: 180
|
||||||
api_key_env: OPENROUTER_API_KEY
|
|
||||||
|
|||||||
@@ -1,8 +1,7 @@
|
|||||||
id: mistral-medium-3-5
|
id: mistral-medium-3-5
|
||||||
endpoint: https://openrouter.ai/api/v1
|
backend: openrouter
|
||||||
model: mistralai/mistral-medium-3-5
|
model: mistralai/mistral-medium-3-5
|
||||||
temperature: 0.15
|
temperature: 0.15
|
||||||
reasoning_effort: high
|
reasoning_effort: high
|
||||||
top_p: 0.98
|
top_p: 0.98
|
||||||
timeout_seconds: 180
|
timeout_seconds: 180
|
||||||
api_key_env: OPENROUTER_API_KEY
|
|
||||||
|
|||||||
@@ -1,7 +1,6 @@
|
|||||||
id: mistral-small-3
|
id: mistral-small-3
|
||||||
endpoint: https://openrouter.ai/api/v1
|
backend: openrouter
|
||||||
model: mistralai/mistral-small-3.2-24b-instruct
|
model: mistralai/mistral-small-3.2-24b-instruct
|
||||||
temperature: 0.05
|
temperature: 0.05
|
||||||
top_p: 1.0
|
top_p: 1.0
|
||||||
timeout_seconds: 180
|
timeout_seconds: 180
|
||||||
api_key_env: OPENROUTER_API_KEY
|
|
||||||
|
|||||||
@@ -1,8 +1,7 @@
|
|||||||
id: mistral-small-4
|
id: mistral-small-4
|
||||||
endpoint: https://openrouter.ai/api/v1
|
backend: openrouter
|
||||||
model: mistralai/mistral-small-2603
|
model: mistralai/mistral-small-2603
|
||||||
temperature: 0.1
|
temperature: 0.1
|
||||||
reasoning_effort: high
|
reasoning_effort: high
|
||||||
top_p: 0.98
|
top_p: 0.98
|
||||||
timeout_seconds: 180
|
timeout_seconds: 180
|
||||||
api_key_env: OPENROUTER_API_KEY
|
|
||||||
|
|||||||
@@ -1,7 +1,6 @@
|
|||||||
id: nemotron-3-ultra
|
id: nemotron-3-ultra
|
||||||
endpoint: https://openrouter.ai/api/v1
|
backend: openrouter
|
||||||
model: nvidia/nemotron-3-ultra-550b-a55b
|
model: nvidia/nemotron-3-ultra-550b-a55b
|
||||||
reasoning_effort: high
|
reasoning_effort: high
|
||||||
timeout_seconds: 180
|
timeout_seconds: 180
|
||||||
api_key_env: OPENROUTER_API_KEY
|
|
||||||
service_tier: flex
|
service_tier: flex
|
||||||
|
|||||||
@@ -1,7 +1,6 @@
|
|||||||
id: gpt-5-mini
|
id: gpt-5-mini
|
||||||
endpoint: https://openrouter.ai/api/v1
|
backend: openrouter
|
||||||
model: "openai/gpt-5.4-mini"
|
model: "openai/gpt-5.4-mini"
|
||||||
reasoning_effort: high
|
reasoning_effort: high
|
||||||
timeout_seconds: 240
|
timeout_seconds: 240
|
||||||
api_key_env: OPENROUTER_API_KEY
|
|
||||||
service_tier: flex
|
service_tier: flex
|
||||||
|
|||||||
@@ -1,7 +1,6 @@
|
|||||||
id: gpt-5-nano
|
id: gpt-5-nano
|
||||||
endpoint: https://openrouter.ai/api/v1
|
backend: openrouter
|
||||||
model: "openai/gpt-5.4-nano"
|
model: "openai/gpt-5.4-nano"
|
||||||
reasoning_effort: high
|
reasoning_effort: high
|
||||||
timeout_seconds: 240
|
timeout_seconds: 240
|
||||||
api_key_env: OPENROUTER_API_KEY
|
|
||||||
service_tier: flex
|
service_tier: flex
|
||||||
|
|||||||
@@ -2,7 +2,6 @@ package builtin
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"embed"
|
"embed"
|
||||||
"strings"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/profile"
|
"gitea.maximumdirect.net/eric/promptkit/internal/profile"
|
||||||
)
|
)
|
||||||
@@ -15,17 +14,3 @@ var assets embed.FS
|
|||||||
func NewRepository() profile.Repository {
|
func NewRepository() profile.Repository {
|
||||||
return profile.NewFSRepository(assets, assetRoot)
|
return profile.NewFSRepository(assets, assetRoot)
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewRepositoryWithPrimary(primary profile.Repository) profile.Repository {
|
|
||||||
if primary == nil {
|
|
||||||
return NewRepository()
|
|
||||||
}
|
|
||||||
return profile.NewOverlayRepository(primary, NewRepository())
|
|
||||||
}
|
|
||||||
|
|
||||||
func NewRepositoryWithDirectory(dir string) profile.Repository {
|
|
||||||
if strings.TrimSpace(dir) == "" {
|
|
||||||
return NewRepository()
|
|
||||||
}
|
|
||||||
return NewRepositoryWithPrimary(profile.NewFilesystemRepository(dir))
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -2,13 +2,11 @@ package builtin
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
|
||||||
"io/fs"
|
"io/fs"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
"gitea.maximumdirect.net/eric/promptkit/internal/backend"
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/profile"
|
|
||||||
"gopkg.in/yaml.v3"
|
"gopkg.in/yaml.v3"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -28,6 +26,12 @@ func TestBuiltInProfilesValidateThroughRepository(t *testing.T) {
|
|||||||
if p.ID != id {
|
if p.ID != id {
|
||||||
t.Fatalf("expected profile id %q, got %q", id, p.ID)
|
t.Fatalf("expected profile id %q, got %q", id, p.ID)
|
||||||
}
|
}
|
||||||
|
if p.BackendID != backend.OpenRouterID {
|
||||||
|
t.Fatalf("expected profile %q to select %q, got %q", id, backend.OpenRouterID, p.BackendID)
|
||||||
|
}
|
||||||
|
if p.Endpoint != "" || p.APIKeyEnv != "" {
|
||||||
|
t.Fatalf("expected profile %q to inherit backend connection settings, got endpoint=%q api_key_env=%q", id, p.Endpoint, p.APIKeyEnv)
|
||||||
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -60,6 +64,15 @@ func loadBuiltInProfileIDs(t *testing.T) map[string]string {
|
|||||||
if _, ok := raw["api_key"]; ok {
|
if _, ok := raw["api_key"]; ok {
|
||||||
t.Fatalf("built-in profile %s contains raw api_key", name)
|
t.Fatalf("built-in profile %s contains raw api_key", name)
|
||||||
}
|
}
|
||||||
|
if raw["backend"] != backend.OpenRouterID {
|
||||||
|
t.Fatalf("built-in profile %s does not select %q", name, backend.OpenRouterID)
|
||||||
|
}
|
||||||
|
if _, ok := raw["endpoint"]; ok {
|
||||||
|
t.Fatalf("built-in profile %s repeats endpoint", name)
|
||||||
|
}
|
||||||
|
if _, ok := raw["api_key_env"]; ok {
|
||||||
|
t.Fatalf("built-in profile %s repeats api_key_env", name)
|
||||||
|
}
|
||||||
id, ok := raw["id"].(string)
|
id, ok := raw["id"].(string)
|
||||||
if !ok || strings.TrimSpace(id) == "" {
|
if !ok || strings.TrimSpace(id) == "" {
|
||||||
t.Fatalf("built-in profile %s has missing id", name)
|
t.Fatalf("built-in profile %s has missing id", name)
|
||||||
@@ -75,53 +88,3 @@ func loadBuiltInProfileIDs(t *testing.T) map[string]string {
|
|||||||
}
|
}
|
||||||
return ids
|
return ids
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRepositoryWithPrimaryUsesPrimaryBeforeBuiltIns(t *testing.T) {
|
|
||||||
repo := NewRepositoryWithPrimary(staticProfileRepo{
|
|
||||||
profiles: map[string]string{"mistral-small-3": "custom-model"},
|
|
||||||
})
|
|
||||||
|
|
||||||
p, err := repo.GetProfile(context.Background(), "mistral-small-3")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("expected profile to load, got %v", err)
|
|
||||||
}
|
|
||||||
if p.Model != "custom-model" {
|
|
||||||
t.Fatalf("expected primary profile to override built-in, got %+v", p)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRepositoryWithPrimaryFallsBackToBuiltIns(t *testing.T) {
|
|
||||||
repo := NewRepositoryWithPrimary(staticProfileRepo{})
|
|
||||||
|
|
||||||
p, err := repo.GetProfile(context.Background(), "mistral-small-3")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("expected built-in profile to load, got %v", err)
|
|
||||||
}
|
|
||||||
if p.ID != "mistral-small-3" {
|
|
||||||
t.Fatalf("unexpected profile: %+v", p)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRepositoryWithPrimaryDoesNotFallBackAfterPrimaryError(t *testing.T) {
|
|
||||||
repo := NewRepositoryWithPrimary(staticProfileRepo{err: profile.ErrInvalidProfile})
|
|
||||||
|
|
||||||
_, err := repo.GetProfile(context.Background(), "mistral-small-3")
|
|
||||||
if !errors.Is(err, profile.ErrInvalidProfile) {
|
|
||||||
t.Fatalf("expected primary error, got %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
type staticProfileRepo struct {
|
|
||||||
profiles map[string]string
|
|
||||||
err error
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r staticProfileRepo) GetProfile(_ context.Context, id string) (*domain.ExecutionProfile, error) {
|
|
||||||
if r.err != nil {
|
|
||||||
return nil, r.err
|
|
||||||
}
|
|
||||||
if model, ok := r.profiles[id]; ok {
|
|
||||||
return &domain.ExecutionProfile{ID: id, Endpoint: "http://primary/v1", Model: model}, nil
|
|
||||||
}
|
|
||||||
return nil, profile.ErrProfileNotFound
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -5,13 +5,14 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"io"
|
||||||
"io/fs"
|
"io/fs"
|
||||||
"os"
|
"os"
|
||||||
"path"
|
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/filecatalog"
|
"gitea.maximumdirect.net/eric/promptkit/internal/filecatalog"
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/jsonvalue"
|
||||||
"gopkg.in/yaml.v3"
|
"gopkg.in/yaml.v3"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -73,7 +74,8 @@ func (r *overlayRepository) GetProfile(ctx context.Context, id string) (*domain.
|
|||||||
}
|
}
|
||||||
|
|
||||||
func loadProfile(ctx context.Context, fsys fs.FS, root string, id string) (*domain.ExecutionProfile, error) {
|
func loadProfile(ctx context.Context, fsys fs.FS, root string, id string) (*domain.ExecutionProfile, error) {
|
||||||
if strings.TrimSpace(id) == "" {
|
id = strings.TrimSpace(id)
|
||||||
|
if id == "" {
|
||||||
return nil, fmt.Errorf("%w: profile id is required", ErrInvalidProfile)
|
return nil, fmt.Errorf("%w: profile id is required", ErrInvalidProfile)
|
||||||
}
|
}
|
||||||
if fsys == nil {
|
if fsys == nil {
|
||||||
@@ -94,41 +96,50 @@ func loadProfile(ctx context.Context, fsys fs.FS, root string, id string) (*doma
|
|||||||
}
|
}
|
||||||
|
|
||||||
relPath := filecatalog.DisplayPath(root, fullPath)
|
relPath := filecatalog.DisplayPath(root, fullPath)
|
||||||
fileMatch := filecatalog.Stem(path.Base(fullPath)) == id
|
|
||||||
data, err := fs.ReadFile(fsys, fullPath)
|
data, err := fs.ReadFile(fsys, fullPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("failed to read profile file %s: %w", relPath, err)
|
return nil, fmt.Errorf("failed to read profile file %s: %w", relPath, err)
|
||||||
}
|
}
|
||||||
metadata := readProfileFileMetadata(data)
|
metadata, metadataErr := readProfileFileMetadata(data)
|
||||||
idMatch := fileMatch || metadata.id == id
|
idMatch := metadata.matchesID(id)
|
||||||
|
if metadataErr != nil {
|
||||||
|
if idMatch {
|
||||||
|
return nil, fmt.Errorf("%w: %s: %v", ErrInvalidYAML, relPath, metadataErr)
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
if metadata.hasRawAPIKey {
|
if metadata.hasRawAPIKey {
|
||||||
if idMatch {
|
if idMatch {
|
||||||
return nil, fmt.Errorf("%w: %s", ErrRawAPIKeyNotAllowed, relPath)
|
return nil, fmt.Errorf("%w: %s", ErrRawAPIKeyNotAllowed, relPath)
|
||||||
}
|
}
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
if !idMatch {
|
||||||
var prof domain.ExecutionProfile
|
|
||||||
decoder := yaml.NewDecoder(bytes.NewReader(data))
|
|
||||||
decoder.KnownFields(true)
|
|
||||||
if err := decoder.Decode(&prof); err != nil {
|
|
||||||
if idMatch {
|
|
||||||
return nil, fmt.Errorf("%w: %s: %v", ErrInvalidYAML, relPath, err)
|
|
||||||
}
|
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
|
prof, err := decodeProfile(data)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("%w: %s: %v", ErrInvalidYAML, relPath, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
prof.ID = strings.TrimSpace(prof.ID)
|
||||||
if prof.ID != id {
|
if prof.ID != id {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if err := validateProfile(&prof); err != nil {
|
prof.BackendID = strings.TrimSpace(prof.BackendID)
|
||||||
|
prof.ExtraParams, err = jsonvalue.CopyMap(prof.ExtraParams)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("%w: %s: %v", ErrInvalidProfile, relPath, err)
|
||||||
|
}
|
||||||
|
if err := normalizeAndValidateProfile(prof); err != nil {
|
||||||
if errors.Is(err, ErrRawAPIKeyNotAllowed) {
|
if errors.Is(err, ErrRawAPIKeyNotAllowed) {
|
||||||
return nil, fmt.Errorf("%w: %s", err, relPath)
|
return nil, fmt.Errorf("%w: %s", err, relPath)
|
||||||
}
|
}
|
||||||
return nil, fmt.Errorf("%w: %s: %v", ErrInvalidProfile, relPath, err)
|
return nil, fmt.Errorf("%w: %s: %v", ErrInvalidProfile, relPath, err)
|
||||||
}
|
}
|
||||||
matches = append(matches, profileMatch{
|
matches = append(matches, profileMatch{
|
||||||
profile: &prof,
|
profile: prof,
|
||||||
path: relPath,
|
path: relPath,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@@ -154,15 +165,36 @@ type profileMatch struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type profileFileMetadata struct {
|
type profileFileMetadata struct {
|
||||||
id string
|
ids []string
|
||||||
hasRawAPIKey bool
|
hasRawAPIKey bool
|
||||||
}
|
}
|
||||||
|
|
||||||
func readProfileFileMetadata(data []byte) profileFileMetadata {
|
func readProfileFileMetadata(data []byte) (profileFileMetadata, error) {
|
||||||
|
decoder := yaml.NewDecoder(bytes.NewReader(data))
|
||||||
var node yaml.Node
|
var node yaml.Node
|
||||||
if err := yaml.NewDecoder(bytes.NewReader(data)).Decode(&node); err != nil {
|
if err := decoder.Decode(&node); err != nil {
|
||||||
return profileFileMetadata{}
|
return profileFileMetadata{}, err
|
||||||
}
|
}
|
||||||
|
metadata := profileMetadataFromNode(&node)
|
||||||
|
documentCount := 1
|
||||||
|
for {
|
||||||
|
var trailing yaml.Node
|
||||||
|
err := decoder.Decode(&trailing)
|
||||||
|
if errors.Is(err, io.EOF) {
|
||||||
|
if documentCount == 1 {
|
||||||
|
return metadata, nil
|
||||||
|
}
|
||||||
|
return metadata, errors.New("profile file must contain exactly one YAML document")
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return metadata, err
|
||||||
|
}
|
||||||
|
documentCount++
|
||||||
|
metadata.merge(profileMetadataFromNode(&trailing))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func profileMetadataFromNode(node *yaml.Node) profileFileMetadata {
|
||||||
if node.Kind != yaml.DocumentNode || len(node.Content) == 0 {
|
if node.Kind != yaml.DocumentNode || len(node.Content) == 0 {
|
||||||
return profileFileMetadata{}
|
return profileFileMetadata{}
|
||||||
}
|
}
|
||||||
@@ -177,7 +209,7 @@ func readProfileFileMetadata(data []byte) profileFileMetadata {
|
|||||||
value := mapping.Content[i+1]
|
value := mapping.Content[i+1]
|
||||||
switch key.Value {
|
switch key.Value {
|
||||||
case "id":
|
case "id":
|
||||||
metadata.id = strings.TrimSpace(value.Value)
|
metadata.ids = append(metadata.ids, strings.TrimSpace(value.Value))
|
||||||
case "api_key":
|
case "api_key":
|
||||||
metadata.hasRawAPIKey = true
|
metadata.hasRawAPIKey = true
|
||||||
}
|
}
|
||||||
@@ -185,29 +217,68 @@ func readProfileFileMetadata(data []byte) profileFileMetadata {
|
|||||||
return metadata
|
return metadata
|
||||||
}
|
}
|
||||||
|
|
||||||
func validateProfile(p *domain.ExecutionProfile) error {
|
func (m profileFileMetadata) matchesID(id string) bool {
|
||||||
|
for _, candidate := range m.ids {
|
||||||
|
if candidate == id {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *profileFileMetadata) merge(other profileFileMetadata) {
|
||||||
|
m.ids = append(m.ids, other.ids...)
|
||||||
|
m.hasRawAPIKey = m.hasRawAPIKey || other.hasRawAPIKey
|
||||||
|
}
|
||||||
|
|
||||||
|
func decodeProfile(data []byte) (*domain.ExecutionProfile, error) {
|
||||||
|
var prof domain.ExecutionProfile
|
||||||
|
decoder := yaml.NewDecoder(bytes.NewReader(data))
|
||||||
|
decoder.KnownFields(true)
|
||||||
|
if err := decoder.Decode(&prof); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if err := requireYAMLStreamEnd(decoder); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return &prof, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func requireYAMLStreamEnd(decoder *yaml.Decoder) error {
|
||||||
|
var trailing yaml.Node
|
||||||
|
err := decoder.Decode(&trailing)
|
||||||
|
if errors.Is(err, io.EOF) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return errors.New("profile file must contain exactly one YAML document")
|
||||||
|
}
|
||||||
|
|
||||||
|
func normalizeAndValidateProfile(p *domain.ExecutionProfile) error {
|
||||||
if strings.TrimSpace(p.ID) == "" {
|
if strings.TrimSpace(p.ID) == "" {
|
||||||
return errors.New("id is required")
|
return errors.New("id is required")
|
||||||
}
|
}
|
||||||
if strings.TrimSpace(p.Endpoint) == "" {
|
p.Endpoint = strings.TrimSpace(p.Endpoint)
|
||||||
return errors.New("endpoint is required")
|
if strings.TrimSpace(p.BackendID) == "" && p.Endpoint == "" {
|
||||||
|
return errors.New("backend or endpoint is required")
|
||||||
|
}
|
||||||
|
if p.Endpoint != "" {
|
||||||
|
endpoint, err := domain.NormalizeOpenAICompatibleBaseEndpoint(p.Endpoint)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
p.Endpoint = endpoint
|
||||||
}
|
}
|
||||||
if strings.TrimSpace(p.Model) == "" {
|
if strings.TrimSpace(p.Model) == "" {
|
||||||
return errors.New("model is required")
|
return errors.New("model is required")
|
||||||
}
|
}
|
||||||
|
|
||||||
if p.Temperature < 0 || p.Temperature > 2 {
|
return domain.ValidateExecutionTargetSettings(domain.ExecutionTarget{
|
||||||
return errors.New("temperature must be between 0 and 2")
|
Temperature: p.Temperature,
|
||||||
}
|
MaxTokens: p.MaxTokens,
|
||||||
if p.MaxTokens < 0 {
|
TopP: p.TopP,
|
||||||
return errors.New("max_tokens must be greater than or equal to 0")
|
TimeoutSeconds: p.TimeoutSeconds,
|
||||||
}
|
})
|
||||||
if p.TopP < 0 || p.TopP > 1 {
|
|
||||||
return errors.New("top_p must be between 0 and 1")
|
|
||||||
}
|
|
||||||
if p.TimeoutSeconds < 0 {
|
|
||||||
return errors.New("timeout_seconds must be greater than or equal to 0")
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
|
|||||||
58
internal/profile/repository_benchmark_test.go
Normal file
58
internal/profile/repository_benchmark_test.go
Normal file
@@ -0,0 +1,58 @@
|
|||||||
|
package profile
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"testing"
|
||||||
|
"testing/fstest"
|
||||||
|
)
|
||||||
|
|
||||||
|
func BenchmarkProfileRepositoryLookup(b *testing.B) {
|
||||||
|
for _, size := range []int{10, 1000} {
|
||||||
|
b.Run(fmt.Sprintf("catalog-%d", size), func(b *testing.B) {
|
||||||
|
files := fstest.MapFS{
|
||||||
|
"target.yaml": profileMapFile(`
|
||||||
|
id: target
|
||||||
|
endpoint: http://localhost:8000/v1
|
||||||
|
model: target-model
|
||||||
|
extra_params:
|
||||||
|
selected: true
|
||||||
|
`),
|
||||||
|
}
|
||||||
|
metadataNames := []string{"target.yaml"}
|
||||||
|
for i := 1; i < size; i++ {
|
||||||
|
name := fmt.Sprintf("profile-%04d.yaml", i)
|
||||||
|
files[name] = profileMapFile(fmt.Sprintf(`
|
||||||
|
id: profile-%04d
|
||||||
|
endpoint: http://localhost:8000/v1
|
||||||
|
model: unrelated-model
|
||||||
|
temperature: 0.5
|
||||||
|
max_tokens: 500
|
||||||
|
extra_params:
|
||||||
|
provider:
|
||||||
|
order:
|
||||||
|
- first
|
||||||
|
- second
|
||||||
|
`, i))
|
||||||
|
metadataNames = append(metadataNames, name)
|
||||||
|
}
|
||||||
|
|
||||||
|
fsys := &recordingProfileFS{FS: files}
|
||||||
|
repo := NewFSRepository(fsys, ".")
|
||||||
|
b.ReportAllocs()
|
||||||
|
b.ResetTimer()
|
||||||
|
for i := 0; i < b.N; i++ {
|
||||||
|
if _, err := repo.GetProfile(context.Background(), "target"); err != nil {
|
||||||
|
b.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
b.StopTimer()
|
||||||
|
|
||||||
|
for _, name := range metadataNames {
|
||||||
|
if got := fsys.openCount(name); got != b.N {
|
||||||
|
b.Fatalf("metadata %q opens = %d, want %d", name, got, b.N)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -2,11 +2,13 @@ package profile
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io/fs"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
|
"sync"
|
||||||
"testing"
|
"testing"
|
||||||
"testing/fstest"
|
"testing/fstest"
|
||||||
|
|
||||||
@@ -52,6 +54,43 @@ func TestFilesystemRepository_GetProfile(t *testing.T) {
|
|||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
|
t.Run("backend and endpoint connection matrix", func(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
connection string
|
||||||
|
wantBackend string
|
||||||
|
wantEndpoint string
|
||||||
|
wantErr bool
|
||||||
|
}{
|
||||||
|
{name: "backend only", connection: "backend: ' openrouter '", wantBackend: "openrouter"},
|
||||||
|
{name: "endpoint only", connection: "endpoint: ' https://localhost:8000/nested/v1 '", wantEndpoint: "https://localhost:8000/nested/v1"},
|
||||||
|
{name: "both", connection: "backend: openrouter\nendpoint: http://localhost:8000/v1", wantBackend: "openrouter", wantEndpoint: "http://localhost:8000/v1"},
|
||||||
|
{name: "neither", wantErr: true},
|
||||||
|
{name: "blank backend", connection: "backend: ' '", wantErr: true},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
id := "connection-" + strings.ReplaceAll(tt.name, " ", "-")
|
||||||
|
writeProfileTestFile(t, filepath.Join(tmpDir, id+".yaml"), "id: "+id+"\nmodel: model\n"+tt.connection+"\n")
|
||||||
|
|
||||||
|
p, err := repo.GetProfile(ctx, id)
|
||||||
|
if tt.wantErr {
|
||||||
|
if !errors.Is(err, ErrInvalidProfile) {
|
||||||
|
t.Fatalf("expected ErrInvalidProfile, got %v", err)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected profile to load, got %v", err)
|
||||||
|
}
|
||||||
|
if p.BackendID != tt.wantBackend || p.Endpoint != tt.wantEndpoint {
|
||||||
|
t.Fatalf("unexpected connection values: backend=%q endpoint=%q", p.BackendID, p.Endpoint)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
t.Run("valid profile with api_key_env", func(t *testing.T) {
|
t.Run("valid profile with api_key_env", func(t *testing.T) {
|
||||||
p, err := repo.GetProfile(ctx, "local-secure")
|
p, err := repo.GetProfile(ctx, "local-secure")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -89,63 +128,6 @@ temperature: 0.1
|
|||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
t.Run("valid profile with JSON-compatible extra params", func(t *testing.T) {
|
|
||||||
writeProfileTestFile(t, filepath.Join(tmpDir, "json-extra-params.yaml"), `
|
|
||||||
id: json-extra-params
|
|
||||||
endpoint: http://localhost:8000/v1
|
|
||||||
model: nested-model
|
|
||||||
extra_params:
|
|
||||||
string_value: enabled
|
|
||||||
number_value: 42
|
|
||||||
boolean_value: true
|
|
||||||
object_value:
|
|
||||||
nested: value
|
|
||||||
count: 2
|
|
||||||
array_value:
|
|
||||||
- first
|
|
||||||
- 3
|
|
||||||
- false
|
|
||||||
`)
|
|
||||||
|
|
||||||
p, err := repo.GetProfile(ctx, "json-extra-params")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("expected no error, got %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
var got map[string]any
|
|
||||||
encoded, err := json.Marshal(p.ExtraParams)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("expected extra_params to marshal as JSON, got %v", err)
|
|
||||||
}
|
|
||||||
if err := json.Unmarshal(encoded, &got); err != nil {
|
|
||||||
t.Fatalf("expected extra_params JSON to decode, got %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if got["string_value"] != "enabled" {
|
|
||||||
t.Fatalf("unexpected string extra param: %#v", got["string_value"])
|
|
||||||
}
|
|
||||||
if got["number_value"] != float64(42) {
|
|
||||||
t.Fatalf("unexpected number extra param: %#v", got["number_value"])
|
|
||||||
}
|
|
||||||
if got["boolean_value"] != true {
|
|
||||||
t.Fatalf("unexpected boolean extra param: %#v", got["boolean_value"])
|
|
||||||
}
|
|
||||||
objectValue, ok := got["object_value"].(map[string]any)
|
|
||||||
if !ok {
|
|
||||||
t.Fatalf("expected object extra param, got %#v", got["object_value"])
|
|
||||||
}
|
|
||||||
if objectValue["nested"] != "value" || objectValue["count"] != float64(2) {
|
|
||||||
t.Fatalf("unexpected object extra param: %#v", objectValue)
|
|
||||||
}
|
|
||||||
arrayValue, ok := got["array_value"].([]any)
|
|
||||||
if !ok {
|
|
||||||
t.Fatalf("expected array extra param, got %#v", got["array_value"])
|
|
||||||
}
|
|
||||||
if len(arrayValue) != 3 || arrayValue[0] != "first" || arrayValue[1] != float64(3) || arrayValue[2] != false {
|
|
||||||
t.Fatalf("unexpected array extra param: %#v", arrayValue)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("duplicate profile IDs fail as ambiguous", func(t *testing.T) {
|
t.Run("duplicate profile IDs fail as ambiguous", func(t *testing.T) {
|
||||||
writeProfileTestFile(t, filepath.Join(tmpDir, "duplicate-profile-a.yaml"), `
|
writeProfileTestFile(t, filepath.Join(tmpDir, "duplicate-profile-a.yaml"), `
|
||||||
id: duplicate-profile
|
id: duplicate-profile
|
||||||
@@ -208,10 +190,10 @@ api_key: secret
|
|||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
t.Run("invalid yaml", func(t *testing.T) {
|
t.Run("unidentifiable invalid yaml is unrelated", func(t *testing.T) {
|
||||||
_, err := repo.GetProfile(ctx, "invalid_yaml")
|
_, err := repo.GetProfile(ctx, "invalid_yaml")
|
||||||
if !errors.Is(err, ErrInvalidYAML) {
|
if !errors.Is(err, ErrProfileNotFound) {
|
||||||
t.Fatalf("expected ErrInvalidYAML, got %v", err)
|
t.Fatalf("expected ErrProfileNotFound, got %v", err)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
@@ -237,14 +219,14 @@ api_key: secret
|
|||||||
})
|
})
|
||||||
|
|
||||||
t.Run("unknown field", func(t *testing.T) {
|
t.Run("unknown field", func(t *testing.T) {
|
||||||
_, err := repo.GetProfile(ctx, "unknown_field")
|
_, err := repo.GetProfile(ctx, "unknown-field")
|
||||||
if !errors.Is(err, ErrInvalidYAML) {
|
if !errors.Is(err, ErrInvalidYAML) {
|
||||||
t.Fatalf("expected ErrInvalidYAML for strict decode unknown field, got %v", err)
|
t.Fatalf("expected ErrInvalidYAML for strict decode unknown field, got %v", err)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
t.Run("raw api_key rejected", func(t *testing.T) {
|
t.Run("raw api_key rejected", func(t *testing.T) {
|
||||||
_, err := repo.GetProfile(ctx, "raw_api_key")
|
_, err := repo.GetProfile(ctx, "raw-api-key")
|
||||||
if !errors.Is(err, ErrRawAPIKeyNotAllowed) {
|
if !errors.Is(err, ErrRawAPIKeyNotAllowed) {
|
||||||
t.Fatalf("expected ErrRawAPIKeyNotAllowed, got %v", err)
|
t.Fatalf("expected ErrRawAPIKeyNotAllowed, got %v", err)
|
||||||
}
|
}
|
||||||
@@ -369,6 +351,513 @@ model: second
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestProfileRepositoriesRejectInvalidEndpoints(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
endpoint string
|
||||||
|
withBackend bool
|
||||||
|
}{
|
||||||
|
{name: "relative", endpoint: "/v1"},
|
||||||
|
{name: "missing host", endpoint: "https:///v1"},
|
||||||
|
{name: "unsupported scheme", endpoint: "ftp://provider.example/v1"},
|
||||||
|
{name: "user information", endpoint: "https://user@provider.example/v1"},
|
||||||
|
{name: "query", endpoint: "https://provider.example/v1?mode=chat"},
|
||||||
|
{name: "fragment", endpoint: "https://provider.example/v1#chat"},
|
||||||
|
{name: "backend with invalid override", endpoint: "/v1", withBackend: true},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
backend := ""
|
||||||
|
if tc.withBackend {
|
||||||
|
backend = "backend: openrouter\n"
|
||||||
|
}
|
||||||
|
repo := NewFSRepository(fstest.MapFS{
|
||||||
|
"profiles/invalid.yaml": profileMapFile(fmt.Sprintf(
|
||||||
|
"id: invalid-endpoint\nmodel: model\n%sendpoint: %q\n",
|
||||||
|
backend,
|
||||||
|
tc.endpoint,
|
||||||
|
)),
|
||||||
|
}, "profiles")
|
||||||
|
|
||||||
|
_, err := repo.GetProfile(context.Background(), "invalid-endpoint")
|
||||||
|
if !errors.Is(err, ErrInvalidProfile) {
|
||||||
|
t.Fatalf("expected ErrInvalidProfile, got %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProfileRepositoriesValidateExtraParams(t *testing.T) {
|
||||||
|
const validProfile = `
|
||||||
|
id: selected-profile
|
||||||
|
endpoint: http://localhost:8000/v1
|
||||||
|
model: model
|
||||||
|
extra_params:
|
||||||
|
string_value: enabled
|
||||||
|
object_value:
|
||||||
|
nested: true
|
||||||
|
array_value:
|
||||||
|
- first
|
||||||
|
- 3
|
||||||
|
`
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
definition string
|
||||||
|
wantErr bool
|
||||||
|
diagnostics []string
|
||||||
|
}{
|
||||||
|
{name: "valid nested values", definition: validProfile},
|
||||||
|
{
|
||||||
|
name: "empty key",
|
||||||
|
definition: `
|
||||||
|
id: selected-profile
|
||||||
|
endpoint: http://localhost:8000/v1
|
||||||
|
model: model
|
||||||
|
extra_params:
|
||||||
|
"": value
|
||||||
|
`,
|
||||||
|
wantErr: true,
|
||||||
|
diagnostics: []string{"extra_params", "key must not be empty"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "non-finite value",
|
||||||
|
definition: `
|
||||||
|
id: selected-profile
|
||||||
|
endpoint: http://localhost:8000/v1
|
||||||
|
model: model
|
||||||
|
extra_params:
|
||||||
|
invalid: .nan
|
||||||
|
`,
|
||||||
|
wantErr: true,
|
||||||
|
diagnostics: []string{"extra_params.invalid", "must be finite"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "nested non-finite value",
|
||||||
|
definition: `
|
||||||
|
id: selected-profile
|
||||||
|
endpoint: http://localhost:8000/v1
|
||||||
|
model: model
|
||||||
|
extra_params:
|
||||||
|
outer:
|
||||||
|
invalid: .inf
|
||||||
|
`,
|
||||||
|
wantErr: true,
|
||||||
|
diagnostics: []string{"extra_params.outer.invalid", "must be finite"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "unsupported decoded value",
|
||||||
|
definition: `
|
||||||
|
id: selected-profile
|
||||||
|
endpoint: http://localhost:8000/v1
|
||||||
|
model: model
|
||||||
|
extra_params:
|
||||||
|
timestamp: 2026-08-11T12:34:56Z
|
||||||
|
`,
|
||||||
|
wantErr: true,
|
||||||
|
diagnostics: []string{"extra_params.timestamp", "unsupported JSON value type"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "excessive nesting",
|
||||||
|
definition: deeplyNestedExtraParamsProfile(101),
|
||||||
|
wantErr: true,
|
||||||
|
diagnostics: []string{"extra_params", "JSON container depth limit exceeded"},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, source := range profileRepositorySources() {
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(source.name+"/"+tc.name, func(t *testing.T) {
|
||||||
|
repo := source.newRepository(t, map[string]string{"selected.yaml": tc.definition})
|
||||||
|
got, err := repo.GetProfile(context.Background(), "selected-profile")
|
||||||
|
if tc.wantErr {
|
||||||
|
if !errors.Is(err, ErrInvalidProfile) {
|
||||||
|
t.Fatalf("expected ErrInvalidProfile, got %v", err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "selected.yaml") {
|
||||||
|
t.Fatalf("expected source path in error, got %v", err)
|
||||||
|
}
|
||||||
|
for _, diagnostic := range tc.diagnostics {
|
||||||
|
if !strings.Contains(err.Error(), diagnostic) {
|
||||||
|
t.Fatalf("expected error to contain %q, got %v", diagnostic, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("load valid profile: %v", err)
|
||||||
|
}
|
||||||
|
if got.ExtraParams["string_value"] != "enabled" {
|
||||||
|
t.Fatalf("unexpected copied extra params: %#v", got.ExtraParams)
|
||||||
|
}
|
||||||
|
objectValue, objectOK := got.ExtraParams["object_value"].(map[string]any)
|
||||||
|
arrayValue, arrayOK := got.ExtraParams["array_value"].([]any)
|
||||||
|
if !objectOK || objectValue["nested"] != true ||
|
||||||
|
!arrayOK || len(arrayValue) != 2 || arrayValue[0] != "first" || arrayValue[1] != 3 {
|
||||||
|
t.Fatalf("unexpected copied nested extra params: %#v", got.ExtraParams)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProfileRepositoriesSelectCanonicalYAMLID(t *testing.T) {
|
||||||
|
const validProfile = `
|
||||||
|
id: selected-profile
|
||||||
|
endpoint: http://localhost:8000/v1
|
||||||
|
model: selected-model
|
||||||
|
`
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
files map[string]string
|
||||||
|
wantErr error
|
||||||
|
diagnostics []string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "same stem unknown field with different id is unrelated",
|
||||||
|
files: map[string]string{
|
||||||
|
"selected-profile.yaml": `
|
||||||
|
id: unrelated-profile
|
||||||
|
endpoint: http://localhost:8000/v1
|
||||||
|
model: unrelated
|
||||||
|
unknown: true
|
||||||
|
`,
|
||||||
|
"valid.yaml": validProfile,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "same stem unidentifiable yaml is unrelated",
|
||||||
|
files: map[string]string{
|
||||||
|
"selected-profile.yaml": "id: [",
|
||||||
|
"valid.yaml": validProfile,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "same stem raw key with different id is unrelated",
|
||||||
|
files: map[string]string{
|
||||||
|
"selected-profile.yaml": `
|
||||||
|
id: unrelated-profile
|
||||||
|
endpoint: http://localhost:8000/v1
|
||||||
|
model: unrelated
|
||||||
|
api_key: secret
|
||||||
|
`,
|
||||||
|
"valid.yaml": validProfile,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "leading and trailing whitespace is normalized",
|
||||||
|
files: map[string]string{
|
||||||
|
"padded.yaml": `
|
||||||
|
id: " selected-profile "
|
||||||
|
endpoint: http://localhost:8000/v1
|
||||||
|
model: selected-model
|
||||||
|
`,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "blank id is unrelated",
|
||||||
|
files: map[string]string{
|
||||||
|
"selected-profile.yaml": `
|
||||||
|
id: " "
|
||||||
|
endpoint: http://localhost:8000/v1
|
||||||
|
model: unrelated
|
||||||
|
`,
|
||||||
|
},
|
||||||
|
wantErr: ErrProfileNotFound,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "normalized duplicates are ambiguous",
|
||||||
|
files: map[string]string{
|
||||||
|
"first.yaml": validProfile,
|
||||||
|
"nested/second.yaml": `
|
||||||
|
id: " selected-profile "
|
||||||
|
endpoint: http://localhost:8000/v1
|
||||||
|
model: duplicate
|
||||||
|
`,
|
||||||
|
},
|
||||||
|
wantErr: ErrInvalidProfile,
|
||||||
|
diagnostics: []string{"duplicate execution profile id", "first.yaml", "nested/second.yaml"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "selected unknown field is authoritative",
|
||||||
|
files: map[string]string{
|
||||||
|
"malformed.yaml": `
|
||||||
|
id: selected-profile
|
||||||
|
endpoint: http://localhost:8000/v1
|
||||||
|
model: selected-model
|
||||||
|
unknown: true
|
||||||
|
`,
|
||||||
|
},
|
||||||
|
wantErr: ErrInvalidYAML,
|
||||||
|
diagnostics: []string{"malformed.yaml"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "selected raw key is authoritative",
|
||||||
|
files: map[string]string{
|
||||||
|
"insecure.yaml": `
|
||||||
|
id: selected-profile
|
||||||
|
endpoint: http://localhost:8000/v1
|
||||||
|
model: selected-model
|
||||||
|
api_key: secret
|
||||||
|
`,
|
||||||
|
},
|
||||||
|
wantErr: ErrRawAPIKeyNotAllowed,
|
||||||
|
diagnostics: []string{"insecure.yaml"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "selected identity in an additional document is authoritative",
|
||||||
|
files: map[string]string{
|
||||||
|
"additional-document.yaml": `
|
||||||
|
---
|
||||||
|
---
|
||||||
|
id: selected-profile
|
||||||
|
endpoint: http://localhost:8000/v1
|
||||||
|
model: selected-model
|
||||||
|
`,
|
||||||
|
},
|
||||||
|
wantErr: ErrInvalidYAML,
|
||||||
|
diagnostics: []string{"additional-document.yaml", "exactly one YAML document"},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, source := range profileRepositorySources() {
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(source.name+"/"+tc.name, func(t *testing.T) {
|
||||||
|
repo := source.newRepository(t, tc.files)
|
||||||
|
got, err := repo.GetProfile(context.Background(), " selected-profile ")
|
||||||
|
if tc.wantErr != nil {
|
||||||
|
if !errors.Is(err, tc.wantErr) {
|
||||||
|
t.Fatalf("expected %v, got %v", tc.wantErr, err)
|
||||||
|
}
|
||||||
|
for _, diagnostic := range tc.diagnostics {
|
||||||
|
if !strings.Contains(err.Error(), diagnostic) {
|
||||||
|
t.Fatalf("expected error to contain %q, got %v", diagnostic, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("load selected profile: %v", err)
|
||||||
|
}
|
||||||
|
if got.ID != "selected-profile" || got.Model != "selected-model" {
|
||||||
|
t.Fatalf("unexpected selected profile: %+v", got)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProfileRepositoriesRequireOneYAMLDocument(t *testing.T) {
|
||||||
|
const profile = `
|
||||||
|
id: selected-profile
|
||||||
|
endpoint: http://localhost:8000/v1
|
||||||
|
model: selected-model
|
||||||
|
`
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
suffix string
|
||||||
|
wantErr bool
|
||||||
|
}{
|
||||||
|
{name: "comments and trailing whitespace", suffix: "\n# trailing comment\n\n"},
|
||||||
|
{name: "second populated document", suffix: "\n---\nid: another\n", wantErr: true},
|
||||||
|
{name: "second empty document", suffix: "\n---\n", wantErr: true},
|
||||||
|
{name: "malformed trailing yaml", suffix: "\n---\n[", wantErr: true},
|
||||||
|
{name: "raw key in trailing document", suffix: "\n---\napi_key: secret\n", wantErr: true},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, source := range profileRepositorySources() {
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(source.name+"/"+tc.name, func(t *testing.T) {
|
||||||
|
repo := source.newRepository(t, map[string]string{"definition.yaml": profile + tc.suffix})
|
||||||
|
got, err := repo.GetProfile(context.Background(), "selected-profile")
|
||||||
|
if tc.wantErr {
|
||||||
|
if !errors.Is(err, ErrInvalidYAML) {
|
||||||
|
t.Fatalf("expected ErrInvalidYAML, got %v", err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "definition.yaml") {
|
||||||
|
t.Fatalf("expected source path in error, got %v", err)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("load one-document profile: %v", err)
|
||||||
|
}
|
||||||
|
if got.ID != "selected-profile" {
|
||||||
|
t.Fatalf("unexpected profile: %+v", got)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProfileRepositoriesPreserveOverlayFallbackRules(t *testing.T) {
|
||||||
|
fallback := staticProfileRepo{profiles: map[string]*domain.ExecutionProfile{
|
||||||
|
"selected-profile": {ID: "selected-profile", Endpoint: "http://fallback", Model: "fallback-model"},
|
||||||
|
}}
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
files map[string]string
|
||||||
|
wantModel string
|
||||||
|
wantErr error
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "same stem malformed different id falls back",
|
||||||
|
files: map[string]string{
|
||||||
|
"selected-profile.yaml": `
|
||||||
|
id: unrelated-profile
|
||||||
|
endpoint: http://localhost:8000/v1
|
||||||
|
model: unrelated
|
||||||
|
unknown: true
|
||||||
|
`,
|
||||||
|
},
|
||||||
|
wantModel: "fallback-model",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "blank id falls back",
|
||||||
|
files: map[string]string{
|
||||||
|
"selected-profile.yaml": `
|
||||||
|
id: " "
|
||||||
|
endpoint: http://localhost:8000/v1
|
||||||
|
model: unrelated
|
||||||
|
`,
|
||||||
|
},
|
||||||
|
wantModel: "fallback-model",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "selected malformed profile stops fallback",
|
||||||
|
files: map[string]string{
|
||||||
|
"other-name.yaml": `
|
||||||
|
id: selected-profile
|
||||||
|
endpoint: http://localhost:8000/v1
|
||||||
|
model: selected
|
||||||
|
unknown: true
|
||||||
|
`,
|
||||||
|
},
|
||||||
|
wantErr: ErrInvalidYAML,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, source := range profileRepositorySources() {
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(source.name+"/"+tc.name, func(t *testing.T) {
|
||||||
|
primary := source.newRepository(t, tc.files)
|
||||||
|
got, err := NewOverlayRepository(primary, fallback).GetProfile(context.Background(), "selected-profile")
|
||||||
|
if tc.wantErr != nil {
|
||||||
|
if !errors.Is(err, tc.wantErr) {
|
||||||
|
t.Fatalf("expected %v, got %v", tc.wantErr, err)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("load fallback profile: %v", err)
|
||||||
|
}
|
||||||
|
if got.Model != tc.wantModel {
|
||||||
|
t.Fatalf("model = %q, want %q", got.Model, tc.wantModel)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProfileRepositoryReadsSourcesFreshOnEveryLookup(t *testing.T) {
|
||||||
|
newSource := func() (*recordingProfileFS, Repository) {
|
||||||
|
fsys := &recordingProfileFS{FS: fstest.MapFS{
|
||||||
|
"target.yaml": profileMapFile(`
|
||||||
|
id: target
|
||||||
|
endpoint: http://localhost:8000/v1
|
||||||
|
model: target-model
|
||||||
|
`),
|
||||||
|
"unrelated.yaml": profileMapFile(`
|
||||||
|
id: unrelated
|
||||||
|
endpoint: http://localhost:8000/v1
|
||||||
|
model: unrelated-model
|
||||||
|
`),
|
||||||
|
}}
|
||||||
|
return fsys, NewFSRepository(fsys, ".")
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Run("selected source", func(t *testing.T) {
|
||||||
|
fsys, repo := newSource()
|
||||||
|
for lookup := 1; lookup <= 2; lookup++ {
|
||||||
|
got, err := repo.GetProfile(context.Background(), "target")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("lookup %d: %v", lookup, err)
|
||||||
|
}
|
||||||
|
if got.Model != "target-model" {
|
||||||
|
t.Fatalf("lookup %d model = %q", lookup, got.Model)
|
||||||
|
}
|
||||||
|
for _, name := range []string{"target.yaml", "unrelated.yaml"} {
|
||||||
|
if count := fsys.openCount(name); count != lookup {
|
||||||
|
t.Fatalf("%s opens after lookup %d = %d, want %d", name, lookup, count, lookup)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("overlay fallthrough", func(t *testing.T) {
|
||||||
|
primaryFS := &recordingProfileFS{FS: fstest.MapFS{
|
||||||
|
"unrelated.yaml": profileMapFile(`
|
||||||
|
id: unrelated
|
||||||
|
endpoint: http://localhost:8000/v1
|
||||||
|
model: unrelated-model
|
||||||
|
`),
|
||||||
|
}}
|
||||||
|
fallbackFS, fallback := newSource()
|
||||||
|
repo := NewOverlayRepository(NewFSRepository(primaryFS, "."), fallback)
|
||||||
|
|
||||||
|
for lookup := 1; lookup <= 2; lookup++ {
|
||||||
|
got, err := repo.GetProfile(context.Background(), "target")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("lookup %d: %v", lookup, err)
|
||||||
|
}
|
||||||
|
if got.Model != "target-model" {
|
||||||
|
t.Fatalf("lookup %d model = %q", lookup, got.Model)
|
||||||
|
}
|
||||||
|
if count := primaryFS.openCount("unrelated.yaml"); count != lookup {
|
||||||
|
t.Fatalf("primary opens after lookup %d = %d, want %d", lookup, count, lookup)
|
||||||
|
}
|
||||||
|
if count := fallbackFS.openCount("target.yaml"); count != lookup {
|
||||||
|
t.Fatalf("fallback opens after lookup %d = %d, want %d", lookup, count, lookup)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProfileRepositoriesRejectInvalidExecutionSettings(t *testing.T) {
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
t.Run("operating-system filesystem", func(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
writeProfileTestFile(t, filepath.Join(dir, "invalid.yaml"), `
|
||||||
|
id: invalid
|
||||||
|
endpoint: http://localhost:8000/v1
|
||||||
|
model: model
|
||||||
|
temperature: .nan
|
||||||
|
`)
|
||||||
|
|
||||||
|
_, err := NewFilesystemRepository(dir).GetProfile(ctx, "invalid")
|
||||||
|
if !errors.Is(err, ErrInvalidProfile) {
|
||||||
|
t.Fatalf("expected ErrInvalidProfile, got %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("fs.FS", func(t *testing.T) {
|
||||||
|
repo := NewFSRepository(fstest.MapFS{
|
||||||
|
"profiles/invalid.yaml": profileMapFile(`
|
||||||
|
id: invalid
|
||||||
|
endpoint: http://localhost:8000/v1
|
||||||
|
model: model
|
||||||
|
top_p: .inf
|
||||||
|
`),
|
||||||
|
}, "profiles")
|
||||||
|
|
||||||
|
_, err := repo.GetProfile(ctx, "invalid")
|
||||||
|
if !errors.Is(err, ErrInvalidProfile) {
|
||||||
|
t.Fatalf("expected ErrInvalidProfile, got %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
func TestOverlayRepository(t *testing.T) {
|
func TestOverlayRepository(t *testing.T) {
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
primaryProfile := &domain.ExecutionProfile{ID: "shared", Endpoint: "http://primary", Model: "primary"}
|
primaryProfile := &domain.ExecutionProfile{ID: "shared", Endpoint: "http://primary", Model: "primary"}
|
||||||
@@ -458,6 +947,77 @@ func TestOverlayRepository(t *testing.T) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type profileRepositorySource struct {
|
||||||
|
name string
|
||||||
|
newRepository func(t *testing.T, files map[string]string) Repository
|
||||||
|
}
|
||||||
|
|
||||||
|
type recordingProfileFS struct {
|
||||||
|
fs.FS
|
||||||
|
mu sync.Mutex
|
||||||
|
opened []string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *recordingProfileFS) Open(name string) (fs.File, error) {
|
||||||
|
f.mu.Lock()
|
||||||
|
f.opened = append(f.opened, name)
|
||||||
|
f.mu.Unlock()
|
||||||
|
return f.FS.Open(name)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *recordingProfileFS) openCount(name string) int {
|
||||||
|
f.mu.Lock()
|
||||||
|
defer f.mu.Unlock()
|
||||||
|
count := 0
|
||||||
|
for _, opened := range f.opened {
|
||||||
|
if opened == name {
|
||||||
|
count++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return count
|
||||||
|
}
|
||||||
|
|
||||||
|
func profileRepositorySources() []profileRepositorySource {
|
||||||
|
return []profileRepositorySource{
|
||||||
|
{
|
||||||
|
name: "operating system",
|
||||||
|
newRepository: func(t *testing.T, files map[string]string) Repository {
|
||||||
|
t.Helper()
|
||||||
|
root := t.TempDir()
|
||||||
|
for name, content := range files {
|
||||||
|
filePath := filepath.Join(root, filepath.FromSlash(name))
|
||||||
|
if err := os.MkdirAll(filepath.Dir(filePath), 0o755); err != nil {
|
||||||
|
t.Fatalf("create profile directory: %v", err)
|
||||||
|
}
|
||||||
|
writeProfileTestFile(t, filePath, content)
|
||||||
|
}
|
||||||
|
return NewFilesystemRepository(root)
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "filesystem",
|
||||||
|
newRepository: func(t *testing.T, files map[string]string) Repository {
|
||||||
|
t.Helper()
|
||||||
|
fsys := make(fstest.MapFS, len(files))
|
||||||
|
for name, content := range files {
|
||||||
|
fsys[name] = profileMapFile(content)
|
||||||
|
}
|
||||||
|
return NewFSRepository(fsys, ".")
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func deeplyNestedExtraParamsProfile(depth int) string {
|
||||||
|
var definition strings.Builder
|
||||||
|
definition.WriteString("id: selected-profile\nendpoint: http://localhost:8000/v1\nmodel: model\nextra_params:\n")
|
||||||
|
for level := 0; level < depth; level++ {
|
||||||
|
fmt.Fprintf(&definition, "%slevel_%d:\n", strings.Repeat(" ", level+1), level)
|
||||||
|
}
|
||||||
|
fmt.Fprintf(&definition, "%svalue: true\n", strings.Repeat(" ", depth+1))
|
||||||
|
return definition.String()
|
||||||
|
}
|
||||||
|
|
||||||
func profileMapFile(content string) *fstest.MapFile {
|
func profileMapFile(content string) *fstest.MapFile {
|
||||||
return &fstest.MapFile{Data: []byte(strings.TrimLeft(content, "\n"))}
|
return &fstest.MapFile{Data: []byte(strings.TrimLeft(content, "\n"))}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -5,10 +5,10 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
|
||||||
"strings"
|
"strings"
|
||||||
"text/template"
|
"text/template"
|
||||||
"unicode/utf8"
|
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
@@ -19,6 +19,8 @@ var (
|
|||||||
ErrInvalidMessageRole = errors.New("invalid or empty message role")
|
ErrInvalidMessageRole = errors.New("invalid or empty message role")
|
||||||
)
|
)
|
||||||
|
|
||||||
|
const artifactTextChunkSize = 64 * 1024
|
||||||
|
|
||||||
type goRenderer struct{}
|
type goRenderer struct{}
|
||||||
|
|
||||||
func NewGoRenderer() Renderer {
|
func NewGoRenderer() Renderer {
|
||||||
@@ -26,11 +28,13 @@ func NewGoRenderer() Renderer {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (r *goRenderer) Render(ctx context.Context, definition *domain.PromptDefinition, inputs map[string]*domain.Artifact, vars map[string]string) (*domain.RenderedPrompt, error) {
|
func (r *goRenderer) Render(ctx context.Context, definition *domain.PromptDefinition, inputs map[string]*domain.Artifact, vars map[string]string) (*domain.RenderedPrompt, error) {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
if definition == nil {
|
if definition == nil {
|
||||||
return nil, fmt.Errorf("%w: nil prompt definition", ErrRenderFailure)
|
return nil, fmt.Errorf("%w: nil prompt definition", ErrRenderFailure)
|
||||||
}
|
}
|
||||||
|
|
||||||
// 1. Verify required inputs
|
|
||||||
for _, in := range definition.Inputs {
|
for _, in := range definition.Inputs {
|
||||||
if !in.Required {
|
if !in.Required {
|
||||||
continue
|
continue
|
||||||
@@ -41,44 +45,54 @@ func (r *goRenderer) Render(ctx context.Context, definition *domain.PromptDefini
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// 2. Setup template functions
|
resolver := newArtifactTextResolver(ctx, inputs)
|
||||||
funcs := template.FuncMap{
|
funcs := template.FuncMap{
|
||||||
"input": func(name string) (string, error) {
|
"input": resolver.resolve,
|
||||||
art, ok := inputs[name]
|
|
||||||
if !ok || art == nil {
|
|
||||||
return "", fmt.Errorf("%w: %s", ErrUnknownInput, name)
|
|
||||||
}
|
|
||||||
return string(art.Body), nil
|
|
||||||
},
|
|
||||||
}
|
}
|
||||||
|
|
||||||
sessionID, err := renderSessionID(definition.SessionID, funcs, vars)
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
sessionID, err := renderSessionID(ctx, definition.SessionID, funcs, vars)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
var renderedMessages []domain.RenderedMessage
|
renderedMessages := make([]domain.RenderedMessage, 0, len(definition.Templates))
|
||||||
|
|
||||||
for i, tmplMsg := range definition.Templates {
|
for i, tmplMsg := range definition.Templates {
|
||||||
select {
|
if err := ctx.Err(); err != nil {
|
||||||
case <-ctx.Done():
|
return nil, err
|
||||||
return nil, ctx.Err()
|
|
||||||
default:
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if tmplMsg.Role == "" {
|
if tmplMsg.Role == "" {
|
||||||
return nil, fmt.Errorf("%w: message %d", ErrInvalidMessageRole, i)
|
return nil, fmt.Errorf("%w: message %d", ErrInvalidMessageRole, i)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Parse and execute template
|
if err := ctx.Err(); err != nil {
|
||||||
tmpl, err := template.New(fmt.Sprintf("msg_%d", i)).Funcs(funcs).Option("missingkey=error").Parse(tmplMsg.Content)
|
return nil, err
|
||||||
if err != nil {
|
}
|
||||||
return nil, fmt.Errorf("%w: message %d: %v", ErrInvalidTemplate, i, err)
|
tmpl, parseErr := template.New(fmt.Sprintf("msg_%d", i)).Funcs(funcs).Option("missingkey=error").Parse(tmplMsg.Content)
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if parseErr != nil {
|
||||||
|
return nil, fmt.Errorf("%w: message %d: %v", ErrInvalidTemplate, i, parseErr)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
var buf bytes.Buffer
|
var buf bytes.Buffer
|
||||||
if err := tmpl.Execute(&buf, vars); err != nil {
|
executeErr := tmpl.Execute(&buf, vars)
|
||||||
return nil, fmt.Errorf("%w: message %d: %w", ErrRenderFailure, i, err)
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if executeErr != nil {
|
||||||
|
return nil, fmt.Errorf("%w: message %d: %w", ErrRenderFailure, i, executeErr)
|
||||||
}
|
}
|
||||||
|
|
||||||
renderedMessages = append(renderedMessages, domain.RenderedMessage{
|
renderedMessages = append(renderedMessages, domain.RenderedMessage{
|
||||||
@@ -86,6 +100,13 @@ func (r *goRenderer) Render(ctx context.Context, definition *domain.PromptDefini
|
|||||||
Content: buf.String(),
|
Content: buf.String(),
|
||||||
CacheControl: cloneCacheControl(tmplMsg.CacheControl),
|
CacheControl: cloneCacheControl(tmplMsg.CacheControl),
|
||||||
})
|
})
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
return &domain.RenderedPrompt{
|
return &domain.RenderedPrompt{
|
||||||
@@ -94,24 +115,80 @@ func (r *goRenderer) Render(ctx context.Context, definition *domain.PromptDefini
|
|||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func renderSessionID(raw string, funcs template.FuncMap, vars map[string]string) (string, error) {
|
type artifactTextResolver struct {
|
||||||
if strings.TrimSpace(raw) == "" {
|
ctx context.Context
|
||||||
return "", nil
|
inputs map[string]*domain.Artifact
|
||||||
|
textByName map[string]string
|
||||||
|
}
|
||||||
|
|
||||||
|
func newArtifactTextResolver(ctx context.Context, inputs map[string]*domain.Artifact) *artifactTextResolver {
|
||||||
|
return &artifactTextResolver{
|
||||||
|
ctx: ctx,
|
||||||
|
inputs: inputs,
|
||||||
|
textByName: make(map[string]string),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *artifactTextResolver) resolve(name string) (string, error) {
|
||||||
|
if err := r.ctx.Err(); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
artifact, ok := r.inputs[name]
|
||||||
|
if !ok || artifact == nil {
|
||||||
|
return "", fmt.Errorf("%w: %s", ErrUnknownInput, name)
|
||||||
|
}
|
||||||
|
if text, ok := r.textByName[name]; ok {
|
||||||
|
return text, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
tmpl, err := template.New("session_id").Funcs(funcs).Option("missingkey=error").Parse(raw)
|
var builder strings.Builder
|
||||||
if err != nil {
|
builder.Grow(len(artifact.Body))
|
||||||
return "", fmt.Errorf("%w: session_id: %v", ErrInvalidTemplate, err)
|
for start := 0; start < len(artifact.Body); start += artifactTextChunkSize {
|
||||||
|
if err := r.ctx.Err(); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
end := min(start+artifactTextChunkSize, len(artifact.Body))
|
||||||
|
_, _ = builder.Write(artifact.Body[start:end])
|
||||||
|
if err := r.ctx.Err(); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := r.ctx.Err(); err != nil {
|
||||||
|
return "", err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
text := builder.String()
|
||||||
|
r.textByName[name] = text
|
||||||
|
return text, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func renderSessionID(ctx context.Context, raw string, funcs template.FuncMap, vars map[string]string) (string, error) {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
tmpl, parseErr := template.New("session_id").Funcs(funcs).Option("missingkey=error").Parse(raw)
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
if parseErr != nil {
|
||||||
|
return "", fmt.Errorf("%w: session_id: %v", ErrInvalidTemplate, parseErr)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
var buf bytes.Buffer
|
var buf bytes.Buffer
|
||||||
if err := tmpl.Execute(&buf, vars); err != nil {
|
executeErr := tmpl.Execute(&buf, vars)
|
||||||
return "", fmt.Errorf("%w: session_id: %w", ErrRenderFailure, err)
|
if err := ctx.Err(); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
if executeErr != nil {
|
||||||
|
return "", fmt.Errorf("%w: session_id: %w", ErrRenderFailure, executeErr)
|
||||||
}
|
}
|
||||||
|
|
||||||
sessionID := strings.TrimSpace(buf.String())
|
sessionID, err := domain.NormalizeSessionID(buf.String())
|
||||||
if n := utf8.RuneCountInString(sessionID); n > domain.SessionIDMaxLength {
|
if err != nil {
|
||||||
return "", fmt.Errorf("%w: session_id length %d exceeds maximum %d", ErrRenderFailure, n, domain.SessionIDMaxLength)
|
return "", fmt.Errorf("%w: session_id: %v", ErrRenderFailure, err)
|
||||||
}
|
}
|
||||||
return sessionID, nil
|
return sessionID, nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
package prompt
|
package prompt
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
"strings"
|
"strings"
|
||||||
@@ -228,6 +229,24 @@ func TestGoRenderer_Render(t *testing.T) {
|
|||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
|
t.Run("malformed rendered session id fails rendering", func(t *testing.T) {
|
||||||
|
def := &domain.PromptDefinition{
|
||||||
|
SessionID: "{{ .session_id }}",
|
||||||
|
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
|
||||||
|
Templates: []domain.PromptMessageTemplate{
|
||||||
|
{Role: "system", Content: "Speak in a {{.tone}} tone."},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err := renderer.Render(ctx, def, inputs, map[string]string{
|
||||||
|
"tone": "concise",
|
||||||
|
"session_id": "session" + string([]byte{0xff}),
|
||||||
|
})
|
||||||
|
if !errors.Is(err, ErrRenderFailure) {
|
||||||
|
t.Fatalf("expected ErrRenderFailure, got %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
t.Run("inserting required input artifact", func(t *testing.T) {
|
t.Run("inserting required input artifact", func(t *testing.T) {
|
||||||
def := &domain.PromptDefinition{
|
def := &domain.PromptDefinition{
|
||||||
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
|
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
|
||||||
@@ -343,3 +362,186 @@ func TestGoRenderer_Render(t *testing.T) {
|
|||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestGoRendererCancellation(t *testing.T) {
|
||||||
|
t.Run("before session parsing", func(t *testing.T) {
|
||||||
|
definition := &domain.PromptDefinition{
|
||||||
|
SessionID: "{{ malformed",
|
||||||
|
Templates: []domain.PromptMessageTemplate{
|
||||||
|
{Role: "user", Content: "not rendered"},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
cancel()
|
||||||
|
|
||||||
|
result, err := NewGoRenderer().Render(ctx, definition, nil, nil)
|
||||||
|
if result != nil || !errors.Is(err, context.Canceled) {
|
||||||
|
t.Fatalf("result=%#v err=%v, want nil/context.Canceled", result, err)
|
||||||
|
}
|
||||||
|
if errors.Is(err, ErrInvalidTemplate) {
|
||||||
|
t.Fatalf("pre-canceled render parsed the malformed session: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
result, err = NewGoRenderer().Render(context.Background(), definition, nil, nil)
|
||||||
|
if result != nil || !errors.Is(err, ErrInvalidTemplate) {
|
||||||
|
t.Fatalf("active render result=%#v err=%v, want nil/ErrInvalidTemplate", result, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("during artifact text conversion", func(t *testing.T) {
|
||||||
|
ctx := newCancelOnCheckContext(3)
|
||||||
|
body := bytes.Repeat([]byte("x"), artifactTextChunkSize*2)
|
||||||
|
original := append([]byte(nil), body...)
|
||||||
|
resolver := newArtifactTextResolver(ctx, map[string]*domain.Artifact{
|
||||||
|
"document": {Body: body},
|
||||||
|
})
|
||||||
|
|
||||||
|
text, err := resolver.resolve("document")
|
||||||
|
if text != "" || !errors.Is(err, context.Canceled) {
|
||||||
|
t.Fatalf("text length=%d err=%v, want empty/context.Canceled", len(text), err)
|
||||||
|
}
|
||||||
|
if _, published := resolver.textByName["document"]; published {
|
||||||
|
t.Fatal("canceled conversion published partial artifact text")
|
||||||
|
}
|
||||||
|
if !bytes.Equal(body, original) {
|
||||||
|
t.Fatal("resolver mutated the artifact body")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("after final message execution", func(t *testing.T) {
|
||||||
|
definition := &domain.PromptDefinition{
|
||||||
|
Templates: []domain.PromptMessageTemplate{
|
||||||
|
{Role: "user", Content: "fully rendered"},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
counter := &checkCountingContext{Context: context.Background()}
|
||||||
|
if _, err := NewGoRenderer().Render(counter, definition, nil, nil); err != nil {
|
||||||
|
t.Fatalf("count render checkpoints: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The final three checks occur after template execution, after the
|
||||||
|
// message is assembled, and immediately before publication.
|
||||||
|
ctx := newCancelOnCheckContext(counter.checks - 2)
|
||||||
|
result, err := NewGoRenderer().Render(ctx, definition, nil, nil)
|
||||||
|
if result != nil || !errors.Is(err, context.Canceled) {
|
||||||
|
t.Fatalf("result=%#v err=%v, want nil/context.Canceled", result, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGoRendererArtifactTextLifecycle(t *testing.T) {
|
||||||
|
body := []byte{'a', 0xff, 'b', 0xfe}
|
||||||
|
original := append([]byte(nil), body...)
|
||||||
|
artifact := &domain.Artifact{Body: body}
|
||||||
|
inputs := map[string]*domain.Artifact{"document": artifact}
|
||||||
|
definition := &domain.PromptDefinition{
|
||||||
|
Templates: []domain.PromptMessageTemplate{
|
||||||
|
{Role: "user", Content: "{{input \"document\"}}|{{input \"document\"}}"},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
first, err := NewGoRenderer().Render(context.Background(), definition, inputs, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("first render: %v", err)
|
||||||
|
}
|
||||||
|
wantFirst := append(append(append([]byte(nil), body...), '|'), body...)
|
||||||
|
if !bytes.Equal([]byte(first.Messages[0].Content), wantFirst) {
|
||||||
|
t.Fatalf("rendered bytes=%v, want %v", []byte(first.Messages[0].Content), wantFirst)
|
||||||
|
}
|
||||||
|
if !bytes.Equal(body, original) {
|
||||||
|
t.Fatalf("renderer mutated artifact body: got %v want %v", body, original)
|
||||||
|
}
|
||||||
|
|
||||||
|
body[0] = 'z'
|
||||||
|
if bytes.Equal([]byte(first.Messages[0].Content), append(append(append([]byte(nil), body...), '|'), body...)) {
|
||||||
|
t.Fatal("completed render aliases the artifact body")
|
||||||
|
}
|
||||||
|
second, err := NewGoRenderer().Render(context.Background(), definition, inputs, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("second render: %v", err)
|
||||||
|
}
|
||||||
|
wantSecond := append(append(append([]byte(nil), body...), '|'), body...)
|
||||||
|
if !bytes.Equal([]byte(second.Messages[0].Content), wantSecond) {
|
||||||
|
t.Fatalf("second render reused text from another call: got %v want %v", []byte(second.Messages[0].Content), wantSecond)
|
||||||
|
}
|
||||||
|
|
||||||
|
nilInputs := map[string]*domain.Artifact{"document": nil}
|
||||||
|
result, err := NewGoRenderer().Render(context.Background(), definition, nilInputs, nil)
|
||||||
|
if result != nil || !errors.Is(err, ErrUnknownInput) || !errors.Is(err, ErrRenderFailure) {
|
||||||
|
t.Fatalf("nil input result=%#v err=%v, want ErrUnknownInput and ErrRenderFailure", result, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func BenchmarkGoRendererArtifactReferences(b *testing.B) {
|
||||||
|
body := bytes.Repeat([]byte("document content "), (artifactTextChunkSize*4)/len("document content "))
|
||||||
|
inputs := map[string]*domain.Artifact{
|
||||||
|
"document": {Body: body},
|
||||||
|
}
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
definition *domain.PromptDefinition
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "one reference",
|
||||||
|
definition: &domain.PromptDefinition{
|
||||||
|
Templates: []domain.PromptMessageTemplate{
|
||||||
|
{Role: "user", Content: "{{input \"document\"}}"},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "repeated across session and messages",
|
||||||
|
definition: &domain.PromptDefinition{
|
||||||
|
SessionID: "document-{{len (input \"document\")}}",
|
||||||
|
Templates: []domain.PromptMessageTemplate{
|
||||||
|
{Role: "system", Content: "{{input \"document\"}}"},
|
||||||
|
{Role: "user", Content: "{{input \"document\"}} {{input \"document\"}}"},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range tests {
|
||||||
|
b.Run(tc.name, func(b *testing.B) {
|
||||||
|
renderer := NewGoRenderer()
|
||||||
|
b.ReportAllocs()
|
||||||
|
b.SetBytes(int64(len(body)))
|
||||||
|
for range b.N {
|
||||||
|
if _, err := renderer.Render(context.Background(), tc.definition, inputs, nil); err != nil {
|
||||||
|
b.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type checkCountingContext struct {
|
||||||
|
context.Context
|
||||||
|
checks int
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *checkCountingContext) Err() error {
|
||||||
|
c.checks++
|
||||||
|
return c.Context.Err()
|
||||||
|
}
|
||||||
|
|
||||||
|
type cancelOnCheckContext struct {
|
||||||
|
context.Context
|
||||||
|
cancel context.CancelFunc
|
||||||
|
remaining int
|
||||||
|
}
|
||||||
|
|
||||||
|
func newCancelOnCheckContext(checks int) *cancelOnCheckContext {
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
return &cancelOnCheckContext{Context: ctx, cancel: cancel, remaining: checks}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *cancelOnCheckContext) Err() error {
|
||||||
|
if c.Context.Err() == nil {
|
||||||
|
c.remaining--
|
||||||
|
if c.remaining == 0 {
|
||||||
|
c.cancel()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return c.Context.Err()
|
||||||
|
}
|
||||||
|
|||||||
98
internal/promptdef/content_source.go
Normal file
98
internal/promptdef/content_source.go
Normal file
@@ -0,0 +1,98 @@
|
|||||||
|
package promptdef
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"io/fs"
|
||||||
|
"os"
|
||||||
|
"path"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/filecatalog"
|
||||||
|
)
|
||||||
|
|
||||||
|
type contentSourceRoot interface {
|
||||||
|
readContentFile(sourcePath string, contentFile string) (string, string, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
type osContentSourceRoot struct {
|
||||||
|
root string
|
||||||
|
sourcePathsRelative bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r osContentSourceRoot) readContentFile(sourcePath string, contentFile string) (string, string, error) {
|
||||||
|
if strings.TrimSpace(contentFile) == "" {
|
||||||
|
return "", "", fmt.Errorf("path is required")
|
||||||
|
}
|
||||||
|
if filepath.IsAbs(contentFile) {
|
||||||
|
return "", "", fmt.Errorf("path %q must be relative", contentFile)
|
||||||
|
}
|
||||||
|
|
||||||
|
root, err := filepath.Abs(r.root)
|
||||||
|
if err != nil {
|
||||||
|
return "", "", fmt.Errorf("resolve source root %q: %w", r.root, err)
|
||||||
|
}
|
||||||
|
canonicalRoot, err := filepath.EvalSymlinks(root)
|
||||||
|
if err != nil {
|
||||||
|
return "", "", fmt.Errorf("resolve source root %q: %w", r.root, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
promptPath := sourcePath
|
||||||
|
if r.sourcePathsRelative && !filepath.IsAbs(promptPath) {
|
||||||
|
promptPath = filepath.Join(root, filepath.FromSlash(promptPath))
|
||||||
|
} else {
|
||||||
|
promptPath, err = filepath.Abs(promptPath)
|
||||||
|
if err != nil {
|
||||||
|
return "", "", fmt.Errorf("resolve prompt source %q: %w", sourcePath, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
resolvedPath := filepath.Clean(filepath.Join(filepath.Dir(promptPath), contentFile))
|
||||||
|
if !containsOSPath(root, resolvedPath) {
|
||||||
|
return "", "", fmt.Errorf("path %q escapes source root %q", contentFile, r.root)
|
||||||
|
}
|
||||||
|
|
||||||
|
canonicalPath, err := filepath.EvalSymlinks(resolvedPath)
|
||||||
|
if err != nil {
|
||||||
|
return "", "", err
|
||||||
|
}
|
||||||
|
if !containsOSPath(canonicalRoot, canonicalPath) {
|
||||||
|
return "", "", fmt.Errorf("path %q escapes source root %q", contentFile, r.root)
|
||||||
|
}
|
||||||
|
|
||||||
|
body, err := os.ReadFile(canonicalPath)
|
||||||
|
if err != nil {
|
||||||
|
return "", "", err
|
||||||
|
}
|
||||||
|
return string(body), resolvedPath, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type fsContentSourceRoot struct {
|
||||||
|
fsys fs.FS
|
||||||
|
root string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r fsContentSourceRoot) readContentFile(sourcePath string, contentFile string) (string, string, error) {
|
||||||
|
root := filecatalog.CleanFSRoot(r.root)
|
||||||
|
cleanSourcePath := path.Clean(sourcePath)
|
||||||
|
if cleanSourcePath == root {
|
||||||
|
root = path.Dir(root)
|
||||||
|
}
|
||||||
|
|
||||||
|
resolvedPath, _, err := filecatalog.ResolveFSPath(root, path.Dir(cleanSourcePath), contentFile)
|
||||||
|
if err != nil {
|
||||||
|
return "", "", err
|
||||||
|
}
|
||||||
|
body, err := fs.ReadFile(r.fsys, resolvedPath)
|
||||||
|
if err != nil {
|
||||||
|
return "", "", err
|
||||||
|
}
|
||||||
|
return string(body), resolvedPath, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func containsOSPath(root string, name string) bool {
|
||||||
|
relative, err := filepath.Rel(root, name)
|
||||||
|
if err != nil || filepath.IsAbs(relative) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return relative != ".." && !strings.HasPrefix(relative, ".."+string(filepath.Separator))
|
||||||
|
}
|
||||||
@@ -5,14 +5,11 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"io"
|
||||||
"io/fs"
|
"io/fs"
|
||||||
"os"
|
|
||||||
"path"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/filecatalog"
|
|
||||||
"gopkg.in/yaml.v3"
|
"gopkg.in/yaml.v3"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -22,13 +19,8 @@ var (
|
|||||||
ErrInvalidPromptDefinition = errors.New("invalid prompt definition configuration")
|
ErrInvalidPromptDefinition = errors.New("invalid prompt definition configuration")
|
||||||
)
|
)
|
||||||
|
|
||||||
type filesystemRepository struct {
|
type sourceRepository struct {
|
||||||
dir string
|
source promptDefinitionSource
|
||||||
}
|
|
||||||
|
|
||||||
type fsRepository struct {
|
|
||||||
fsys fs.FS
|
|
||||||
root string
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type promptDefinitionFile struct {
|
type promptDefinitionFile struct {
|
||||||
@@ -69,19 +61,47 @@ type promptOutputContractFile struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func NewFilesystemRepository(dir string) Repository {
|
func NewFilesystemRepository(dir string) Repository {
|
||||||
return &filesystemRepository{dir: dir}
|
return &sourceRepository{
|
||||||
|
source: osPromptSource{
|
||||||
|
root: dir,
|
||||||
|
contentRoot: osContentSourceRoot{root: dir},
|
||||||
|
},
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewFSRepository(fsys fs.FS, root string) Repository {
|
func NewFSRepository(fsys fs.FS, root string) Repository {
|
||||||
return &fsRepository{fsys: fsys, root: root}
|
return &sourceRepository{
|
||||||
|
source: fsPromptSource{
|
||||||
|
fsys: fsys,
|
||||||
|
root: root,
|
||||||
|
contentRoot: fsContentSourceRoot{fsys: fsys, root: root},
|
||||||
|
},
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *filesystemRepository) GetPromptDefinition(ctx context.Context, id string, version string) (*domain.PromptDefinition, error) {
|
// NewFileRepository constructs a repository for one operating-system prompt file.
|
||||||
|
func NewFileRepository(fsys fs.FS, file string, sourceDir string) Repository {
|
||||||
|
return &sourceRepository{
|
||||||
|
source: fsPromptSource{
|
||||||
|
fsys: fsys,
|
||||||
|
root: file,
|
||||||
|
contentRoot: osContentSourceRoot{
|
||||||
|
root: sourceDir,
|
||||||
|
sourcePathsRelative: true,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *sourceRepository) GetPromptDefinition(ctx context.Context, id string, version string) (*domain.PromptDefinition, error) {
|
||||||
if strings.TrimSpace(id) == "" {
|
if strings.TrimSpace(id) == "" {
|
||||||
return nil, fmt.Errorf("%w: prompt id is required", ErrInvalidPromptDefinition)
|
return nil, fmt.Errorf("%w: prompt id is required", ErrInvalidPromptDefinition)
|
||||||
}
|
}
|
||||||
|
if r == nil || r.source == nil {
|
||||||
|
return nil, errors.New("failed to read prompt definition directory: source is nil")
|
||||||
|
}
|
||||||
|
|
||||||
files, err := filecatalog.FindYAMLFiles(ctx, r.dir)
|
files, err := r.source.findYAMLFiles(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("failed to read prompt definition directory: %w", err)
|
return nil, fmt.Errorf("failed to read prompt definition directory: %w", err)
|
||||||
}
|
}
|
||||||
@@ -94,34 +114,25 @@ func (r *filesystemRepository) GetPromptDefinition(ctx context.Context, id strin
|
|||||||
default:
|
default:
|
||||||
}
|
}
|
||||||
|
|
||||||
relPath := filecatalog.RelativePath(r.dir, fullPath)
|
relPath := r.source.displayPath(fullPath)
|
||||||
fileMatch := filecatalog.Stem(filepath.Base(fullPath)) == id
|
data, err := r.source.readDefinition(fullPath)
|
||||||
|
|
||||||
raw, err := loadPromptDefinitionFile(fullPath)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if fileMatch || promptDefinitionFileHasID(fullPath, id) {
|
return nil, fmt.Errorf("failed to read prompt definition file %s: %w", relPath, err)
|
||||||
|
}
|
||||||
|
raw, err := decodePromptDefinition(data)
|
||||||
|
if err != nil {
|
||||||
|
if promptDefinitionDataMatches(data, id, version) {
|
||||||
return nil, fmt.Errorf("%w: %s: %v", ErrInvalidYAML, relPath, err)
|
return nil, fmt.Errorf("%w: %s: %v", ErrInvalidYAML, relPath, err)
|
||||||
}
|
}
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
if !promptDefinitionMatches(raw, id, version) {
|
||||||
def, err := normalizePromptDefinition(raw, fullPath)
|
|
||||||
if err != nil {
|
|
||||||
if fileMatch || strings.TrimSpace(raw.ID) == id {
|
|
||||||
return nil, fmt.Errorf("%w: %s: %v", ErrInvalidPromptDefinition, relPath, err)
|
|
||||||
}
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
if def.ID != id {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if version != "" && def.Version != version {
|
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
matches = append(matches, promptDefinitionMatch{
|
matches = append(matches, promptDefinitionMatch{
|
||||||
def: def,
|
raw: raw,
|
||||||
path: relPath,
|
sourcePath: fullPath,
|
||||||
|
path: relPath,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -137,130 +148,21 @@ func (r *filesystemRepository) GetPromptDefinition(ctx context.Context, id strin
|
|||||||
}
|
}
|
||||||
|
|
||||||
if len(matches) == 1 {
|
if len(matches) == 1 {
|
||||||
return matches[0].def, nil
|
match := matches[0]
|
||||||
|
def, err := normalizePromptDefinition(match.raw, r.source, match.sourcePath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("%w: %s: %v", ErrInvalidPromptDefinition, match.path, err)
|
||||||
|
}
|
||||||
|
return def, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil, ErrPromptDefinitionNotFound
|
return nil, ErrPromptDefinitionNotFound
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *fsRepository) GetPromptDefinition(ctx context.Context, id string, version string) (*domain.PromptDefinition, error) {
|
|
||||||
return loadPromptDefinition(ctx, r.fsys, r.root, id, version)
|
|
||||||
}
|
|
||||||
|
|
||||||
type promptDefinitionMatch struct {
|
type promptDefinitionMatch struct {
|
||||||
def *domain.PromptDefinition
|
raw *promptDefinitionFile
|
||||||
path string
|
sourcePath string
|
||||||
}
|
path string
|
||||||
|
|
||||||
func loadPromptDefinitionFile(path string) (*promptDefinitionFile, error) {
|
|
||||||
data, err := os.ReadFile(path)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("failed to read prompt definition file: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
var raw promptDefinitionFile
|
|
||||||
decoder := yaml.NewDecoder(bytes.NewReader(data))
|
|
||||||
decoder.KnownFields(true)
|
|
||||||
if err := decoder.Decode(&raw); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return &raw, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func promptDefinitionFileHasID(path string, id string) bool {
|
|
||||||
data, err := os.ReadFile(path)
|
|
||||||
if err != nil {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
var raw struct {
|
|
||||||
ID string `yaml:"id"`
|
|
||||||
}
|
|
||||||
if err := yaml.NewDecoder(bytes.NewReader(data)).Decode(&raw); err != nil {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
return strings.TrimSpace(raw.ID) == id
|
|
||||||
}
|
|
||||||
|
|
||||||
func loadPromptDefinition(ctx context.Context, fsys fs.FS, root string, id string, version string) (*domain.PromptDefinition, error) {
|
|
||||||
if strings.TrimSpace(id) == "" {
|
|
||||||
return nil, fmt.Errorf("%w: prompt id is required", ErrInvalidPromptDefinition)
|
|
||||||
}
|
|
||||||
if fsys == nil {
|
|
||||||
return nil, fmt.Errorf("failed to read prompt definition directory: filesystem is nil")
|
|
||||||
}
|
|
||||||
|
|
||||||
files, err := filecatalog.FindFSYAMLFiles(ctx, fsys, root)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("failed to read prompt definition directory: %w", err)
|
|
||||||
}
|
|
||||||
cleanRoot := filecatalog.CleanFSRoot(root)
|
|
||||||
rootInfo, err := fs.Stat(fsys, cleanRoot)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("failed to read prompt definition directory: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
var matches []promptDefinitionMatch
|
|
||||||
for _, fullPath := range files {
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
return nil, ctx.Err()
|
|
||||||
default:
|
|
||||||
}
|
|
||||||
|
|
||||||
relPath := filecatalog.DisplayPath(root, fullPath)
|
|
||||||
fileMatch := filecatalog.Stem(path.Base(fullPath)) == id
|
|
||||||
data, err := fs.ReadFile(fsys, fullPath)
|
|
||||||
if err != nil {
|
|
||||||
if fileMatch {
|
|
||||||
return nil, fmt.Errorf("%w: %s: failed to read prompt definition file: %v", ErrInvalidYAML, relPath, err)
|
|
||||||
}
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
raw, err := decodePromptDefinition(data)
|
|
||||||
if err != nil {
|
|
||||||
if fileMatch || promptDefinitionDataHasID(data, id) {
|
|
||||||
return nil, fmt.Errorf("%w: %s: %v", ErrInvalidYAML, relPath, err)
|
|
||||||
}
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
def, err := normalizePromptDefinitionFromFS(raw, fsys, root, fullPath, rootInfo.IsDir())
|
|
||||||
if err != nil {
|
|
||||||
if fileMatch || strings.TrimSpace(raw.ID) == id {
|
|
||||||
return nil, fmt.Errorf("%w: %s: %v", ErrInvalidPromptDefinition, relPath, err)
|
|
||||||
}
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
if def.ID != id {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if version != "" && def.Version != version {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
matches = append(matches, promptDefinitionMatch{
|
|
||||||
def: def,
|
|
||||||
path: relPath,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(matches) > 1 {
|
|
||||||
paths := make([]string, 0, len(matches))
|
|
||||||
for _, match := range matches {
|
|
||||||
paths = append(paths, match.path)
|
|
||||||
}
|
|
||||||
if version != "" {
|
|
||||||
return nil, fmt.Errorf("%w: duplicate prompt definition id %q version %q found in: %s", ErrInvalidPromptDefinition, id, version, strings.Join(paths, ", "))
|
|
||||||
}
|
|
||||||
return nil, fmt.Errorf("%w: duplicate prompt definition id %q found in: %s", ErrInvalidPromptDefinition, id, strings.Join(paths, ", "))
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(matches) == 1 {
|
|
||||||
return matches[0].def, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil, ErrPromptDefinitionNotFound
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func decodePromptDefinition(data []byte) (*promptDefinitionFile, error) {
|
func decodePromptDefinition(data []byte) (*promptDefinitionFile, error) {
|
||||||
@@ -270,59 +172,44 @@ func decodePromptDefinition(data []byte) (*promptDefinitionFile, error) {
|
|||||||
if err := decoder.Decode(&raw); err != nil {
|
if err := decoder.Decode(&raw); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
var additional yaml.Node
|
||||||
|
if err := decoder.Decode(&additional); err != io.EOF {
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return nil, errors.New("prompt definition file must contain exactly one YAML document")
|
||||||
|
}
|
||||||
return &raw, nil
|
return &raw, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func promptDefinitionDataHasID(data []byte, id string) bool {
|
func promptDefinitionDataMatches(data []byte, id string, version string) bool {
|
||||||
var raw struct {
|
var raw struct {
|
||||||
ID string `yaml:"id"`
|
ID string `yaml:"id"`
|
||||||
|
Version string `yaml:"version"`
|
||||||
}
|
}
|
||||||
if err := yaml.NewDecoder(bytes.NewReader(data)).Decode(&raw); err != nil {
|
if err := yaml.NewDecoder(bytes.NewReader(data)).Decode(&raw); err != nil {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
return strings.TrimSpace(raw.ID) == id
|
return promptSelectorMatches(raw.ID, raw.Version, id, version)
|
||||||
}
|
}
|
||||||
|
|
||||||
func normalizePromptDefinition(raw *promptDefinitionFile, sourcePath string) (*domain.PromptDefinition, error) {
|
func promptDefinitionMatches(raw *promptDefinitionFile, id string, version string) bool {
|
||||||
promptDir := filepath.Dir(sourcePath)
|
if raw == nil {
|
||||||
return normalizePromptDefinitionWithContent(raw, func(contentFile string) (string, string, error) {
|
return false
|
||||||
resolvedPath := strings.TrimSpace(contentFile)
|
}
|
||||||
if !filepath.IsAbs(resolvedPath) {
|
return promptSelectorMatches(raw.ID, raw.Version, id, version)
|
||||||
resolvedPath = filepath.Join(promptDir, resolvedPath)
|
|
||||||
}
|
|
||||||
resolvedPath = filepath.Clean(resolvedPath)
|
|
||||||
|
|
||||||
body, err := os.ReadFile(resolvedPath)
|
|
||||||
if err != nil {
|
|
||||||
return "", "", err
|
|
||||||
}
|
|
||||||
return string(body), resolvedPath, nil
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func normalizePromptDefinitionFromFS(raw *promptDefinitionFile, fsys fs.FS, root string, sourcePath string, rootIsDir bool) (*domain.PromptDefinition, error) {
|
func promptSelectorMatches(rawID string, rawVersion string, id string, version string) bool {
|
||||||
promptDir := path.Dir(sourcePath)
|
if strings.TrimSpace(rawID) != id {
|
||||||
return normalizePromptDefinitionWithContent(raw, func(contentFile string) (string, string, error) {
|
return false
|
||||||
var resolvedPath string
|
}
|
||||||
if rootIsDir {
|
return version == "" || strings.TrimSpace(rawVersion) == version
|
||||||
var err error
|
}
|
||||||
resolvedPath, _, err = filecatalog.ResolveFSPath(root, promptDir, contentFile)
|
|
||||||
if err != nil {
|
|
||||||
return "", "", err
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
resolvedPath = strings.TrimSpace(contentFile)
|
|
||||||
if !path.IsAbs(resolvedPath) {
|
|
||||||
resolvedPath = path.Join(promptDir, resolvedPath)
|
|
||||||
}
|
|
||||||
resolvedPath = strings.TrimPrefix(path.Clean(resolvedPath), "/")
|
|
||||||
}
|
|
||||||
|
|
||||||
body, err := fs.ReadFile(fsys, resolvedPath)
|
func normalizePromptDefinition(raw *promptDefinitionFile, sourceRoot contentSourceRoot, sourcePath string) (*domain.PromptDefinition, error) {
|
||||||
if err != nil {
|
return normalizePromptDefinitionWithContent(raw, func(contentFile string) (string, string, error) {
|
||||||
return "", "", err
|
return sourceRoot.readContentFile(sourcePath, contentFile)
|
||||||
}
|
|
||||||
return string(body), resolvedPath, nil
|
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -402,17 +289,14 @@ func normalizePromptDefinitionWithContent(raw *promptDefinitionFile, readContent
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
if !isValidOutputFormat(raw.Output.Format) {
|
outputContract := domain.OutputContract{
|
||||||
return nil, fmt.Errorf("invalid output format: %q", raw.Output.Format)
|
Format: raw.Output.Format,
|
||||||
|
ValidationMode: raw.Output.ValidationMode,
|
||||||
|
SchemaPath: strings.TrimSpace(raw.Output.SchemaPath),
|
||||||
|
RepairAttempts: raw.Output.RepairAttempts,
|
||||||
}
|
}
|
||||||
if !isValidValidationMode(raw.Output.ValidationMode) {
|
if err := domain.ValidateOutputContract(outputContract); err != nil {
|
||||||
return nil, fmt.Errorf("invalid validation mode: %q", raw.Output.ValidationMode)
|
return nil, fmt.Errorf("output: %w", err)
|
||||||
}
|
|
||||||
if raw.Output.ValidationMode == domain.ValidationJSONSchema && strings.TrimSpace(raw.Output.SchemaPath) == "" {
|
|
||||||
return nil, errors.New("output.schema_path is required when output.validation_mode is json_schema")
|
|
||||||
}
|
|
||||||
if raw.Output.RepairAttempts < 0 {
|
|
||||||
return nil, errors.New("output.repair_attempts must be greater than or equal to 0")
|
|
||||||
}
|
}
|
||||||
|
|
||||||
defaultProfile := ""
|
defaultProfile := ""
|
||||||
@@ -432,12 +316,7 @@ func normalizePromptDefinitionWithContent(raw *promptDefinitionFile, readContent
|
|||||||
Inputs: inputs,
|
Inputs: inputs,
|
||||||
Templates: templates,
|
Templates: templates,
|
||||||
OutputFormat: raw.Output.Format,
|
OutputFormat: raw.Output.Format,
|
||||||
Validation: domain.OutputContract{
|
Validation: outputContract,
|
||||||
Format: raw.Output.Format,
|
|
||||||
ValidationMode: raw.Output.ValidationMode,
|
|
||||||
SchemaPath: strings.TrimSpace(raw.Output.SchemaPath),
|
|
||||||
RepairAttempts: raw.Output.RepairAttempts,
|
|
||||||
},
|
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -464,21 +343,3 @@ func normalizeCacheControl(raw *cacheControlFile) (*domain.CacheControl, error)
|
|||||||
TTL: ttl,
|
TTL: ttl,
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func isValidOutputFormat(f domain.OutputFormat) bool {
|
|
||||||
switch f {
|
|
||||||
case domain.FormatText, domain.FormatMarkdown, domain.FormatJSON:
|
|
||||||
return true
|
|
||||||
default:
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func isValidValidationMode(m domain.ValidationMode) bool {
|
|
||||||
switch m {
|
|
||||||
case domain.ValidationNone, domain.ValidationBasic, domain.ValidationJSON, domain.ValidationJSONSchema:
|
|
||||||
return true
|
|
||||||
default:
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -3,23 +3,51 @@ package promptdef
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
|
"fmt"
|
||||||
"io/fs"
|
"io/fs"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
|
"sync"
|
||||||
"testing"
|
"testing"
|
||||||
"testing/fstest"
|
"testing/fstest"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestFilesystemRepository_GetPromptDefinition(t *testing.T) {
|
func TestPromptRepositoryDefinitionFixtures(t *testing.T) {
|
||||||
|
sources := []struct {
|
||||||
|
name string
|
||||||
|
newRepository func(string) Repository
|
||||||
|
contentPathsAreFull bool
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "operating system",
|
||||||
|
newRepository: NewFilesystemRepository,
|
||||||
|
contentPathsAreFull: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "filesystem",
|
||||||
|
newRepository: func(root string) Repository {
|
||||||
|
return NewFSRepository(os.DirFS(root), ".")
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, source := range sources {
|
||||||
|
t.Run(source.name, func(t *testing.T) {
|
||||||
|
testPromptRepositoryDefinitionFixtures(t, source.newRepository, source.contentPathsAreFull)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func testPromptRepositoryDefinitionFixtures(t *testing.T, newRepository func(string) Repository, contentPathsAreFull bool) {
|
||||||
tmpDir := t.TempDir()
|
tmpDir := t.TempDir()
|
||||||
if err := copyTree("testdata", tmpDir); err != nil {
|
if err := copyTree("testdata", tmpDir); err != nil {
|
||||||
t.Fatalf("failed to copy testdata: %v", err)
|
t.Fatalf("failed to copy testdata: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
repo := NewFilesystemRepository(tmpDir)
|
repo := newRepository(tmpDir)
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
|
|
||||||
t.Run("valid inline prompt", func(t *testing.T) {
|
t.Run("valid inline prompt", func(t *testing.T) {
|
||||||
@@ -64,8 +92,8 @@ func TestFilesystemRepository_GetPromptDefinition(t *testing.T) {
|
|||||||
if p.Templates[1].ContentFile == "" {
|
if p.Templates[1].ContentFile == "" {
|
||||||
t.Fatal("expected ContentFile source metadata to be preserved")
|
t.Fatal("expected ContentFile source metadata to be preserved")
|
||||||
}
|
}
|
||||||
if !filepath.IsAbs(p.Templates[1].ContentFile) {
|
if filepath.IsAbs(p.Templates[1].ContentFile) != contentPathsAreFull {
|
||||||
t.Fatalf("expected resolved content_file path to be absolute, got %q", p.Templates[1].ContentFile)
|
t.Fatalf("unexpected content_file path representation: %q", p.Templates[1].ContentFile)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
@@ -287,20 +315,20 @@ output:
|
|||||||
targetErr error
|
targetErr error
|
||||||
errSubstrs []string
|
errSubstrs []string
|
||||||
}{
|
}{
|
||||||
{name: "invalid YAML", id: "invalid_yaml", targetErr: ErrInvalidYAML},
|
{name: "unidentifiable invalid YAML is unrelated", id: "invalid-yaml", targetErr: ErrPromptDefinitionNotFound},
|
||||||
{name: "missing id", id: "missing_id", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"id is required"}},
|
{name: "missing id is not selected by filename", id: "missing_id", targetErr: ErrPromptDefinitionNotFound},
|
||||||
{name: "no messages", id: "no_messages", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"at least one message is required"}},
|
{name: "no messages", id: "no-messages", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"at least one message is required"}},
|
||||||
{name: "both content and content_file", id: "both_content_and_content_file", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"exactly one"}},
|
{name: "both content and content_file", id: "both-content-and-content-file", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"exactly one"}},
|
||||||
{name: "neither content nor content_file", id: "neither_content_nor_content_file", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"exactly one"}},
|
{name: "neither content nor content_file", id: "neither-content-nor-content-file", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"exactly one"}},
|
||||||
{name: "missing content_file", id: "missing_content_file", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"failed to read content_file"}},
|
{name: "missing content_file", id: "missing-content-file", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"failed to read content_file"}},
|
||||||
{name: "duplicate input names", id: "duplicate_input_names", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"duplicate input name"}},
|
{name: "duplicate input names", id: "duplicate-input-names", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"duplicate input name"}},
|
||||||
{name: "invalid validation mode", id: "invalid_validation_mode", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"invalid validation mode"}},
|
{name: "invalid validation mode", id: "invalid-validation-mode", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"invalid validation mode"}},
|
||||||
{name: "json_schema without schema_path", id: "json_schema_without_schema_path", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"schema_path"}},
|
{name: "json_schema without schema_path", id: "json-schema-without-schema-path", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"schema_path"}},
|
||||||
{name: "unknown input field", id: "unknown_input_field", targetErr: ErrInvalidYAML, errSubstrs: []string{"field unknown_input_setting not found"}},
|
{name: "unknown input field", id: "unknown-input-field", targetErr: ErrInvalidYAML, errSubstrs: []string{"field unknown_input_setting not found"}},
|
||||||
{name: "empty cache control type", id: "empty_cache_control_type", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"cache_control", "type is required"}},
|
{name: "empty cache control type", id: "empty-cache-control-type", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"cache_control", "type is required"}},
|
||||||
{name: "unsupported cache control type", id: "unsupported_cache_control_type", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"cache_control", "unsupported type"}},
|
{name: "unsupported cache control type", id: "unsupported-cache-control-type", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"cache_control", "unsupported type"}},
|
||||||
{name: "unsupported cache control ttl", id: "unsupported_cache_control_ttl", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"cache_control", "unsupported ttl"}},
|
{name: "unsupported cache control ttl", id: "unsupported-cache-control-ttl", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"cache_control", "unsupported ttl"}},
|
||||||
{name: "unknown cache control field", id: "unknown_cache_control_field", targetErr: ErrInvalidYAML, errSubstrs: []string{"field unexpected not found"}},
|
{name: "unknown cache control field", id: "unknown-cache-control-field", targetErr: ErrInvalidYAML, errSubstrs: []string{"field unexpected not found"}},
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, tc := range cases {
|
for _, tc := range cases {
|
||||||
@@ -396,7 +424,7 @@ output:
|
|||||||
|
|
||||||
for _, tc := range tests {
|
for _, tc := range tests {
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
repo := NewFSRepository(fstest.MapFS{
|
fsys := &recordingFS{FS: fstest.MapFS{
|
||||||
"prompts/prompt.yaml": &fstest.MapFile{Data: []byte(`
|
"prompts/prompt.yaml": &fstest.MapFile{Data: []byte(`
|
||||||
id: fs-escaped-prompt
|
id: fs-escaped-prompt
|
||||||
version: "1.0.0"
|
version: "1.0.0"
|
||||||
@@ -409,7 +437,8 @@ output:
|
|||||||
repair_attempts: 0
|
repair_attempts: 0
|
||||||
`)},
|
`)},
|
||||||
"outside.tmpl": &fstest.MapFile{Data: []byte(`Outside root.`)},
|
"outside.tmpl": &fstest.MapFile{Data: []byte(`Outside root.`)},
|
||||||
}, "prompts")
|
}}
|
||||||
|
repo := NewFSRepository(fsys, "prompts")
|
||||||
|
|
||||||
_, err := repo.GetPromptDefinition(context.Background(), "fs-escaped-prompt", "")
|
_, err := repo.GetPromptDefinition(context.Background(), "fs-escaped-prompt", "")
|
||||||
if !errors.Is(err, ErrInvalidPromptDefinition) {
|
if !errors.Is(err, ErrInvalidPromptDefinition) {
|
||||||
@@ -418,64 +447,611 @@ output:
|
|||||||
if !strings.Contains(err.Error(), tc.wantErr) {
|
if !strings.Contains(err.Error(), tc.wantErr) {
|
||||||
t.Fatalf("expected error to contain %q, got %v", tc.wantErr, err)
|
t.Fatalf("expected error to contain %q, got %v", tc.wantErr, err)
|
||||||
}
|
}
|
||||||
|
if fsys.wasOpened("outside.tmpl") {
|
||||||
|
t.Fatal("rejected content path opened the outside file")
|
||||||
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestFSRepositoryRejectsDuplicatePromptIDs(t *testing.T) {
|
type recordingFS struct {
|
||||||
repo := NewFSRepository(fstest.MapFS{
|
fs.FS
|
||||||
"one.yaml": &fstest.MapFile{Data: []byte(`
|
mu sync.Mutex
|
||||||
id: duplicate-fs-prompt
|
opened []string
|
||||||
version: "1.0.0"
|
}
|
||||||
messages:
|
|
||||||
- role: user
|
|
||||||
content: First.
|
|
||||||
output:
|
|
||||||
format: text
|
|
||||||
validation_mode: none
|
|
||||||
repair_attempts: 0
|
|
||||||
`)},
|
|
||||||
"nested/two.yaml": &fstest.MapFile{Data: []byte(`
|
|
||||||
id: duplicate-fs-prompt
|
|
||||||
version: "1.0.0"
|
|
||||||
messages:
|
|
||||||
- role: user
|
|
||||||
content: Second.
|
|
||||||
output:
|
|
||||||
format: text
|
|
||||||
validation_mode: none
|
|
||||||
repair_attempts: 0
|
|
||||||
`)},
|
|
||||||
}, ".")
|
|
||||||
|
|
||||||
_, err := repo.GetPromptDefinition(context.Background(), "duplicate-fs-prompt", "")
|
func TestPromptRepositoryReturnsDefinitionReadFailures(t *testing.T) {
|
||||||
if !errors.Is(err, ErrInvalidPromptDefinition) {
|
readErr := errors.New("definition read failed")
|
||||||
t.Fatalf("expected ErrInvalidPromptDefinition, got %v", err)
|
fsys := &definitionReadFailureFS{
|
||||||
|
FS: fstest.MapFS{
|
||||||
|
"prompts/target.yaml": &fstest.MapFile{Data: []byte("unread")},
|
||||||
|
},
|
||||||
|
target: "prompts/target.yaml",
|
||||||
|
err: readErr,
|
||||||
}
|
}
|
||||||
if !strings.Contains(err.Error(), "one.yaml") || !strings.Contains(err.Error(), "nested/two.yaml") {
|
repo := NewFSRepository(fsys, "prompts")
|
||||||
t.Fatalf("expected duplicate paths in error, got %v", err)
|
|
||||||
|
definition, err := repo.GetPromptDefinition(context.Background(), "target", "1")
|
||||||
|
if definition != nil || !errors.Is(err, readErr) {
|
||||||
|
t.Fatalf("GetPromptDefinition() = (%#v, %v), want nil and definition read error", definition, err)
|
||||||
|
}
|
||||||
|
if errors.Is(err, ErrPromptDefinitionNotFound) {
|
||||||
|
t.Fatalf("definition read error was classified as absence: %v", err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "target.yaml") {
|
||||||
|
t.Fatalf("definition read error lacks source context: %v", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestFSRepositoryRejectsUnknownYAMLFields(t *testing.T) {
|
type definitionReadFailureFS struct {
|
||||||
repo := NewFSRepository(fstest.MapFS{
|
fs.FS
|
||||||
"not_named_like_id.yaml": &fstest.MapFile{Data: []byte(`
|
target string
|
||||||
id: strict-fs-prompt
|
err error
|
||||||
version: "1.0.0"
|
}
|
||||||
unknown: true
|
|
||||||
|
func (f *definitionReadFailureFS) Open(name string) (fs.File, error) {
|
||||||
|
if name == f.target {
|
||||||
|
return nil, f.err
|
||||||
|
}
|
||||||
|
return f.FS.Open(name)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *recordingFS) Open(name string) (fs.File, error) {
|
||||||
|
f.mu.Lock()
|
||||||
|
f.opened = append(f.opened, name)
|
||||||
|
f.mu.Unlock()
|
||||||
|
return f.FS.Open(name)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *recordingFS) wasOpened(name string) bool {
|
||||||
|
return f.openCount(name) > 0
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *recordingFS) openCount(name string) int {
|
||||||
|
f.mu.Lock()
|
||||||
|
defer f.mu.Unlock()
|
||||||
|
count := 0
|
||||||
|
for _, opened := range f.opened {
|
||||||
|
if opened == name {
|
||||||
|
count++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return count
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPromptRepositorySelectionUsesYAMLMetadata(t *testing.T) {
|
||||||
|
const validDefinition = `
|
||||||
|
id: selected-prompt
|
||||||
|
version: "1"
|
||||||
messages:
|
messages:
|
||||||
- role: user
|
- role: user
|
||||||
content: Invalid.
|
content: selected
|
||||||
output:
|
output:
|
||||||
format: text
|
format: text
|
||||||
validation_mode: none
|
validation_mode: none
|
||||||
repair_attempts: 0
|
`
|
||||||
`)},
|
tests := []struct {
|
||||||
}, ".")
|
name string
|
||||||
|
files map[string]string
|
||||||
|
wantErr error
|
||||||
|
diagnostics []string
|
||||||
|
wantContent string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "same-stem strict error with different YAML ID is unrelated",
|
||||||
|
files: map[string]string{
|
||||||
|
"selected-prompt.yaml": `
|
||||||
|
id: another-prompt
|
||||||
|
version: "1"
|
||||||
|
unknown: true
|
||||||
|
`,
|
||||||
|
"valid.yaml": validDefinition,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "unidentifiable same-stem YAML is unrelated",
|
||||||
|
files: map[string]string{
|
||||||
|
"selected-prompt.yaml": "id: [",
|
||||||
|
"valid.yaml": validDefinition,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "same ID invalid different version is unrelated",
|
||||||
|
files: map[string]string{
|
||||||
|
"invalid-version.yaml": `
|
||||||
|
id: selected-prompt
|
||||||
|
version: "2"
|
||||||
|
output:
|
||||||
|
format: text
|
||||||
|
validation_mode: none
|
||||||
|
`,
|
||||||
|
"valid.yaml": validDefinition,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "selected content is resolved relative to its definition",
|
||||||
|
files: map[string]string{
|
||||||
|
"nested/selected.yaml": `
|
||||||
|
id: selected-prompt
|
||||||
|
version: "1"
|
||||||
|
messages:
|
||||||
|
- role: user
|
||||||
|
content_file: ./content/selected.tmpl
|
||||||
|
output:
|
||||||
|
format: text
|
||||||
|
validation_mode: none
|
||||||
|
`,
|
||||||
|
"nested/content/selected.tmpl": "selected from file",
|
||||||
|
},
|
||||||
|
wantContent: "selected from file",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "duplicate selected definitions are ambiguous",
|
||||||
|
files: map[string]string{
|
||||||
|
"selected-a.yaml": validDefinition,
|
||||||
|
"nested/selected-b.yaml": `
|
||||||
|
id: selected-prompt
|
||||||
|
version: "1"
|
||||||
|
messages:
|
||||||
|
- role: user
|
||||||
|
content: duplicate
|
||||||
|
output:
|
||||||
|
format: text
|
||||||
|
validation_mode: none
|
||||||
|
`,
|
||||||
|
},
|
||||||
|
wantErr: ErrInvalidPromptDefinition,
|
||||||
|
diagnostics: []string{"duplicate prompt definition id", "selected-a.yaml", "nested/selected-b.yaml"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "selected strict error is authoritative",
|
||||||
|
files: map[string]string{
|
||||||
|
"selected-strict.yaml": `
|
||||||
|
id: selected-prompt
|
||||||
|
version: "1"
|
||||||
|
unknown: true
|
||||||
|
`,
|
||||||
|
},
|
||||||
|
wantErr: ErrInvalidYAML,
|
||||||
|
diagnostics: []string{"selected-strict.yaml"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "selected semantic error is authoritative",
|
||||||
|
files: map[string]string{
|
||||||
|
"selected-invalid.yaml": `
|
||||||
|
id: selected-prompt
|
||||||
|
version: "1"
|
||||||
|
output:
|
||||||
|
format: text
|
||||||
|
validation_mode: none
|
||||||
|
`,
|
||||||
|
},
|
||||||
|
wantErr: ErrInvalidPromptDefinition,
|
||||||
|
diagnostics: []string{"selected-invalid.yaml", "at least one message"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "selected content error includes definition context",
|
||||||
|
files: map[string]string{
|
||||||
|
"selected-missing-content.yaml": `
|
||||||
|
id: selected-prompt
|
||||||
|
version: "1"
|
||||||
|
messages:
|
||||||
|
- role: user
|
||||||
|
content_file: missing.tmpl
|
||||||
|
output:
|
||||||
|
format: text
|
||||||
|
validation_mode: none
|
||||||
|
`,
|
||||||
|
},
|
||||||
|
wantErr: ErrInvalidPromptDefinition,
|
||||||
|
diagnostics: []string{"selected-missing-content.yaml", "failed to read content_file", "missing.tmpl"},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
_, err := repo.GetPromptDefinition(context.Background(), "strict-fs-prompt", "")
|
for _, source := range promptRepositorySources() {
|
||||||
if !errors.Is(err, ErrInvalidYAML) {
|
for _, tc := range tests {
|
||||||
t.Fatalf("expected ErrInvalidYAML, got %v", err)
|
t.Run(source.name+"/"+tc.name, func(t *testing.T) {
|
||||||
|
repo := source.newRepository(t, tc.files)
|
||||||
|
got, err := repo.GetPromptDefinition(context.Background(), "selected-prompt", "1")
|
||||||
|
if tc.wantErr != nil {
|
||||||
|
if !errors.Is(err, tc.wantErr) {
|
||||||
|
t.Fatalf("expected %v, got %v", tc.wantErr, err)
|
||||||
|
}
|
||||||
|
for _, diagnostic := range tc.diagnostics {
|
||||||
|
if !strings.Contains(err.Error(), diagnostic) {
|
||||||
|
t.Fatalf("expected error to contain %q, got %v", diagnostic, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("load selected prompt: %v", err)
|
||||||
|
}
|
||||||
|
wantContent := tc.wantContent
|
||||||
|
if wantContent == "" {
|
||||||
|
wantContent = "selected"
|
||||||
|
}
|
||||||
|
if got.ID != "selected-prompt" || got.Version != "1" || got.Templates[0].Content != wantContent {
|
||||||
|
t.Fatalf("unexpected selected definition: %+v", got)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPromptRepositoryHonorsCancellation(t *testing.T) {
|
||||||
|
for _, source := range promptRepositorySources() {
|
||||||
|
t.Run(source.name, func(t *testing.T) {
|
||||||
|
repo := source.newRepository(t, map[string]string{
|
||||||
|
"definition.yaml": `
|
||||||
|
id: cancelled-prompt
|
||||||
|
version: "1"
|
||||||
|
messages:
|
||||||
|
- role: user
|
||||||
|
content: selected
|
||||||
|
output:
|
||||||
|
format: text
|
||||||
|
validation_mode: none
|
||||||
|
`,
|
||||||
|
})
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
cancel()
|
||||||
|
|
||||||
|
_, err := repo.GetPromptDefinition(ctx, "cancelled-prompt", "1")
|
||||||
|
if !errors.Is(err, context.Canceled) {
|
||||||
|
t.Fatalf("expected context cancellation, got %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPromptRepositoryRequiresOneYAMLDocument(t *testing.T) {
|
||||||
|
const definition = `
|
||||||
|
id: one-document
|
||||||
|
version: "1"
|
||||||
|
messages:
|
||||||
|
- role: user
|
||||||
|
content: selected
|
||||||
|
output:
|
||||||
|
format: text
|
||||||
|
validation_mode: none
|
||||||
|
`
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
suffix string
|
||||||
|
wantErr bool
|
||||||
|
}{
|
||||||
|
{name: "comments and trailing whitespace", suffix: "\n# trailing comment\n\n"},
|
||||||
|
{name: "second populated document", suffix: "\n---\nid: another\n", wantErr: true},
|
||||||
|
{name: "second empty document", suffix: "\n---\n", wantErr: true},
|
||||||
|
{name: "malformed trailing YAML", suffix: "\n---\n[", wantErr: true},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, source := range promptRepositorySources() {
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(source.name+"/"+tc.name, func(t *testing.T) {
|
||||||
|
repo := source.newRepository(t, map[string]string{"definition.yaml": definition + tc.suffix})
|
||||||
|
_, err := repo.GetPromptDefinition(context.Background(), "one-document", "1")
|
||||||
|
if tc.wantErr {
|
||||||
|
if !errors.Is(err, ErrInvalidYAML) {
|
||||||
|
t.Fatalf("expected ErrInvalidYAML, got %v", err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "definition.yaml") {
|
||||||
|
t.Fatalf("expected source path in error, got %v", err)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("load one document: %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPromptRepositoryReadsOnlySelectedContent(t *testing.T) {
|
||||||
|
fsys := &recordingFS{FS: fstest.MapFS{
|
||||||
|
"target.yaml": &fstest.MapFile{Data: []byte(`
|
||||||
|
id: target
|
||||||
|
version: "1"
|
||||||
|
messages:
|
||||||
|
- role: user
|
||||||
|
content_file: target.tmpl
|
||||||
|
output:
|
||||||
|
format: text
|
||||||
|
validation_mode: none
|
||||||
|
`)},
|
||||||
|
"target.tmpl": &fstest.MapFile{Data: []byte("selected")},
|
||||||
|
"unrelated.yaml": &fstest.MapFile{Data: []byte(`
|
||||||
|
id: unrelated
|
||||||
|
version: "1"
|
||||||
|
messages:
|
||||||
|
- role: user
|
||||||
|
content_file: unrelated.tmpl
|
||||||
|
output:
|
||||||
|
format: text
|
||||||
|
validation_mode: none
|
||||||
|
`)},
|
||||||
|
"unrelated.tmpl": &fstest.MapFile{Data: []byte("unrelated")},
|
||||||
|
"other-version.yaml": &fstest.MapFile{Data: []byte(`
|
||||||
|
id: target
|
||||||
|
version: "2"
|
||||||
|
messages:
|
||||||
|
- role: user
|
||||||
|
content_file: other-version.tmpl
|
||||||
|
output:
|
||||||
|
format: text
|
||||||
|
validation_mode: none
|
||||||
|
`)},
|
||||||
|
"other-version.tmpl": &fstest.MapFile{Data: []byte("other version")},
|
||||||
|
}}
|
||||||
|
repo := NewFSRepository(fsys, ".")
|
||||||
|
|
||||||
|
for lookup := 1; lookup <= 2; lookup++ {
|
||||||
|
got, err := repo.GetPromptDefinition(context.Background(), "target", "1")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("lookup %d: %v", lookup, err)
|
||||||
|
}
|
||||||
|
if got.Templates[0].Content != "selected" {
|
||||||
|
t.Fatalf("lookup %d content = %q", lookup, got.Templates[0].Content)
|
||||||
|
}
|
||||||
|
if count := fsys.openCount("target.tmpl"); count != lookup {
|
||||||
|
t.Fatalf("selected content opens after lookup %d = %d, want %d", lookup, count, lookup)
|
||||||
|
}
|
||||||
|
for _, name := range []string{"unrelated.tmpl", "other-version.tmpl"} {
|
||||||
|
if count := fsys.openCount(name); count != 0 {
|
||||||
|
t.Fatalf("unselected content %q opened %d times", name, count)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, name := range []string{"target.yaml", "unrelated.yaml", "other-version.yaml"} {
|
||||||
|
if count := fsys.openCount(name); count != lookup {
|
||||||
|
t.Fatalf("metadata %q opens after lookup %d = %d, want %d", name, lookup, count, lookup)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPromptDefinitionNormalizationRules(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
definition string
|
||||||
|
wantErr bool
|
||||||
|
wantDiagnostic string
|
||||||
|
wantSchemaPath string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "missing version",
|
||||||
|
definition: `
|
||||||
|
id: normalization-rule
|
||||||
|
messages:
|
||||||
|
- role: user
|
||||||
|
content: test
|
||||||
|
output:
|
||||||
|
format: text
|
||||||
|
validation_mode: none
|
||||||
|
`,
|
||||||
|
wantErr: true,
|
||||||
|
wantDiagnostic: "version",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "blank input name",
|
||||||
|
definition: `
|
||||||
|
id: normalization-rule
|
||||||
|
version: "1"
|
||||||
|
inputs:
|
||||||
|
- name: " "
|
||||||
|
messages:
|
||||||
|
- role: user
|
||||||
|
content: test
|
||||||
|
output:
|
||||||
|
format: text
|
||||||
|
validation_mode: none
|
||||||
|
`,
|
||||||
|
wantErr: true,
|
||||||
|
wantDiagnostic: "input 0",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "blank message role",
|
||||||
|
definition: `
|
||||||
|
id: normalization-rule
|
||||||
|
version: "1"
|
||||||
|
messages:
|
||||||
|
- role: " "
|
||||||
|
content: test
|
||||||
|
output:
|
||||||
|
format: text
|
||||||
|
validation_mode: none
|
||||||
|
`,
|
||||||
|
wantErr: true,
|
||||||
|
wantDiagnostic: "role",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "invalid output format",
|
||||||
|
definition: `
|
||||||
|
id: normalization-rule
|
||||||
|
version: "1"
|
||||||
|
messages:
|
||||||
|
- role: user
|
||||||
|
content: test
|
||||||
|
output:
|
||||||
|
format: binary
|
||||||
|
validation_mode: none
|
||||||
|
`,
|
||||||
|
wantErr: true,
|
||||||
|
wantDiagnostic: "format",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "negative repair attempts",
|
||||||
|
definition: `
|
||||||
|
id: normalization-rule
|
||||||
|
version: "1"
|
||||||
|
messages:
|
||||||
|
- role: user
|
||||||
|
content: test
|
||||||
|
output:
|
||||||
|
format: text
|
||||||
|
validation_mode: none
|
||||||
|
repair_attempts: -1
|
||||||
|
`,
|
||||||
|
wantErr: true,
|
||||||
|
wantDiagnostic: "repair_attempts",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "explicit blank default profile",
|
||||||
|
definition: `
|
||||||
|
id: normalization-rule
|
||||||
|
version: "1"
|
||||||
|
default_profile: " "
|
||||||
|
messages:
|
||||||
|
- role: user
|
||||||
|
content: test
|
||||||
|
output:
|
||||||
|
format: text
|
||||||
|
validation_mode: none
|
||||||
|
`,
|
||||||
|
wantErr: true,
|
||||||
|
wantDiagnostic: "default_profile",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "schema path normalization",
|
||||||
|
definition: `
|
||||||
|
id: normalization-rule
|
||||||
|
version: "1"
|
||||||
|
messages:
|
||||||
|
- role: user
|
||||||
|
content: test
|
||||||
|
output:
|
||||||
|
format: json
|
||||||
|
validation_mode: json_schema
|
||||||
|
schema_path: ' schema.json '
|
||||||
|
`,
|
||||||
|
wantSchemaPath: "schema.json",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
repo := NewFSRepository(fstest.MapFS{
|
||||||
|
"definition.yaml": &fstest.MapFile{Data: []byte(tt.definition)},
|
||||||
|
}, ".")
|
||||||
|
|
||||||
|
got, err := repo.GetPromptDefinition(context.Background(), "normalization-rule", "")
|
||||||
|
if tt.wantErr {
|
||||||
|
if !errors.Is(err, ErrInvalidPromptDefinition) {
|
||||||
|
t.Fatalf("expected ErrInvalidPromptDefinition, got %v", err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), tt.wantDiagnostic) {
|
||||||
|
t.Fatalf("expected error containing %q, got %v", tt.wantDiagnostic, err)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("load prompt definition: %v", err)
|
||||||
|
}
|
||||||
|
if got.Validation.SchemaPath != tt.wantSchemaPath {
|
||||||
|
t.Fatalf("schema path = %q, want %q", got.Validation.SchemaPath, tt.wantSchemaPath)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type promptRepositorySource struct {
|
||||||
|
name string
|
||||||
|
newRepository func(t *testing.T, files map[string]string) Repository
|
||||||
|
}
|
||||||
|
|
||||||
|
func promptRepositorySources() []promptRepositorySource {
|
||||||
|
return []promptRepositorySource{
|
||||||
|
{
|
||||||
|
name: "operating system",
|
||||||
|
newRepository: func(t *testing.T, files map[string]string) Repository {
|
||||||
|
t.Helper()
|
||||||
|
root := t.TempDir()
|
||||||
|
for name, content := range files {
|
||||||
|
filePath := filepath.Join(root, filepath.FromSlash(name))
|
||||||
|
if err := os.MkdirAll(filepath.Dir(filePath), 0o755); err != nil {
|
||||||
|
t.Fatalf("create prompt directory: %v", err)
|
||||||
|
}
|
||||||
|
writePromptTestFile(t, filePath, content)
|
||||||
|
}
|
||||||
|
return NewFilesystemRepository(root)
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "filesystem",
|
||||||
|
newRepository: func(t *testing.T, files map[string]string) Repository {
|
||||||
|
t.Helper()
|
||||||
|
fsys := make(fstest.MapFS, len(files))
|
||||||
|
for name, content := range files {
|
||||||
|
fsys[name] = &fstest.MapFile{Data: []byte(strings.TrimLeft(content, "\n"))}
|
||||||
|
}
|
||||||
|
return NewFSRepository(fsys, ".")
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func BenchmarkPromptRepositoryLookup(b *testing.B) {
|
||||||
|
for _, size := range []int{10, 1000} {
|
||||||
|
b.Run(fmt.Sprintf("catalog-%d", size), func(b *testing.B) {
|
||||||
|
files := fstest.MapFS{
|
||||||
|
"target.yaml": &fstest.MapFile{Data: []byte(`
|
||||||
|
id: target
|
||||||
|
version: "1"
|
||||||
|
messages:
|
||||||
|
- role: user
|
||||||
|
content_file: target.tmpl
|
||||||
|
output:
|
||||||
|
format: text
|
||||||
|
validation_mode: none
|
||||||
|
`)},
|
||||||
|
"target.tmpl": &fstest.MapFile{Data: []byte("selected")},
|
||||||
|
}
|
||||||
|
metadataNames := []string{"target.yaml"}
|
||||||
|
contentNames := make([]string, 0, size-1)
|
||||||
|
for i := 1; i < size; i++ {
|
||||||
|
definitionName := fmt.Sprintf("prompt-%04d.yaml", i)
|
||||||
|
contentName := fmt.Sprintf("prompt-%04d.tmpl", i)
|
||||||
|
files[definitionName] = &fstest.MapFile{Data: []byte(fmt.Sprintf(`
|
||||||
|
id: prompt-%04d
|
||||||
|
version: "1"
|
||||||
|
messages:
|
||||||
|
- role: user
|
||||||
|
content_file: %s
|
||||||
|
output:
|
||||||
|
format: text
|
||||||
|
validation_mode: none
|
||||||
|
`, i, contentName))}
|
||||||
|
files[contentName] = &fstest.MapFile{Data: []byte("unrelated")}
|
||||||
|
metadataNames = append(metadataNames, definitionName)
|
||||||
|
contentNames = append(contentNames, contentName)
|
||||||
|
}
|
||||||
|
|
||||||
|
fsys := &recordingFS{FS: files}
|
||||||
|
repo := NewFSRepository(fsys, ".")
|
||||||
|
b.ReportAllocs()
|
||||||
|
b.ResetTimer()
|
||||||
|
for i := 0; i < b.N; i++ {
|
||||||
|
if _, err := repo.GetPromptDefinition(context.Background(), "target", "1"); err != nil {
|
||||||
|
b.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
b.StopTimer()
|
||||||
|
|
||||||
|
if got := fsys.openCount("target.tmpl"); got != b.N {
|
||||||
|
b.Fatalf("selected content opens = %d, want %d", got, b.N)
|
||||||
|
}
|
||||||
|
for _, name := range contentNames {
|
||||||
|
if got := fsys.openCount(name); got != 0 {
|
||||||
|
b.Fatalf("unrelated content %q opened %d times", name, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, name := range metadataNames {
|
||||||
|
if got := fsys.openCount(name); got != b.N {
|
||||||
|
b.Fatalf("metadata %q opens = %d, want %d", name, got, b.N)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
63
internal/promptdef/source.go
Normal file
63
internal/promptdef/source.go
Normal file
@@ -0,0 +1,63 @@
|
|||||||
|
package promptdef
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"io/fs"
|
||||||
|
"os"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/filecatalog"
|
||||||
|
)
|
||||||
|
|
||||||
|
type promptDefinitionSource interface {
|
||||||
|
contentSourceRoot
|
||||||
|
findYAMLFiles(context.Context) ([]string, error)
|
||||||
|
readDefinition(string) ([]byte, error)
|
||||||
|
displayPath(string) string
|
||||||
|
}
|
||||||
|
|
||||||
|
type osPromptSource struct {
|
||||||
|
root string
|
||||||
|
contentRoot osContentSourceRoot
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s osPromptSource) findYAMLFiles(ctx context.Context) ([]string, error) {
|
||||||
|
return filecatalog.FindYAMLFiles(ctx, s.root)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s osPromptSource) readDefinition(name string) ([]byte, error) {
|
||||||
|
return os.ReadFile(name)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s osPromptSource) displayPath(name string) string {
|
||||||
|
return filecatalog.RelativePath(s.root, name)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s osPromptSource) readContentFile(sourcePath string, contentFile string) (string, string, error) {
|
||||||
|
return s.contentRoot.readContentFile(sourcePath, contentFile)
|
||||||
|
}
|
||||||
|
|
||||||
|
type fsPromptSource struct {
|
||||||
|
fsys fs.FS
|
||||||
|
root string
|
||||||
|
contentRoot contentSourceRoot
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s fsPromptSource) findYAMLFiles(ctx context.Context) ([]string, error) {
|
||||||
|
if s.fsys == nil {
|
||||||
|
return nil, errors.New("filesystem is nil")
|
||||||
|
}
|
||||||
|
return filecatalog.FindFSYAMLFiles(ctx, s.fsys, s.root)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s fsPromptSource) readDefinition(name string) ([]byte, error) {
|
||||||
|
return fs.ReadFile(s.fsys, name)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s fsPromptSource) displayPath(name string) string {
|
||||||
|
return filecatalog.DisplayPath(s.root, name)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s fsPromptSource) readContentFile(sourcePath string, contentFile string) (string, string, error) {
|
||||||
|
return s.contentRoot.readContentFile(sourcePath, contentFile)
|
||||||
|
}
|
||||||
24
internal/usecase/capacity_error.go
Normal file
24
internal/usecase/capacity_error.go
Normal file
@@ -0,0 +1,24 @@
|
|||||||
|
package usecase
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/capacity"
|
||||||
|
)
|
||||||
|
|
||||||
|
// CapacityError identifies bounded admission rejected for one selected backend.
|
||||||
|
type CapacityError struct {
|
||||||
|
BackendID string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *CapacityError) Error() string {
|
||||||
|
if e == nil || strings.TrimSpace(e.BackendID) == "" {
|
||||||
|
return capacity.ErrCapacityExceeded.Error()
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("backend %q admission: %v", e.BackendID, capacity.ErrCapacityExceeded)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *CapacityError) Unwrap() error {
|
||||||
|
return capacity.ErrCapacityExceeded
|
||||||
|
}
|
||||||
83
internal/usecase/execution_settings_test.go
Normal file
83
internal/usecase/execution_settings_test.go
Normal file
@@ -0,0 +1,83 @@
|
|||||||
|
package usecase
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"math"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestRunnerPrepareExecutionRejectsInvalidExecutionSettings(t *testing.T) {
|
||||||
|
runner := NewRunner(
|
||||||
|
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
|
||||||
|
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
||||||
|
nil,
|
||||||
|
defaultArtifactReader(),
|
||||||
|
defaultRenderer(),
|
||||||
|
&fakeLLM{forbid: true},
|
||||||
|
nil,
|
||||||
|
nil,
|
||||||
|
)
|
||||||
|
|
||||||
|
_, err := runner.PrepareExecution(context.Background(), domain.RunRequest{
|
||||||
|
PromptID: "p",
|
||||||
|
ProfileID: "exec",
|
||||||
|
Inputs: singleInputRef(),
|
||||||
|
Execution: &domain.ExecutionTargetOverride{TopP: float64Ptr(math.Inf(-1))},
|
||||||
|
})
|
||||||
|
if !errors.Is(err, ErrInvalidRequest) {
|
||||||
|
t.Fatalf("expected ErrInvalidRequest, got %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunnerPrepareExecutionValidatesAndNormalizesRequestEndpoints(t *testing.T) {
|
||||||
|
newRunner := func() *Runner {
|
||||||
|
return NewRunner(
|
||||||
|
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
|
||||||
|
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
||||||
|
nil,
|
||||||
|
defaultArtifactReader(),
|
||||||
|
defaultRenderer(),
|
||||||
|
&fakeLLM{forbid: true},
|
||||||
|
nil,
|
||||||
|
nil,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
invalidEndpoints := []string{
|
||||||
|
"/v1",
|
||||||
|
"https:///v1",
|
||||||
|
"ftp://provider.example/v1",
|
||||||
|
"https://user@provider.example/v1",
|
||||||
|
"https://provider.example/v1?mode=chat",
|
||||||
|
"https://provider.example/v1#chat",
|
||||||
|
}
|
||||||
|
for _, endpoint := range invalidEndpoints {
|
||||||
|
t.Run(endpoint, func(t *testing.T) {
|
||||||
|
_, err := newRunner().PrepareExecution(context.Background(), domain.RunRequest{
|
||||||
|
PromptID: "p",
|
||||||
|
ProfileID: "exec",
|
||||||
|
Inputs: singleInputRef(),
|
||||||
|
Execution: &domain.ExecutionTargetOverride{Endpoint: endpoint},
|
||||||
|
})
|
||||||
|
if !errors.Is(err, ErrInvalidRequest) {
|
||||||
|
t.Fatalf("expected ErrInvalidRequest, got %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
prepared, err := newRunner().PrepareExecution(context.Background(), domain.RunRequest{
|
||||||
|
PromptID: "p",
|
||||||
|
ProfileID: "exec",
|
||||||
|
Inputs: singleInputRef(),
|
||||||
|
Execution: &domain.ExecutionTargetOverride{Endpoint: " https://provider.example/nested/v1 "},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("prepare normalized endpoint: %v", err)
|
||||||
|
}
|
||||||
|
if got := prepared.Details().EffectiveModelParams.Endpoint; got != "https://provider.example/nested/v1" {
|
||||||
|
t.Fatalf("effective endpoint = %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
19
internal/usecase/generation_request.go
Normal file
19
internal/usecase/generation_request.go
Normal file
@@ -0,0 +1,19 @@
|
|||||||
|
package usecase
|
||||||
|
|
||||||
|
import "gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||||
|
|
||||||
|
func newGenerationRequest(
|
||||||
|
prompt domain.RenderedPrompt,
|
||||||
|
sessionID string,
|
||||||
|
target domain.ExecutionTarget,
|
||||||
|
targetPresence domain.ExecutionTargetPresence,
|
||||||
|
structuredOutput *domain.StructuredOutputSpec,
|
||||||
|
) domain.GenerateRequest {
|
||||||
|
prompt.SessionID = sessionID
|
||||||
|
return domain.GenerateRequest{
|
||||||
|
Prompt: prompt,
|
||||||
|
Target: target,
|
||||||
|
TargetPresence: targetPresence,
|
||||||
|
StructuredOutput: structuredOutput,
|
||||||
|
}
|
||||||
|
}
|
||||||
185
internal/usecase/output_contract_test.go
Normal file
185
internal/usecase/output_contract_test.go
Normal file
@@ -0,0 +1,185 @@
|
|||||||
|
package usecase
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||||
|
)
|
||||||
|
|
||||||
|
type outputContractTestCollaborators struct {
|
||||||
|
artifacts *fakeArtifactReader
|
||||||
|
renderer *fakeRenderer
|
||||||
|
llm *fakeLLM
|
||||||
|
validator *recordingValidationPreparer
|
||||||
|
admitter *fakeRunAdmitter
|
||||||
|
}
|
||||||
|
|
||||||
|
func newOutputContractTestRunner() (*Runner, outputContractTestCollaborators) {
|
||||||
|
collaborators := outputContractTestCollaborators{
|
||||||
|
artifacts: defaultArtifactReader(),
|
||||||
|
renderer: defaultRenderer(),
|
||||||
|
llm: &fakeLLM{forbid: true},
|
||||||
|
validator: &recordingValidationPreparer{plan: &recordingPreparedValidation{}},
|
||||||
|
admitter: &fakeRunAdmitter{},
|
||||||
|
}
|
||||||
|
return NewRunner(
|
||||||
|
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
|
||||||
|
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
||||||
|
nil,
|
||||||
|
collaborators.artifacts,
|
||||||
|
collaborators.renderer,
|
||||||
|
collaborators.llm,
|
||||||
|
collaborators.validator,
|
||||||
|
collaborators.admitter,
|
||||||
|
), collaborators
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunnerPreparationNormalizesOutputContractConsistently(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
override domain.OutputContract
|
||||||
|
want domain.OutputContract
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "empty replacement format defaults to text",
|
||||||
|
override: domain.OutputContract{ValidationMode: domain.ValidationNone},
|
||||||
|
want: domain.OutputContract{Format: domain.FormatText, ValidationMode: domain.ValidationNone},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "markdown basic replacement",
|
||||||
|
override: domain.OutputContract{Format: domain.FormatMarkdown, ValidationMode: domain.ValidationBasic},
|
||||||
|
want: domain.OutputContract{Format: domain.FormatMarkdown, ValidationMode: domain.ValidationBasic},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "json replacement preserves non-schema fields",
|
||||||
|
override: domain.OutputContract{
|
||||||
|
Format: domain.FormatJSON,
|
||||||
|
ValidationMode: domain.ValidationJSON,
|
||||||
|
SchemaPath: "ignored.json",
|
||||||
|
RepairAttempts: 2,
|
||||||
|
},
|
||||||
|
want: domain.OutputContract{
|
||||||
|
Format: domain.FormatJSON,
|
||||||
|
ValidationMode: domain.ValidationJSON,
|
||||||
|
SchemaPath: "ignored.json",
|
||||||
|
RepairAttempts: 2,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
runner, _ := newOutputContractTestRunner()
|
||||||
|
req := domain.RunRequest{
|
||||||
|
PromptID: "p",
|
||||||
|
ProfileID: "exec",
|
||||||
|
Inputs: singleInputRef(),
|
||||||
|
Validation: &tt.override,
|
||||||
|
}
|
||||||
|
|
||||||
|
prepared, err := runner.Prepare(context.Background(), req)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("prepare: %v", err)
|
||||||
|
}
|
||||||
|
preparedExecution, err := runner.PrepareExecution(context.Background(), req)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("prepare execution: %v", err)
|
||||||
|
}
|
||||||
|
details := preparedExecution.Details()
|
||||||
|
if details == nil {
|
||||||
|
t.Fatal("prepared execution returned nil details")
|
||||||
|
}
|
||||||
|
if prepared.OutputContract != tt.want || details.OutputContract != tt.want {
|
||||||
|
t.Fatalf("output contracts = (%+v, %+v), want %+v", prepared.OutputContract, details.OutputContract, tt.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunnerPreparationRejectsInvalidOutputContractsBeforeCompletion(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
override domain.OutputContract
|
||||||
|
}{
|
||||||
|
{name: "unsupported format", override: domain.OutputContract{Format: "binary", ValidationMode: domain.ValidationNone}},
|
||||||
|
{name: "empty validation mode", override: domain.OutputContract{Format: domain.FormatText}},
|
||||||
|
{name: "negative repair attempts", override: domain.OutputContract{Format: domain.FormatText, ValidationMode: domain.ValidationNone, RepairAttempts: -1}},
|
||||||
|
{name: "json schema without path", override: domain.OutputContract{Format: domain.FormatJSON, ValidationMode: domain.ValidationJSONSchema}},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
for _, operation := range []string{"Prepare", "PrepareExecution"} {
|
||||||
|
t.Run(tt.name+"/"+operation, func(t *testing.T) {
|
||||||
|
runner, collaborators := newOutputContractTestRunner()
|
||||||
|
req := domain.RunRequest{
|
||||||
|
PromptID: "p",
|
||||||
|
ProfileID: "exec",
|
||||||
|
Inputs: singleInputRef(),
|
||||||
|
Validation: &tt.override,
|
||||||
|
}
|
||||||
|
|
||||||
|
var err error
|
||||||
|
switch operation {
|
||||||
|
case "Prepare":
|
||||||
|
var prepared *domain.PreparedRun
|
||||||
|
prepared, err = runner.Prepare(context.Background(), req)
|
||||||
|
if prepared != nil {
|
||||||
|
t.Fatalf("expected no partial prepared run, got %+v", prepared)
|
||||||
|
}
|
||||||
|
case "PrepareExecution":
|
||||||
|
var prepared *PreparedExecution
|
||||||
|
prepared, err = runner.PrepareExecution(context.Background(), req)
|
||||||
|
if prepared != nil {
|
||||||
|
t.Fatalf("expected no partial prepared execution, got %+v", prepared)
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
t.Fatalf("unknown operation %q", operation)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !errors.Is(err, ErrInvalidRequest) {
|
||||||
|
t.Fatalf("expected ErrInvalidRequest, got %v", err)
|
||||||
|
}
|
||||||
|
assertOutputContractCompletionSkipped(t, collaborators)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunnerRunRejectsInvalidOutputContractBeforeAdmission(t *testing.T) {
|
||||||
|
runner, collaborators := newOutputContractTestRunner()
|
||||||
|
result, err := runner.Run(context.Background(), domain.RunRequest{
|
||||||
|
PromptID: "p",
|
||||||
|
ProfileID: "exec",
|
||||||
|
Inputs: singleInputRef(),
|
||||||
|
Validation: &domain.OutputContract{
|
||||||
|
Format: domain.OutputFormat("binary"),
|
||||||
|
ValidationMode: domain.ValidationNone,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if result != nil {
|
||||||
|
t.Fatalf("expected no partial result, got %+v", result)
|
||||||
|
}
|
||||||
|
if !errors.Is(err, ErrInvalidRequest) {
|
||||||
|
t.Fatalf("expected ErrInvalidRequest, got %v", err)
|
||||||
|
}
|
||||||
|
assertOutputContractCompletionSkipped(t, collaborators)
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertOutputContractCompletionSkipped(t *testing.T, collaborators outputContractTestCollaborators) {
|
||||||
|
t.Helper()
|
||||||
|
if collaborators.artifacts.calls != 0 || collaborators.renderer.calls != 0 ||
|
||||||
|
collaborators.validator.prepareCalls != 0 || collaborators.validator.directValidateCalls != 0 ||
|
||||||
|
len(collaborators.admitter.backendIDs) != 0 || collaborators.llm.calls != 0 {
|
||||||
|
t.Fatalf(
|
||||||
|
"invalid output contract reached downstream work: artifacts=%d renderer=%d prepare_validation=%d validation=%d admissions=%d generation=%d",
|
||||||
|
collaborators.artifacts.calls,
|
||||||
|
collaborators.renderer.calls,
|
||||||
|
collaborators.validator.prepareCalls,
|
||||||
|
collaborators.validator.directValidateCalls,
|
||||||
|
len(collaborators.admitter.backendIDs),
|
||||||
|
collaborators.llm.calls,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
245
internal/usecase/prepared_execution.go
Normal file
245
internal/usecase/prepared_execution.go
Normal file
@@ -0,0 +1,245 @@
|
|||||||
|
package usecase
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/jsonvalue"
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/validate"
|
||||||
|
)
|
||||||
|
|
||||||
|
type preparedExecutionState uint8
|
||||||
|
|
||||||
|
const (
|
||||||
|
preparedExecutionReady preparedExecutionState = iota
|
||||||
|
preparedExecutionClaimed
|
||||||
|
preparedExecutionDiscarded
|
||||||
|
)
|
||||||
|
|
||||||
|
// PreparedExecution owns one frozen, single-use runner execution.
|
||||||
|
type PreparedExecution struct {
|
||||||
|
owner *Runner
|
||||||
|
mu sync.Mutex
|
||||||
|
state preparedExecutionState
|
||||||
|
details *domain.PreparedRun
|
||||||
|
payload *preparedExecutionPayload
|
||||||
|
}
|
||||||
|
|
||||||
|
type preparedExecutionPayload struct {
|
||||||
|
prepared *domain.PreparedRun
|
||||||
|
validation validate.PreparedValidation
|
||||||
|
directKey string
|
||||||
|
}
|
||||||
|
|
||||||
|
// PrepareExecution completes preparation without generation or admission and
|
||||||
|
// returns a runner-bound, single-use execution.
|
||||||
|
func (r *Runner) PrepareExecution(ctx context.Context, req domain.RunRequest) (*PreparedExecution, error) {
|
||||||
|
state, err := r.resolvePreparation(ctx, req, time.Now().UTC())
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
operation, err := r.completePreparation(ctx, req, state)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
executionSnapshot, err := clonePreparedRun(operation.run)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("%w: failed to copy prepared execution: %v", ErrInvalidRequest, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
details, err := clonePreparedRun(executionSnapshot)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("%w: failed to copy prepared execution details: %v", ErrInvalidRequest, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return &PreparedExecution{
|
||||||
|
owner: r,
|
||||||
|
state: preparedExecutionReady,
|
||||||
|
details: details,
|
||||||
|
payload: &preparedExecutionPayload{
|
||||||
|
prepared: executionSnapshot,
|
||||||
|
validation: operation.validation,
|
||||||
|
directKey: state.effectiveModel.APIKey,
|
||||||
|
},
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Details returns a fresh credential-redacted copy of the prepared run.
|
||||||
|
func (p *PreparedExecution) Details() *domain.PreparedRun {
|
||||||
|
if p == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
p.mu.Lock()
|
||||||
|
detailsSnapshot := p.details
|
||||||
|
p.mu.Unlock()
|
||||||
|
|
||||||
|
details, err := clonePreparedRun(detailsSnapshot)
|
||||||
|
if err != nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return details
|
||||||
|
}
|
||||||
|
|
||||||
|
// Discard invalidates an unclaimed execution and drops its private payload.
|
||||||
|
func (p *PreparedExecution) Discard() {
|
||||||
|
if p == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
p.mu.Lock()
|
||||||
|
if p.state != preparedExecutionReady {
|
||||||
|
p.mu.Unlock()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
p.state = preparedExecutionDiscarded
|
||||||
|
payload := p.payload
|
||||||
|
p.payload = nil
|
||||||
|
p.mu.Unlock()
|
||||||
|
|
||||||
|
payload.clear()
|
||||||
|
}
|
||||||
|
|
||||||
|
// RunPrepared claims and executes one prepared execution owned by this runner.
|
||||||
|
func (r *Runner) RunPrepared(ctx context.Context, prepared *PreparedExecution) (*domain.RunResult, error) {
|
||||||
|
payload, err := prepared.claim(r)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer payload.clear()
|
||||||
|
|
||||||
|
runID, err := newRunID()
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to create run id: %w", err)
|
||||||
|
}
|
||||||
|
start := time.Now().UTC()
|
||||||
|
|
||||||
|
target := payload.prepared.EffectiveModelParams
|
||||||
|
if err := validateAPIKey(target.APIKeyEnv, payload.directKey, target.APIKeyRequired); err != nil {
|
||||||
|
return nil, fmt.Errorf("%w: %w", ErrInvalidRequest, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
release, err := r.admitRun(ctx, target.BackendID)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer release()
|
||||||
|
|
||||||
|
return r.executePreparedRun(ctx, payload.prepared, payload.directKey, runID, start, func(
|
||||||
|
ctx context.Context,
|
||||||
|
artifact *domain.Artifact,
|
||||||
|
attemptsUsed int,
|
||||||
|
) (domain.ValidationResult, error) {
|
||||||
|
result, validationErr := payload.validation.Validate(ctx, artifact)
|
||||||
|
if validationErr != nil {
|
||||||
|
return domain.ValidationResult{}, validationErr
|
||||||
|
}
|
||||||
|
result.RepairAttempts = attemptsUsed
|
||||||
|
return result, nil
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *PreparedExecution) claim(owner *Runner) (*preparedExecutionPayload, error) {
|
||||||
|
if p == nil || owner == nil || p.owner != owner {
|
||||||
|
return nil, fmt.Errorf("%w: prepared execution does not belong to this runner", ErrInvalidRequest)
|
||||||
|
}
|
||||||
|
|
||||||
|
p.mu.Lock()
|
||||||
|
defer p.mu.Unlock()
|
||||||
|
if p.state != preparedExecutionReady || p.payload == nil {
|
||||||
|
return nil, fmt.Errorf("%w: prepared execution is not ready", ErrInvalidRequest)
|
||||||
|
}
|
||||||
|
p.state = preparedExecutionClaimed
|
||||||
|
payload := p.payload
|
||||||
|
p.payload = nil
|
||||||
|
return payload, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *preparedExecutionPayload) clear() {
|
||||||
|
if p == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if p.prepared != nil {
|
||||||
|
p.prepared.EffectiveModelParams.APIKey = ""
|
||||||
|
}
|
||||||
|
p.prepared = nil
|
||||||
|
p.validation = nil
|
||||||
|
p.directKey = ""
|
||||||
|
}
|
||||||
|
|
||||||
|
type noOpPreparedValidation struct {
|
||||||
|
contract domain.OutputContract
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p noOpPreparedValidation) Validate(
|
||||||
|
ctx context.Context,
|
||||||
|
_ *domain.Artifact,
|
||||||
|
) (domain.ValidationResult, error) {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return domain.ValidationResult{}, err
|
||||||
|
}
|
||||||
|
return domain.ValidationResult{
|
||||||
|
Status: domain.ValidationSkipped,
|
||||||
|
Mode: p.contract.ValidationMode,
|
||||||
|
SchemaPath: p.contract.SchemaPath,
|
||||||
|
RepairAttempts: p.contract.RepairAttempts,
|
||||||
|
IsValid: true,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (noOpPreparedValidation) SchemaDocument() any {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func clonePreparedRun(source *domain.PreparedRun) (*domain.PreparedRun, error) {
|
||||||
|
if source == nil {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
copied := *source
|
||||||
|
extraParams, err := jsonvalue.CopyMap(source.EffectiveModelParams.ExtraParams)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
copied.EffectiveModelParams.ExtraParams = extraParams
|
||||||
|
|
||||||
|
if source.InputHashes != nil {
|
||||||
|
copied.InputHashes = make(map[string]string, len(source.InputHashes))
|
||||||
|
for name, hash := range source.InputHashes {
|
||||||
|
copied.InputHashes[name] = hash
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if source.Messages != nil {
|
||||||
|
copied.Messages = make([]domain.RenderedMessage, len(source.Messages))
|
||||||
|
for i, message := range source.Messages {
|
||||||
|
copied.Messages[i] = message
|
||||||
|
if message.CacheControl != nil {
|
||||||
|
cacheControl := *message.CacheControl
|
||||||
|
copied.Messages[i].CacheControl = &cacheControl
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if source.StructuredOutput != nil {
|
||||||
|
structuredOutput := *source.StructuredOutput
|
||||||
|
copied.StructuredOutput = &structuredOutput
|
||||||
|
if source.StructuredOutput.JSONSchema != nil {
|
||||||
|
jsonSchema := *source.StructuredOutput.JSONSchema
|
||||||
|
copied.StructuredOutput.JSONSchema = &jsonSchema
|
||||||
|
schema, copyErr := cloneJSONValue(source.StructuredOutput.JSONSchema.Schema)
|
||||||
|
if copyErr != nil {
|
||||||
|
return nil, copyErr
|
||||||
|
}
|
||||||
|
copied.StructuredOutput.JSONSchema.Schema = schema
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return &copied, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func cloneJSONValue(source any) (any, error) {
|
||||||
|
return jsonvalue.Copy(source)
|
||||||
|
}
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user