Compare commits
30 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 31f2ce3a09 | |||
| fd06e4ca6b | |||
| e63b8de1e9 | |||
| 9354d2b373 | |||
| 01ca5430bd | |||
| ae2179d103 | |||
| a248433d0f | |||
| bd6cffc9d0 | |||
| e40c4f182b | |||
| 7428e50c2c | |||
| 63c67a4520 | |||
| 25a7052a3d | |||
| fc3255967e | |||
| e920168b30 | |||
| 272b6a4bc1 | |||
| dde48a31fc | |||
| 242eace4a7 | |||
| 0bf5f88136 | |||
| 369ab5392d | |||
| 2ba0146e5d | |||
| 6112c2af0c | |||
| f5e12c00f5 | |||
| 49fe402dd2 | |||
| c301eb8d55 | |||
| c13e9710d9 | |||
| 87b5ec3d75 | |||
| cb4028a637 | |||
| 5a1bff4529 | |||
| 805a7f965d | |||
| 147f5e5ff5 |
@@ -33,6 +33,15 @@ boundary and constraints that framework work must preserve.
|
|||||||
|
|
||||||
## Release Guidance
|
## Release Guidance
|
||||||
|
|
||||||
|
Consumers upgrading from `v0.4.0` to `v0.5.0` should read the
|
||||||
|
[v0.5.0 changelog and migration guide](docs/releases/v0.5.0.md).
|
||||||
|
|
||||||
|
Consumers upgrading from `v0.3.0` to `v0.4.0` should read the
|
||||||
|
[v0.4.0 changelog and adoption guide](docs/releases/v0.4.0.md).
|
||||||
|
|
||||||
|
Consumers upgrading from `v0.2.0` to `v0.3.0` should read the
|
||||||
|
[v0.3.0 changelog](docs/releases/v0.3.0.md).
|
||||||
|
|
||||||
Consumers moving from `v0.1.0` to `v0.2.0` should read the
|
Consumers moving from `v0.1.0` to `v0.2.0` should read the
|
||||||
[v0.2.0 changelog and migration guide](docs/releases/v0.2.0.md).
|
[v0.2.0 changelog and migration guide](docs/releases/v0.2.0.md).
|
||||||
|
|
||||||
|
|||||||
37
backends.go
37
backends.go
@@ -9,6 +9,11 @@ import (
|
|||||||
// backend.
|
// backend.
|
||||||
const BackendOpenRouter = backend.OpenRouterID
|
const BackendOpenRouter = backend.OpenRouterID
|
||||||
|
|
||||||
|
// BackendLocal is the case-sensitive conventional ID used by [LocalBackend].
|
||||||
|
// It is not a built-in or reserved backend and must be registered with
|
||||||
|
// [WithBackend].
|
||||||
|
const BackendLocal = "local"
|
||||||
|
|
||||||
// Backend configures one engine-scoped OpenAI-compatible backend.
|
// Backend configures one engine-scoped OpenAI-compatible backend.
|
||||||
//
|
//
|
||||||
// Backend has no stable JSON representation. Use keyed literals so additions
|
// Backend has no stable JSON representation. Use keyed literals so additions
|
||||||
@@ -35,15 +40,35 @@ type Backend struct {
|
|||||||
// calls allowed for this backend within one Engine. Zero leaves the backend
|
// calls allowed for this backend within one Engine. Zero leaves the backend
|
||||||
// unlimited. A negative value makes NewEngine fail with ErrInvalidConfig.
|
// unlimited. A negative value makes NewEngine fail with ErrInvalidConfig.
|
||||||
ConcurrencyLimit int
|
ConcurrencyLimit int
|
||||||
// QueueCapacity controls how many additional Run calls may be admitted
|
// QueueCapacity controls how many additional Run or RunPrepared calls may
|
||||||
// beyond ConcurrencyLimit. Nil uses 1024 when ConcurrencyLimit is positive;
|
// be admitted beyond ConcurrencyLimit. Nil uses 1024 when ConcurrencyLimit
|
||||||
// a pointer uses its exact value, including zero. The pointed-to value must
|
// is positive; a pointer uses its exact value, including zero. The pointed-to
|
||||||
// be non-negative, and QueueCapacity must be nil when ConcurrencyLimit is
|
// value must be non-negative, and QueueCapacity must be nil when
|
||||||
// zero. Their sum must fit in an int. WithBackend copies the value and does
|
// ConcurrencyLimit is zero. Their sum must fit in an int. WithBackend copies
|
||||||
// not retain the pointer.
|
// the value and does not retain the pointer.
|
||||||
QueueCapacity *int
|
QueueCapacity *int
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// LocalBackend returns a caller-owned Backend for a conventional local
|
||||||
|
// OpenAI-compatible endpoint. It sets ID to BackendLocal and copies endpoint
|
||||||
|
// and concurrencyLimit into Endpoint and ConcurrencyLimit without
|
||||||
|
// normalization or validation. APIKeyEnv, ExtraParams, and QueueCapacity keep
|
||||||
|
// their zero values.
|
||||||
|
//
|
||||||
|
// LocalBackend does not read environment variables, register the value, or
|
||||||
|
// mutate engine or package state. Supply the returned value through
|
||||||
|
// [WithBackend]; [NewEngine] then applies the ordinary backend validation and
|
||||||
|
// concurrency semantics, including default queue capacity for a positive
|
||||||
|
// limit, unlimited behavior for zero, and ErrInvalidConfig for a negative
|
||||||
|
// limit.
|
||||||
|
func LocalBackend(endpoint string, concurrencyLimit int) Backend {
|
||||||
|
return Backend{
|
||||||
|
ID: BackendLocal,
|
||||||
|
Endpoint: endpoint,
|
||||||
|
ConcurrencyLimit: concurrencyLimit,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// WithBackend adds one Backend registration to the constructed Engine.
|
// WithBackend adds one Backend registration to the constructed Engine.
|
||||||
//
|
//
|
||||||
// Registrations accumulate in option order. Every normalized ID must be unique
|
// Registrations accumulate in option order. Every normalized ID must be unique
|
||||||
|
|||||||
@@ -60,7 +60,18 @@ func TestEngineRejectsRunBeforeCompletionWhenAdmissionIsFull(t *testing.T) {
|
|||||||
|
|
||||||
awaitCapacitySignal(t, reader.entered, "first artifact read")
|
awaitCapacitySignal(t, reader.entered, "first artifact read")
|
||||||
|
|
||||||
result, err := engine.Run(context.Background(), capacityInputRequest("http://second.example/v1"))
|
canceledContext, cancel := context.WithCancel(context.Background())
|
||||||
|
cancel()
|
||||||
|
result, err := engine.Run(canceledContext, capacityInputRequest("http://canceled.example/v1"))
|
||||||
|
if result != nil || !errors.Is(err, context.Canceled) {
|
||||||
|
t.Fatalf("canceled capacity admission=(%+v, %v), want context cancellation", result, err)
|
||||||
|
}
|
||||||
|
var canceledCapacityErr *promptkit.CapacityError
|
||||||
|
if errors.Is(err, promptkit.ErrCapacityExceeded) || errors.As(err, &canceledCapacityErr) {
|
||||||
|
t.Fatalf("canceled admission exposed capacity rejection: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
result, err = engine.Run(context.Background(), capacityInputRequest("http://second.example/v1"))
|
||||||
if result != nil {
|
if result != nil {
|
||||||
t.Fatalf("capacity rejection returned partial result: %+v", result)
|
t.Fatalf("capacity rejection returned partial result: %+v", result)
|
||||||
}
|
}
|
||||||
@@ -70,6 +81,21 @@ func TestEngineRejectsRunBeforeCompletionWhenAdmissionIsFull(t *testing.T) {
|
|||||||
if errors.Is(err, promptkit.ErrInvalidRequest) || errors.Is(err, promptkit.ErrLLMGenerate) {
|
if errors.Is(err, promptkit.ErrInvalidRequest) || errors.Is(err, promptkit.ErrLLMGenerate) {
|
||||||
t.Fatalf("capacity rejection had an unrelated category: %v", err)
|
t.Fatalf("capacity rejection had an unrelated category: %v", err)
|
||||||
}
|
}
|
||||||
|
var capacityErr *promptkit.CapacityError
|
||||||
|
if !errors.As(err, &capacityErr) || capacityErr == nil {
|
||||||
|
t.Fatalf("capacity rejection=%v, want CapacityError", err)
|
||||||
|
}
|
||||||
|
if capacityErr.BackendID != "limited" {
|
||||||
|
t.Fatalf("capacity backend ID=%q, want limited", capacityErr.BackendID)
|
||||||
|
}
|
||||||
|
capacityErr.BackendID = "changed"
|
||||||
|
|
||||||
|
result, err = engine.Run(context.Background(), capacityInputRequest("http://third.example/v1"))
|
||||||
|
var subsequentCapacityErr *promptkit.CapacityError
|
||||||
|
if result != nil || !errors.As(err, &subsequentCapacityErr) ||
|
||||||
|
subsequentCapacityErr == nil || subsequentCapacityErr.BackendID != "limited" {
|
||||||
|
t.Fatalf("subsequent capacity rejection=(%+v, %v), want independent limited CapacityError", result, err)
|
||||||
|
}
|
||||||
if calls := reader.callCount(); calls != 1 {
|
if calls := reader.callCount(); calls != 1 {
|
||||||
t.Fatalf("artifact calls=%d, want only the admitted run", calls)
|
t.Fatalf("artifact calls=%d, want only the admitted run", calls)
|
||||||
}
|
}
|
||||||
@@ -174,6 +200,19 @@ func TestCapacityExceededSentinelContract(t *testing.T) {
|
|||||||
if promptkit.ErrCapacityExceeded == nil {
|
if promptkit.ErrCapacityExceeded == nil {
|
||||||
t.Fatal("ErrCapacityExceeded is nil")
|
t.Fatal("ErrCapacityExceeded is nil")
|
||||||
}
|
}
|
||||||
|
var nilCapacityErr *promptkit.CapacityError
|
||||||
|
zeroCapacityErr := &promptkit.CapacityError{}
|
||||||
|
populatedCapacityErr := &promptkit.CapacityError{BackendID: "limited"}
|
||||||
|
for _, capacityErr := range []error{nilCapacityErr, zeroCapacityErr} {
|
||||||
|
if !errors.Is(capacityErr, promptkit.ErrCapacityExceeded) {
|
||||||
|
t.Fatalf("capacity error=%v, want ErrCapacityExceeded", capacityErr)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
var discoveredCapacityErr *promptkit.CapacityError
|
||||||
|
if !errors.As(populatedCapacityErr, &discoveredCapacityErr) || discoveredCapacityErr != populatedCapacityErr {
|
||||||
|
t.Fatalf("populated capacity error is not discoverable: %v", populatedCapacityErr)
|
||||||
|
}
|
||||||
|
|
||||||
for _, unrelated := range []error{
|
for _, unrelated := range []error{
|
||||||
promptkit.ErrInvalidConfig,
|
promptkit.ErrInvalidConfig,
|
||||||
promptkit.ErrInvalidRequest,
|
promptkit.ErrInvalidRequest,
|
||||||
@@ -181,7 +220,8 @@ func TestCapacityExceededSentinelContract(t *testing.T) {
|
|||||||
promptkit.ErrValidation,
|
promptkit.ErrValidation,
|
||||||
} {
|
} {
|
||||||
if errors.Is(promptkit.ErrCapacityExceeded, unrelated) ||
|
if errors.Is(promptkit.ErrCapacityExceeded, unrelated) ||
|
||||||
errors.Is(unrelated, promptkit.ErrCapacityExceeded) {
|
errors.Is(unrelated, promptkit.ErrCapacityExceeded) ||
|
||||||
|
errors.Is(populatedCapacityErr, unrelated) {
|
||||||
t.Fatalf("ErrCapacityExceeded aliases unrelated sentinel %v", unrelated)
|
t.Fatalf("ErrCapacityExceeded aliases unrelated sentinel %v", unrelated)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
40
capacity_error.go
Normal file
40
capacity_error.go
Normal file
@@ -0,0 +1,40 @@
|
|||||||
|
package promptkit
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// CapacityError reports bounded admission rejected for a selected backend.
|
||||||
|
//
|
||||||
|
// Engine-produced values identify only rejection at Promptkit's bounded
|
||||||
|
// [Engine.Run] or [Engine.RunPrepared] admission boundary. BackendID is the
|
||||||
|
// normalized registered backend ID used for routing and capacity; endpoint
|
||||||
|
// overrides do not change it. Every engine-produced value is nonnil and has a
|
||||||
|
// nonblank BackendID. Provider errors, active-generation waiting, and caller
|
||||||
|
// cancellation are not represented by this type.
|
||||||
|
//
|
||||||
|
// Callers own returned values and may mutate BackendID without affecting engine
|
||||||
|
// state or another error. CapacityError and its default Go encoding have no
|
||||||
|
// stable JSON contract. Consumer-constructed values do not establish that an
|
||||||
|
// engine rejected work.
|
||||||
|
type CapacityError struct {
|
||||||
|
// BackendID is the normalized registered backend ID whose admission was
|
||||||
|
// rejected.
|
||||||
|
BackendID string
|
||||||
|
}
|
||||||
|
|
||||||
|
// Error returns diagnostic wording that is not a parsing contract. It is safe
|
||||||
|
// to call on a nil receiver or a value with a blank BackendID.
|
||||||
|
func (e *CapacityError) Error() string {
|
||||||
|
if e == nil || strings.TrimSpace(e.BackendID) == "" {
|
||||||
|
return ErrCapacityExceeded.Error()
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("backend %q admission: %v", e.BackendID, ErrCapacityExceeded)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Unwrap returns ErrCapacityExceeded so errors.Is and errors.As can be used
|
||||||
|
// together. It is safe to call on a nil receiver or a zero value.
|
||||||
|
func (e *CapacityError) Unwrap() error {
|
||||||
|
return ErrCapacityExceeded
|
||||||
|
}
|
||||||
34
convert.go
34
convert.go
@@ -170,6 +170,40 @@ func fromDomainExecutionTarget(target domain.ExecutionTarget) ExecutionTarget {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func fromDomainProfileInspection(inspection *domain.ProfileInspection) *ProfileInspection {
|
||||||
|
if inspection == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return &ProfileInspection{
|
||||||
|
ProfileID: inspection.ProfileID,
|
||||||
|
EffectiveModelParams: fromDomainExecutionTarget(inspection.EffectiveModelParams),
|
||||||
|
APIKeyRequired: inspection.APIKeyRequired,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func fromDomainPromptInspection(inspection *domain.PromptInspection) *PromptInspection {
|
||||||
|
if inspection == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
inputs := make([]PromptInputDefinition, len(inspection.Inputs))
|
||||||
|
for i, input := range inspection.Inputs {
|
||||||
|
inputs[i] = PromptInputDefinition{
|
||||||
|
Name: input.Name,
|
||||||
|
Required: input.Required,
|
||||||
|
ContentType: input.ContentType,
|
||||||
|
Description: input.Description,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return &PromptInspection{
|
||||||
|
PromptID: inspection.PromptID,
|
||||||
|
PromptVersion: inspection.PromptVersion,
|
||||||
|
PromptHash: inspection.PromptHash,
|
||||||
|
DefaultProfileID: inspection.DefaultProfileID,
|
||||||
|
Inputs: inputs,
|
||||||
|
OutputContract: fromDomainOutputContract(inspection.OutputContract),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func fromDomainExecutionTargetPresence(presence domain.ExecutionTargetPresence) ExecutionTargetPresence {
|
func fromDomainExecutionTargetPresence(presence domain.ExecutionTargetPresence) ExecutionTargetPresence {
|
||||||
return ExecutionTargetPresence{
|
return ExecutionTargetPresence{
|
||||||
Temperature: presence.Temperature,
|
Temperature: presence.Temperature,
|
||||||
|
|||||||
41
doc.go
41
doc.go
@@ -3,23 +3,28 @@
|
|||||||
//
|
//
|
||||||
// Applications construct an [Engine] with [NewEngine], select filesystem or
|
// Applications construct an [Engine] with [NewEngine], select filesystem or
|
||||||
// in-memory sources and optional engine-scoped [Backend] registrations, and
|
// in-memory sources and optional engine-scoped [Backend] registrations, and
|
||||||
// call [Engine.Prepare] or [Engine.Run]. Concrete registries, repositories,
|
// call [Engine.InspectPrompt], [Engine.InspectProfile], [Engine.Prepare],
|
||||||
// validators, and the built-in OpenAI-compatible client remain internal
|
// [Engine.PrepareExecution], [Engine.Run], or [Engine.RunPrepared]. Concrete
|
||||||
// implementation details.
|
// registries, repositories, validators, and the built-in OpenAI-compatible
|
||||||
|
// client remain internal implementation details.
|
||||||
//
|
//
|
||||||
// # Concurrency and ownership
|
// # Concurrency and ownership
|
||||||
//
|
//
|
||||||
// An Engine supports concurrent Prepare and Run calls. Engine-local backend
|
// An Engine supports concurrent InspectPrompt, InspectProfile, Prepare,
|
||||||
// policies bound admitted Run calls and model generations where configured,
|
// PrepareExecution, Run, and RunPrepared calls. Engine-local backend policies
|
||||||
// while different backend pools and unlimited backends continue independently.
|
// bound admitted Run and RunPrepared calls and model generations where
|
||||||
// An injected [LLMClient] or [ArtifactReader] can therefore still receive
|
// configured, while different backend pools and unlimited backends continue
|
||||||
// concurrent calls and must be safe for that use.
|
// independently. An injected [LLMClient] or [ArtifactReader] can therefore
|
||||||
|
// still receive concurrent calls and must be safe for that use.
|
||||||
//
|
//
|
||||||
// NewEngine copies in-memory profiles and backend definitions. Prepare and Run
|
// NewEngine copies in-memory profiles and backend definitions. Prepare,
|
||||||
// copy request maps, slices, pointer values, and JSON-compatible extra
|
// PrepareExecution, and Run copy request maps, slices, pointer values, and
|
||||||
// parameters before using them. Returned values and values passed to extension
|
// JSON-compatible extra parameters before using them. InspectPrompt and
|
||||||
// interfaces are likewise isolated from engine state. Callers own those copies
|
// InspectProfile return copied inspection values. Returned values and values
|
||||||
// and may mutate them after the call that supplied or returned them.
|
// passed to extension interfaces are likewise isolated from engine state.
|
||||||
|
// Callers own those copies and may mutate them after the call that supplied or
|
||||||
|
// returned them. Returned structured errors are likewise caller-owned and may
|
||||||
|
// be mutated without affecting engine state or another error.
|
||||||
//
|
//
|
||||||
// # Security and sensitive data
|
// # Security and sensitive data
|
||||||
//
|
//
|
||||||
@@ -45,10 +50,12 @@
|
|||||||
// [GenerateResponse], [ExecutionTargetPresence], and the string value types
|
// [GenerateResponse], [ExecutionTargetPresence], and the string value types
|
||||||
// used by those values.
|
// used by those values.
|
||||||
//
|
//
|
||||||
// Construction values, including [Config], [Backend], [RunRequest],
|
// Construction, inspection, handle, and error values, including [Config],
|
||||||
// [ArtifactRef], [ExecutionTargetOverride], [Profile], and
|
// [Backend], [RunRequest], [ArtifactRef], [ExecutionTargetOverride], [Profile],
|
||||||
// [OpenAICompatibleProfileConfig], do not have stable JSON representations.
|
// [OpenAICompatibleProfileConfig], [ProfileInspection],
|
||||||
// Direct API keys are nevertheless excluded from JSON for every public value.
|
// [PromptInputDefinition], [PromptInspection], [PreparedExecution], and
|
||||||
|
// [CapacityError], do not have stable JSON representations. Direct API keys
|
||||||
|
// are nevertheless excluded from JSON for every public value.
|
||||||
//
|
//
|
||||||
// JSON timestamps use time.Time's RFC 3339 encoding and are omitted when zero.
|
// JSON timestamps use time.Time's RFC 3339 encoding and are omitted when zero.
|
||||||
// PreparedRun and RunResult durations are encoded as integer milliseconds in
|
// PreparedRun and RunResult durations are encoded as integer milliseconds in
|
||||||
|
|||||||
@@ -40,11 +40,67 @@ validation, and default transport behavior. Source discovery, format
|
|||||||
validation, and profile precedence are defined by the
|
validation, and profile precedence are defined by the
|
||||||
[framework format reference](../formats.md).
|
[framework format reference](../formats.md).
|
||||||
|
|
||||||
|
## Supply Embedded Application Defaults
|
||||||
|
|
||||||
|
Use `WithFallbackProfileFS` when an application packages profile definitions
|
||||||
|
that should apply unless an operator provides an ordinary configured profile
|
||||||
|
with the same ID. For example, an application can embed its defaults while
|
||||||
|
continuing to use `ProfileDir` for operator overrides:
|
||||||
|
|
||||||
|
```go
|
||||||
|
//go:embed profiles/*.yaml
|
||||||
|
var applicationProfiles embed.FS
|
||||||
|
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{
|
||||||
|
PromptDir: "prompts",
|
||||||
|
ProfileDir: operatorProfileDir,
|
||||||
|
},
|
||||||
|
promptkit.WithFallbackProfileFS(applicationProfiles, "profiles"),
|
||||||
|
)
|
||||||
|
```
|
||||||
|
|
||||||
|
Keep application-owned profile IDs and definitions in the embedded source.
|
||||||
|
Use the ordinary configured profile source for operator overrides. Leave
|
||||||
|
`operatorProfileDir` empty when the operator did not configure an override
|
||||||
|
directory; a non-empty path names an authoritative higher-precedence source,
|
||||||
|
so an unavailable or unreadable directory is an error rather than a reason to
|
||||||
|
fall back. The
|
||||||
|
[framework format reference](../formats.md#source-and-profile-precedence)
|
||||||
|
owns the exact profile format and lookup order; the
|
||||||
|
[`WithFallbackProfileFS` GoDoc](../../engine.go) owns its option contract and
|
||||||
|
validation rules.
|
||||||
|
|
||||||
|
## Inspect A Prompt Before Preparation
|
||||||
|
|
||||||
|
Use [`Engine.InspectPrompt`](../../engine.go) to check one configured prompt's
|
||||||
|
declared inputs and output workflow without creating placeholder inputs or
|
||||||
|
resolving a profile:
|
||||||
|
|
||||||
|
```go
|
||||||
|
inspection, err := engine.InspectPrompt(ctx, "meeting.summary", "")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, input := range inspection.Inputs {
|
||||||
|
// Compare the declared input with application configuration.
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Use this configuration-time boundary when the application needs only the
|
||||||
|
declared prompt interface. Use `InspectProfile` separately when it must also
|
||||||
|
check a configured profile. Use `Prepare` when it needs inputs, schemas, or
|
||||||
|
rendered messages, and use prepared execution when that work must remain tied
|
||||||
|
to later execution. The method's [GoDoc](../../engine.go) owns exact fields,
|
||||||
|
hash, ownership, and error semantics.
|
||||||
|
|
||||||
## Prepare Without Model Execution
|
## Prepare Without Model Execution
|
||||||
|
|
||||||
[`Engine.Prepare`](../../engine.go) resolves the selected prompt and profile,
|
[`Engine.Prepare`](../../engine.go) resolves the selected prompt and profile,
|
||||||
loads inputs and any structured-output schema, and renders messages without
|
loads inputs and any structured-output schema, and renders messages without
|
||||||
calling a model client:
|
calling a model client. Choose it when the prepared value is the final
|
||||||
|
inspection or persistence result and no later execution must be tied to that
|
||||||
|
exact snapshot:
|
||||||
|
|
||||||
```go
|
```go
|
||||||
prepared, err := engine.Prepare(ctx, promptkit.RunRequest{
|
prepared, err := engine.Prepare(ctx, promptkit.RunRequest{
|
||||||
@@ -61,12 +117,48 @@ shows a complete runnable setup with a prompt file, in-memory profile, and
|
|||||||
inline input. Exact request requirements and prepared-result fields belong to
|
inline input. Exact request requirements and prepared-result fields belong to
|
||||||
the [`RunRequest` and `PreparedRun` GoDoc](../../types.go).
|
the [`RunRequest` and `PreparedRun` GoDoc](../../types.go).
|
||||||
|
|
||||||
|
## Prepare Now And Execute The Same Snapshot Later
|
||||||
|
|
||||||
|
Use [`Engine.PrepareExecution`](../../engine.go) when an application must
|
||||||
|
inspect or persist preflight details before deciding whether to start model
|
||||||
|
work, while ensuring that later execution uses those exact rendered messages,
|
||||||
|
inputs, target settings, and validation resources:
|
||||||
|
|
||||||
|
```go
|
||||||
|
preparedExecution, err := engine.PrepareExecution(ctx, promptkit.RunRequest{
|
||||||
|
PromptID: "meeting.summary",
|
||||||
|
Inputs: map[string]promptkit.ArtifactRef{
|
||||||
|
"note": promptkit.Inline("Synthetic meeting notes"),
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer preparedExecution.Discard()
|
||||||
|
|
||||||
|
details := preparedExecution.Details()
|
||||||
|
// Inspect or persist an application-selected safe subset of details.
|
||||||
|
|
||||||
|
result, err := engine.RunPrepared(ctx, preparedExecution)
|
||||||
|
```
|
||||||
|
|
||||||
|
Preparation does not call the model or reserve backend capacity.
|
||||||
|
`RunPrepared` executes from the retained snapshot rather than reloading
|
||||||
|
consumer sources. The handle is opaque in-process state, while `Details`
|
||||||
|
contains rendered content and remains subject to the application's data
|
||||||
|
handling policy. The
|
||||||
|
[`PreparedExecution` and method GoDoc](../../prepared_execution.go) and
|
||||||
|
[engine operation GoDoc](../../engine.go) own exact lifecycle, engine-binding,
|
||||||
|
credential, cancellation, timing, and error semantics.
|
||||||
|
|
||||||
## Execute And Validate
|
## Execute And Validate
|
||||||
|
|
||||||
[`Engine.Run`](../../engine.go) performs the same preparation, invokes the
|
[`Engine.Run`](../../engine.go) performs the same preparation, invokes the
|
||||||
configured model client, classifies the generated artifact, and validates the
|
configured model client, classifies the generated artifact, and validates the
|
||||||
content. A completed content check may return `ValidationFailed` in the result;
|
content in one call. Choose it when the application does not need a preflight
|
||||||
an operational inability to validate returns an error.
|
boundary tied to the eventual execution. A completed content check may return
|
||||||
|
`ValidationFailed` in the result; an operational inability to validate returns
|
||||||
|
an error.
|
||||||
|
|
||||||
The maintained
|
The maintained
|
||||||
[offline execution example](../../examples/go-library/run/main.go) injects a
|
[offline execution example](../../examples/go-library/run/main.go) injects a
|
||||||
@@ -90,13 +182,41 @@ replace execution settings or the complete output contract.
|
|||||||
The [public value GoDoc](../../types.go) defines nil, empty, zero, replacement,
|
The [public value GoDoc](../../types.go) defines nil, empty, zero, replacement,
|
||||||
copy, and credential behavior. The
|
copy, and credential behavior. The
|
||||||
[framework format reference](../formats.md) defines how those request values
|
[framework format reference](../formats.md) defines how those request values
|
||||||
interact with prompt definitions, file-backed profiles, built-ins, schemas,
|
interact with prompt definitions, file-backed and application fallback
|
||||||
and framework defaults.
|
profiles, built-ins, schemas, and framework defaults.
|
||||||
|
|
||||||
For programmatic profiles,
|
For programmatic profiles,
|
||||||
[`OpenAICompatibleProfile`](../../profiles.go) converts ordinary
|
[`OpenAICompatibleProfile`](../../profiles.go) converts ordinary
|
||||||
OpenAI-compatible settings into a value accepted by `WithProfiles`.
|
OpenAI-compatible settings into a value accepted by `WithProfiles`.
|
||||||
|
|
||||||
|
### Inspect A Profile Before Prompt Work
|
||||||
|
|
||||||
|
Use [`Engine.InspectProfile`](../../engine.go) to validate one configured
|
||||||
|
profile without constructing a synthetic prompt or placeholder inputs. It
|
||||||
|
resolves the profile's effective target but does not prepare or execute a
|
||||||
|
prompt:
|
||||||
|
|
||||||
|
```go
|
||||||
|
inspection, err := engine.InspectProfile(ctx, profileID)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
target := inspection.EffectiveModelParams
|
||||||
|
if target.APIKeyEnv != "" {
|
||||||
|
// Apply application policy for the named environment variable.
|
||||||
|
} else if inspection.APIKeyRequired {
|
||||||
|
// Arrange a direct credential before later execution.
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Use this configuration-time boundary when only the profile and its target need
|
||||||
|
checking. Use `Prepare` when the application also needs prompt, input, schema,
|
||||||
|
or rendering work; use prepared execution when that work must remain tied to a
|
||||||
|
later execution. Inspection reports credential requirements but leaves the
|
||||||
|
timing of credential enforcement to the application. The method's
|
||||||
|
[GoDoc](../../engine.go) owns its exact result and error contract.
|
||||||
|
|
||||||
### Set A Per-Run Session And Reasoning
|
### Set A Per-Run Session And Reasoning
|
||||||
|
|
||||||
Supply a direct session ID when one prompt should be correlated with a
|
Supply a direct session ID when one prompt should be correlated with a
|
||||||
@@ -124,35 +244,86 @@ providers. The
|
|||||||
[`RunRequest` and `ExecutionTargetOverride` GoDoc](../../types.go) owns the
|
[`RunRequest` and `ExecutionTargetOverride` GoDoc](../../types.go) owns the
|
||||||
exact normalization, precedence, error, copying, and exposure contract.
|
exact normalization, precedence, error, copying, and exposure contract.
|
||||||
|
|
||||||
### Register A Custom Backend
|
### Configure A Local OpenAI-Compatible Endpoint
|
||||||
|
|
||||||
Register a reusable OpenAI-compatible connection once, then select it from a
|
Choose the smallest configuration that fits how the endpoint will be reused.
|
||||||
profile. This local backend limits model generation to two simultaneous calls;
|
|
||||||
because `QueueCapacity` is omitted, the engine admits up to 1024 additional
|
#### Use An Endpoint-Only Profile
|
||||||
calls waiting behind them:
|
|
||||||
|
Put the endpoint directly on an in-memory profile when only that profile needs
|
||||||
|
it and shared backend identity or capacity policy is unnecessary:
|
||||||
|
|
||||||
```go
|
```go
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{
|
engine, err := promptkit.NewEngine(promptkit.Config{
|
||||||
PromptDir: "prompts",
|
PromptDir: "prompts",
|
||||||
},
|
},
|
||||||
promptkit.WithBackend(promptkit.Backend{
|
promptkit.WithProfiles(promptkit.Profile{
|
||||||
ID: "local",
|
ID: "local-summary",
|
||||||
Endpoint: "http://localhost:8000/v1",
|
Endpoint: "http://localhost:8000/v1",
|
||||||
APIKeyEnv: "LOCAL_LLM_API_KEY",
|
Model: "example-model",
|
||||||
ConcurrencyLimit: 2,
|
|
||||||
}),
|
}),
|
||||||
|
)
|
||||||
|
```
|
||||||
|
|
||||||
|
Endpoint-only profiles have an empty backend ID and remain unrestricted by
|
||||||
|
backend capacity policy.
|
||||||
|
|
||||||
|
#### Use The Conventional Local Backend
|
||||||
|
|
||||||
|
Use `LocalBackend` when profiles should share the conventional `local`
|
||||||
|
identity, endpoint, and concurrency limit:
|
||||||
|
|
||||||
|
```go
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{
|
||||||
|
PromptDir: "prompts",
|
||||||
|
},
|
||||||
|
promptkit.WithBackend(
|
||||||
|
promptkit.LocalBackend("http://localhost:8000/v1", 2),
|
||||||
|
),
|
||||||
promptkit.WithProfiles(promptkit.Profile{
|
promptkit.WithProfiles(promptkit.Profile{
|
||||||
ID: "local-summary",
|
ID: "local-summary",
|
||||||
BackendID: "local",
|
BackendID: promptkit.BackendLocal,
|
||||||
Model: "example-model",
|
Model: "example-model",
|
||||||
}),
|
}),
|
||||||
)
|
)
|
||||||
```
|
```
|
||||||
|
|
||||||
|
The helper is explicit: it does not pre-register a backend or read environment
|
||||||
|
variables. Supplying a positive limit leaves queue capacity omitted, so normal
|
||||||
|
backend registration selects the existing default waiting capacity of 1024.
|
||||||
|
The returned value still enters the engine through `WithBackend`.
|
||||||
|
|
||||||
|
#### Configure A Complete Backend
|
||||||
|
|
||||||
|
Use a keyed `Backend` value for authentication, extra request parameters, an
|
||||||
|
explicit queue capacity, a custom ID, or multiple local endpoints:
|
||||||
|
|
||||||
|
```go
|
||||||
|
noWaiting := 0
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{
|
||||||
|
PromptDir: "prompts",
|
||||||
|
},
|
||||||
|
promptkit.WithBackend(promptkit.Backend{
|
||||||
|
ID: "local-gpu",
|
||||||
|
Endpoint: "http://gpu-host:8000/v1",
|
||||||
|
APIKeyEnv: "LOCAL_GPU_API_KEY",
|
||||||
|
ExtraParams: map[string]any{"provider_option": "enabled"},
|
||||||
|
ConcurrencyLimit: 2,
|
||||||
|
QueueCapacity: &noWaiting,
|
||||||
|
}),
|
||||||
|
promptkit.WithProfiles(promptkit.Profile{
|
||||||
|
ID: "gpu-summary",
|
||||||
|
BackendID: "local-gpu",
|
||||||
|
Model: "example-model",
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
```
|
||||||
|
|
||||||
|
Use distinct custom IDs when registering multiple local endpoints.
|
||||||
Registrations belong to one engine and custom IDs cannot replace built-ins.
|
Registrations belong to one engine and custom IDs cannot replace built-ins.
|
||||||
The [`Backend` and `WithBackend` GoDoc](../../backends.go) defines validation,
|
The [`Backend`, `LocalBackend`, and `WithBackend` GoDoc](../../backends.go)
|
||||||
copying, uniqueness, exact concurrency-field semantics, and request-default
|
defines exact construction, validation, copying, uniqueness, concurrency, and
|
||||||
behavior.
|
request-default behavior.
|
||||||
|
|
||||||
Both file-backed and in-memory profiles select a registration through
|
Both file-backed and in-memory profiles select a registration through
|
||||||
`backend` or `Profile.BackendID`. Profile and request endpoint overrides retain
|
`backend` or `Profile.BackendID`. Profile and request endpoint overrides retain
|
||||||
@@ -173,8 +344,8 @@ zero:
|
|||||||
```go
|
```go
|
||||||
noWaiting := 0
|
noWaiting := 0
|
||||||
backend := promptkit.Backend{
|
backend := promptkit.Backend{
|
||||||
ID: "local",
|
ID: "local-gpu",
|
||||||
Endpoint: "http://localhost:8000/v1",
|
Endpoint: "http://gpu-host:8000/v1",
|
||||||
ConcurrencyLimit: 2,
|
ConcurrencyLimit: 2,
|
||||||
QueueCapacity: &noWaiting,
|
QueueCapacity: &noWaiting,
|
||||||
}
|
}
|
||||||
@@ -236,6 +407,11 @@ When a limited backend has admitted all active and waiting calls, handle
|
|||||||
```go
|
```go
|
||||||
result, err := engine.Run(ctx, request)
|
result, err := engine.Run(ctx, request)
|
||||||
if errors.Is(err, promptkit.ErrCapacityExceeded) {
|
if errors.Is(err, promptkit.ErrCapacityExceeded) {
|
||||||
|
var capacityErr *promptkit.CapacityError
|
||||||
|
if errors.As(err, &capacityErr) {
|
||||||
|
// Record capacityErr.BackendID using application-owned diagnostics.
|
||||||
|
}
|
||||||
|
|
||||||
// Apply application policy: shed work, report overload, or retry later.
|
// Apply application policy: shed work, report overload, or retry later.
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
@@ -243,8 +419,9 @@ if errors.Is(err, promptkit.ErrCapacityExceeded) {
|
|||||||
A rejected call returns no partial result and does not invoke the model
|
A rejected call returns no partial result and does not invoke the model
|
||||||
client. Promptkit does not prescribe retries or map this error to an HTTP
|
client. Promptkit does not prescribe retries or map this error to an HTTP
|
||||||
status; those choices remain with the consuming application. The
|
status; those choices remain with the consuming application. The
|
||||||
[`Engine.Run` and error GoDoc](../../engine.go) owns exact error and
|
[`CapacityError` GoDoc](../../capacity_error.go) owns the exact typed-error
|
||||||
cancellation identities.
|
contract, while the [`Engine.Run` and error GoDoc](../../engine.go) owns broad
|
||||||
|
error and cancellation identities.
|
||||||
|
|
||||||
## Application Boundary
|
## Application Boundary
|
||||||
|
|
||||||
|
|||||||
@@ -44,98 +44,3 @@ Start with:
|
|||||||
For cross-cutting changes, follow every applicable row. Do not create
|
For cross-cutting changes, follow every applicable row. Do not create
|
||||||
placeholder documents for packages, APIs, or integrations that do not yet
|
placeholder documents for packages, APIs, or integrations that do not yet
|
||||||
exist.
|
exist.
|
||||||
|
|
||||||
## Maintainer-Run Validation
|
|
||||||
|
|
||||||
Promptkit does not currently use hosted CI. Maintainers are responsible for
|
|
||||||
running the documented checks before accepting changes. Run the default Go
|
|
||||||
validation from the Promptkit repository root:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
go test ./...
|
|
||||||
go test -race ./...
|
|
||||||
go vet ./...
|
|
||||||
go build ./...
|
|
||||||
go run ./examples/go-library/prepare
|
|
||||||
```
|
|
||||||
|
|
||||||
Check formatting across every tracked Go file:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
gofmt -l $(git ls-files '*.go')
|
|
||||||
```
|
|
||||||
|
|
||||||
The formatting command must produce no paths. Follow every added or changed
|
|
||||||
Markdown link and confirm its target exists. Finally, check whitespace:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
git diff --check
|
|
||||||
```
|
|
||||||
|
|
||||||
Documentation-only work does not require unrelated new tests, but it still
|
|
||||||
requires link validation and `git diff --check`. Run the Go validation whenever
|
|
||||||
documentation changes commands, examples, generated output, or another
|
|
||||||
behavior checked by the module.
|
|
||||||
|
|
||||||
## Focused Validation
|
|
||||||
|
|
||||||
Use focused checks while iterating, then run the complete validation sequence
|
|
||||||
before accepting the change. The root package supports:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
go test .
|
|
||||||
go vet .
|
|
||||||
go build .
|
|
||||||
```
|
|
||||||
|
|
||||||
Filter tests by name without assuming a fixed internal package layout:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
go test ./... -run 'TestName'
|
|
||||||
```
|
|
||||||
|
|
||||||
Replace `TestName` with a useful regular expression. Target only paths that
|
|
||||||
exist, and consult the internal component overview for their owning
|
|
||||||
documentation. A filtered or package-specific run does not replace the
|
|
||||||
complete repository validation.
|
|
||||||
|
|
||||||
## Coordinated Work With Scriptorium
|
|
||||||
|
|
||||||
Promptkit and Scriptorium must remain independently valid. For temporary local
|
|
||||||
integration, use either a Go workspace outside both repositories or an
|
|
||||||
uncommitted replacement in the consuming module.
|
|
||||||
|
|
||||||
If the repositories are sibling directories, run the workspace commands from
|
|
||||||
their parent directory:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
go work init ./promptkit ./scriptorium
|
|
||||||
go work sync
|
|
||||||
```
|
|
||||||
|
|
||||||
Use the workspace only for coordinated local checks. From the same parent
|
|
||||||
directory, remove it when finished:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
rm -f go.work go.work.sum
|
|
||||||
```
|
|
||||||
|
|
||||||
Alternatively, from the Scriptorium repository root, temporarily point its
|
|
||||||
Promptkit dependency at the sibling checkout:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
go mod edit -replace gitea.maximumdirect.net/eric/promptkit=../promptkit
|
|
||||||
```
|
|
||||||
|
|
||||||
After coordinated checks, remove the replacement and reconcile module
|
|
||||||
metadata:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
go mod edit -dropreplace gitea.maximumdirect.net/eric/promptkit
|
|
||||||
go mod tidy
|
|
||||||
```
|
|
||||||
|
|
||||||
Never commit `go.work`, `go.work.sum`, or a local filesystem `replace`
|
|
||||||
directive. Before committing in either repository, inspect its module files and
|
|
||||||
working tree independently. Published consumer versions must depend on a tagged
|
|
||||||
Promptkit version, not a workspace, local replacement, or unpublished commit.
|
|
||||||
|
|||||||
@@ -58,6 +58,11 @@ When a request omits a version, the selected prompt ID must identify exactly
|
|||||||
one definition. When it supplies a version, the ID and version pair must be
|
one definition. When it supplies a version, the ID and version pair must be
|
||||||
unique.
|
unique.
|
||||||
|
|
||||||
|
Exact prompt inspection uses this same configured source, strict decoding,
|
||||||
|
referenced content-file resolution, and ID/version selection. It reports the
|
||||||
|
selected definition's declared metadata without changing the prompt format or
|
||||||
|
executing the definition.
|
||||||
|
|
||||||
### Inputs
|
### Inputs
|
||||||
|
|
||||||
Each `inputs` item has these fields:
|
Each `inputs` item has these fields:
|
||||||
@@ -152,7 +157,7 @@ extra_params:
|
|||||||
| Field | Required | Meaning |
|
| Field | Required | Meaning |
|
||||||
| --- | --- | --- |
|
| --- | --- | --- |
|
||||||
| `id` | yes | Non-empty profile identifier. IDs must be unique within one source. |
|
| `id` | yes | Non-empty profile identifier. IDs must be unique within one source. |
|
||||||
| `backend` | unless `endpoint` is present | Backend registry ID. It is trimmed and registry membership is checked when the profile is prepared. |
|
| `backend` | unless `endpoint` is present | Backend registry ID. It is trimmed and registry membership is checked when the profile is prepared or inspected. |
|
||||||
| `endpoint` | unless `backend` is present | Non-empty OpenAI-compatible base URL, including an API version path when required. When both connection fields are present, this overrides the backend endpoint without changing backend identity. |
|
| `endpoint` | unless `backend` is present | Non-empty OpenAI-compatible base URL, including an API version path when required. When both connection fields are present, this overrides the backend endpoint without changing backend identity. |
|
||||||
| `model` | yes | Non-empty provider model name. |
|
| `model` | yes | Non-empty provider model name. |
|
||||||
| `temperature` | no | Number from 0 through 2. |
|
| `temperature` | no | Number from 0 through 2. |
|
||||||
@@ -183,25 +188,27 @@ they also cannot collide with the standard fields listed in the
|
|||||||
|
|
||||||
Execution settings resolve in this order:
|
Execution settings resolve in this order:
|
||||||
|
|
||||||
1. framework defaults;
|
1. the framework timeout baseline;
|
||||||
2. the selected backend, when the profile names one;
|
2. the selected backend, when the profile names one;
|
||||||
3. the selected profile; and
|
3. the selected profile; and
|
||||||
4. request `ExecutionTargetOverride` values.
|
4. request `ExecutionTargetOverride` values.
|
||||||
|
|
||||||
The framework defaults are:
|
The framework baseline is:
|
||||||
|
|
||||||
| Setting | Default |
|
| Setting | Default |
|
||||||
| --- | --- |
|
| --- | --- |
|
||||||
| `temperature` | `0` |
|
| `temperature` | Unspecified and omitted from compatible provider requests unless a profile or runtime override selects it. |
|
||||||
| `max_tokens` | `0` |
|
| `max_tokens` | Unspecified and omitted from compatible provider requests unless a profile or runtime override selects it. |
|
||||||
| `top_p` | `1` |
|
| `top_p` | Unspecified and omitted from compatible provider requests unless a profile or runtime override selects it. |
|
||||||
| `timeout_seconds` | `600` |
|
| `timeout_seconds` | `600` |
|
||||||
|
|
||||||
Numeric zero in a file or in-memory profile means that the profile does not
|
Numeric zero in a file or in-memory profile does not select a numeric value.
|
||||||
replace the framework default. Numeric request overrides use pointers, so an
|
For `temperature`, `max_tokens`, and `top_p`, it leaves the provider control
|
||||||
explicit zero is preserved. In particular, an explicit request
|
unspecified. For `timeout_seconds`, it retains the framework deadline. Numeric
|
||||||
`timeout_seconds` of zero disables the per-generation deadline while leaving
|
request overrides use pointers, so an explicit zero is retained and sent to
|
||||||
the caller context and transport timeout intact.
|
compatible providers. In particular, an explicit request `timeout_seconds` of
|
||||||
|
zero disables the per-generation deadline while leaving the caller context and
|
||||||
|
transport timeout intact.
|
||||||
|
|
||||||
Non-empty profile strings replace backend defaults, and non-empty request
|
Non-empty profile strings replace backend defaults, and non-empty request
|
||||||
strings replace both. Request reasoning is the exception: a nil
|
strings replace both. Request reasoning is the exception: a nil
|
||||||
@@ -219,18 +226,25 @@ defines how the effective settings are serialized.
|
|||||||
### Source And Profile Precedence
|
### Source And Profile Precedence
|
||||||
|
|
||||||
An explicit request profile ID takes precedence over the prompt's
|
An explicit request profile ID takes precedence over the prompt's
|
||||||
`default_profile`. If neither is present, preparation fails.
|
`default_profile`. If neither is present, preparation fails. Exact profile
|
||||||
|
inspection instead takes one explicit profile ID and does not use a prompt
|
||||||
|
default.
|
||||||
|
|
||||||
Profile sources resolve matching IDs in this order:
|
Profile sources resolve matching IDs in this order:
|
||||||
|
|
||||||
1. in-memory profiles supplied with `WithProfiles`;
|
1. in-memory profiles supplied with `WithProfiles`;
|
||||||
2. a profile file, `fs.FS`, or configured profile directory; and
|
2. the ordinary configured source selected by a profile file, `fs.FS`, or
|
||||||
3. embedded built-in profiles.
|
configured profile directory;
|
||||||
|
3. application fallback profiles supplied with `WithFallbackProfileFS`; and
|
||||||
|
4. embedded built-in profiles.
|
||||||
|
|
||||||
A higher-precedence source falls back only when the profile is absent. An
|
A profile source supplies a complete definition; definitions and their fields
|
||||||
invalid matching profile is an error and does not fall back. In-memory
|
are not merged across sources. A higher-precedence source falls back only when
|
||||||
`Profile` values follow the same ranges as YAML profiles. They use
|
the requested profile ID is absent. An invalid matching profile is an error and
|
||||||
`APIKeyRequired` for request-scoped credentials instead of `api_key_env`.
|
does not fall back. In-memory `Profile` values follow the same ranges as YAML
|
||||||
|
profiles. They use `APIKeyRequired` for request-scoped credentials instead of
|
||||||
|
`api_key_env`. Preparation and exact profile inspection use this same source
|
||||||
|
precedence.
|
||||||
|
|
||||||
## Built-In Profile Catalog
|
## Built-In Profile Catalog
|
||||||
|
|
||||||
@@ -238,8 +252,8 @@ Every built-in selects the `openrouter` backend. The engine's built-in backend
|
|||||||
registry supplies `https://openrouter.ai/api/v1` and the environment-variable
|
registry supplies `https://openrouter.ai/api/v1` and the environment-variable
|
||||||
name `OPENROUTER_API_KEY`, so individual profiles contain only model and
|
name `OPENROUTER_API_KEY`, so individual profiles contain only model and
|
||||||
generation settings. Built-in profile files do not repeat those connection
|
generation settings. Built-in profile files do not repeat those connection
|
||||||
values. A custom or in-memory profile with the same profile ID takes
|
values. A configured, application fallback, or in-memory profile with the same
|
||||||
precedence.
|
profile ID takes precedence.
|
||||||
|
|
||||||
| Provider | ID | Model |
|
| Provider | ID | Model |
|
||||||
| --- | --- | --- |
|
| --- | --- | --- |
|
||||||
|
|||||||
@@ -53,8 +53,9 @@ never also sent as a session header.
|
|||||||
|
|
||||||
The client conditionally includes:
|
The client conditionally includes:
|
||||||
|
|
||||||
- `temperature`, `max_tokens`, and `top_p` when non-zero or explicitly
|
- `temperature`, `max_tokens`, and `top_p` only when selected by a profile or
|
||||||
present;
|
runtime override, including an explicit runtime zero; they are absent when
|
||||||
|
unspecified;
|
||||||
- non-empty `service_tier` and effective `reasoning_effort`; an explicitly
|
- non-empty `service_tier` and effective `reasoning_effort`; an explicitly
|
||||||
disabled reasoning setting is empty and therefore omitted; and
|
disabled reasoning setting is empty and therefore omitted; and
|
||||||
- `response_format` for JSON Schema structured output, including its name,
|
- `response_format` for JSON Schema structured output, including its name,
|
||||||
|
|||||||
@@ -29,21 +29,28 @@ One pool owns immutable active and total limits plus mutex-protected admission
|
|||||||
count, active count, and ordered waiter list. Pool state exists only for the
|
count, active count, and ordered waiter list. Pool state exists only for the
|
||||||
lifetime of its engine.
|
lifetime of its engine.
|
||||||
|
|
||||||
## Bounded Run Admission
|
## Bounded Execution Admission
|
||||||
|
|
||||||
The runner asks the manager to admit a run after resolving the prompt, profile,
|
For ordinary `Run`, the runner asks the manager to admit after resolving the
|
||||||
selected backend, effective execution target, credentials, and output contract,
|
prompt, profile, selected backend, effective execution target, credentials, and
|
||||||
but before schema loading, artifact loading, or rendering. Admission is
|
output contract, but before schema loading, artifact loading, or rendering.
|
||||||
immediate: a limited pool either reserves a slot or returns the internal
|
`PrepareExecution` performs no admission. `RunPrepared` claims its handle,
|
||||||
`ErrCapacityExceeded` identity. The root facade maps that identity to the
|
rechecks credential availability, and then asks the manager to admit the
|
||||||
public error without treating it as an invalid request or generation failure.
|
frozen backend before generation.
|
||||||
|
|
||||||
|
Admission is immediate: a limited pool either reserves a slot or returns only
|
||||||
|
the internal `ErrCapacityExceeded` identity. The runner attaches the selected
|
||||||
|
backend identity at its use-case boundary, and the root facade translates that
|
||||||
|
typed value without treating it as an invalid request or generation failure.
|
||||||
|
|
||||||
The total admitted bound is the active-generation limit plus its configured
|
The total admitted bound is the active-generation limit plus its configured
|
||||||
waiting capacity. The returned release function is idempotent. The runner
|
waiting capacity. The returned release function is idempotent. The runner
|
||||||
defers it as soon as admission succeeds and holds the lease across remaining
|
defers it as soon as admission succeeds. An ordinary run holds the lease across
|
||||||
preparation, initial generation, validation, every repair attempt, and all
|
remaining preparation, initial generation, validation, every repair attempt,
|
||||||
failure or cancellation exits. A repair is part of its original admission and
|
and all failure or cancellation exits. Prepared execution holds the normal
|
||||||
does not reserve another bounded slot.
|
lease across generation, validation, every internal repair attempt, and all
|
||||||
|
execution exits. A repair is part of its original admission and does not
|
||||||
|
reserve another bounded slot.
|
||||||
|
|
||||||
## FIFO Generation Permits
|
## FIFO Generation Permits
|
||||||
|
|
||||||
@@ -90,11 +97,17 @@ unlimited admission. The
|
|||||||
FIFO transfer, canceled-waiter removal, grant/cancel races, independent pools,
|
FIFO transfer, canceled-waiter removal, grant/cancel races, independent pools,
|
||||||
unlimited calls, passthrough behavior, and panic release.
|
unlimited calls, passthrough behavior, and panic release.
|
||||||
|
|
||||||
The [runner tests](../../internal/usecase/runner_test.go) own early admission,
|
The [runner tests](../../internal/usecase/runner_test.go) own ordinary early
|
||||||
lease lifetime, failure release, and shared initial/repair scheduling. The
|
admission, lease lifetime, failure release, and shared initial/repair
|
||||||
|
scheduling. The
|
||||||
|
[prepared-execution use-case tests](../../internal/usecase/prepared_execution_test.go)
|
||||||
|
own deferred admission, credential ordering, and prepared-execution lease
|
||||||
|
release. The
|
||||||
[external package capacity tests](../../capacity_contract_test.go) own the
|
[external package capacity tests](../../capacity_contract_test.go) own the
|
||||||
assembled public-engine behavior for configured limits, capacity errors,
|
assembled public-engine behavior for configured limits, capacity errors,
|
||||||
endpoint identity, engine independence, and injected clients. The
|
endpoint identity, engine independence, and injected clients. The
|
||||||
|
[prepared-execution contract tests](../../prepared_execution_contract_test.go)
|
||||||
|
own the public prepared-capacity boundary. The
|
||||||
[root error-boundary tests](../../errors_internal_test.go) own preservation of
|
[root error-boundary tests](../../errors_internal_test.go) own preservation of
|
||||||
the public generation category and context identity when generation is
|
the public generation category and context identity when generation is
|
||||||
canceled.
|
canceled.
|
||||||
|
|||||||
@@ -43,6 +43,21 @@ without making the model client depend on registry configuration.
|
|||||||
The implementation has no retry loop, tool-call support, provider catalog,
|
The implementation has no retry loop, tool-call support, provider catalog,
|
||||||
inbound HTTP behavior, or durable session store.
|
inbound HTTP behavior, or durable session store.
|
||||||
|
|
||||||
|
## Prepared Generation
|
||||||
|
|
||||||
|
For [`RunPrepared`](../../engine.go), the runner supplies the model client with
|
||||||
|
the target, rendered messages, and structured-output constraint retained by
|
||||||
|
executable preparation. Execution does not reopen or rerender consumer
|
||||||
|
sources.
|
||||||
|
|
||||||
|
Before backend admission, the runner rechecks that the frozen credential
|
||||||
|
environment-variable name is available. The handle does not retain the
|
||||||
|
environment value; the model client resolves the value visible when generation
|
||||||
|
begins. A direct request key remains in private execution state only until the
|
||||||
|
claimed execution finishes or an unclaimed handle is discarded. Exact public
|
||||||
|
ownership and redaction semantics belong to the
|
||||||
|
[`PreparedExecution` GoDoc](../../prepared_execution.go).
|
||||||
|
|
||||||
## Failure Categories
|
## Failure Categories
|
||||||
|
|
||||||
The package preserves distinct error identities for invalid client
|
The package preserves distinct error identities for invalid client
|
||||||
|
|||||||
@@ -11,23 +11,23 @@ contributor workflow and validation.
|
|||||||
|
|
||||||
| Component | Implemented responsibility | References |
|
| Component | Implemented responsibility | References |
|
||||||
| --- | --- | --- |
|
| --- | --- | --- |
|
||||||
| Root `promptkit` package | Provides the supported engine facade, source, backend-registration, and injection options, public request and result values, profile construction, extension interfaces, value conversion, redacted formatting, public error mapping, and engine-local assembly. | [Package GoDoc](../../doc.go), [backend API](../../backends.go), [engine assembly](../../engine.go) |
|
| Root `promptkit` package | Provides the supported engine facade, source, backend-registration, and injection options, public request, result, prompt-inspection, and profile-inspection values, opaque prepared-execution handles, profile construction, extension interfaces, value conversion, redacted formatting, typed capacity errors, public error mapping, and engine-local profile-source assembly including application fallbacks. | [Package GoDoc](../../doc.go), [prepared execution](../../prepared_execution.go), [backend API](../../backends.go), [engine assembly](../../engine.go) |
|
||||||
| `examples/go-library/prepare` | Demonstrates an offline downstream consumer using a prompt file, in-memory profile, inline input, and `Prepare`. It is not a public library package. | [Example program](../../examples/go-library/prepare/main.go) |
|
| `examples/go-library/prepare` | Demonstrates an offline downstream consumer using a prompt file, in-memory profile, inline input, and `Prepare`. It is not a public library package. | [Example program](../../examples/go-library/prepare/main.go) |
|
||||||
| `examples/go-library/run` | Demonstrates an offline downstream consumer using a prompt file, in-memory profile, inline input, an injected deterministic model client, and `Run`. It is not a public library package. | [Example program](../../examples/go-library/run/main.go) |
|
| `examples/go-library/run` | Demonstrates an offline downstream consumer using a prompt file, in-memory profile, inline input, an injected deterministic model client, and `Run`. It is not a public library package. | [Example program](../../examples/go-library/run/main.go) |
|
||||||
| `internal/backend` | Constructs each engine's immutable registry from the built-in OpenRouter definition and consumer additions, validates and defensively copies definitions through the shared JSON-value package, and consumes the LLM-owned OpenAI-compatible reserved request-field rule. | [Backend registry](../../internal/backend/registry.go) |
|
| `internal/backend` | Constructs each engine's immutable registry from the built-in OpenRouter definition and consumer additions, validates and defensively copies definitions through the shared JSON-value package, and consumes the LLM-owned OpenAI-compatible reserved request-field rule. | [Backend registry](../../internal/backend/registry.go) |
|
||||||
| `internal/capacity` | Owns engine-local bounded run admission and FIFO model-generation permits for limited backend IDs, including cancellation-safe waiter removal and client wrapping. | [Internal capacity management](capacity.md) |
|
| `internal/capacity` | Owns engine-local bounded execution admission and FIFO model-generation permits for limited backend IDs, including cancellation-safe waiter removal and client wrapping. | [Internal capacity management](capacity.md) |
|
||||||
| `internal/domain` | Defines internal framework values for requests, artifacts, prompt definitions, profiles, execution targets, rendering, generation, and validation. | [Domain declarations](../../internal/domain/domain.go) |
|
| `internal/domain` | Defines internal framework values for requests, artifacts, prompt definitions, profiles, execution targets, rendering, generation, and validation. | [Domain declarations](../../internal/domain/domain.go) |
|
||||||
| `internal/defaults` | Defines application-neutral framework constants and constructs the default execution target. It contains no CLI, server, or inbound HTTP limits. | [Framework defaults](../../internal/defaults/defaults.go) |
|
| `internal/defaults` | Defines application-neutral framework constants and constructs the default execution target. It contains no CLI, server, or inbound HTTP limits. | [Framework defaults](../../internal/defaults/defaults.go) |
|
||||||
| `internal/filecatalog` | Provides deterministic YAML discovery and path helpers for operating-system filesystems and `fs.FS` sources. | [File catalog](../../internal/filecatalog/catalog.go) |
|
| `internal/filecatalog` | Provides deterministic YAML discovery and path helpers for operating-system filesystems and `fs.FS` sources. | [File catalog](../../internal/filecatalog/catalog.go) |
|
||||||
| `internal/jsonvalue` | Validates and deeply copies JSON-compatible extra-parameter trees while preserving supported concrete value types. | [JSON values](../../internal/jsonvalue/jsonvalue.go) |
|
| `internal/jsonvalue` | Validates and deeply copies JSON-compatible extra-parameter and prepared-schema trees while preserving supported concrete value types. | [JSON values](../../internal/jsonvalue/jsonvalue.go) |
|
||||||
| `internal/promptdef` | Loads strictly decoded, validated prompt definitions from filesystem and `fs.FS` sources, including version selection and contained file-backed message content. | [Framework formats](../formats.md), [prompt-definition repository](../../internal/promptdef/filesystem_repository.go) |
|
| `internal/promptdef` | Loads strictly decoded, validated prompt definitions from filesystem and `fs.FS` sources, including version selection and contained file-backed message content. | [Framework formats](../formats.md), [prompt-definition repository](../../internal/promptdef/filesystem_repository.go) |
|
||||||
| `internal/profile` | Loads strictly decoded, validated execution profiles, including backend selection, from filesystem and `fs.FS` sources and composes repositories with error-preserving fallback. | [Framework formats](../formats.md), [profile repositories](../../internal/profile/filesystem_repository.go) |
|
| `internal/profile` | Loads strictly decoded, validated execution profiles, including backend selection, from filesystem and `fs.FS` sources and composes repositories with error-preserving fallback. | [Framework formats](../formats.md), [profile repositories](../../internal/profile/filesystem_repository.go) |
|
||||||
| `internal/profile/builtin` | Embeds the built-in profile catalog, whose entries select OpenRouter, and combines it with an optional primary repository. | [Built-in catalog](../formats.md#built-in-profile-catalog), [repository](../../internal/profile/builtin/repository.go) |
|
| `internal/profile/builtin` | Embeds the built-in profile catalog, whose entries select OpenRouter. | [Built-in catalog](../formats.md#built-in-profile-catalog), [repository](../../internal/profile/builtin/repository.go) |
|
||||||
| `internal/prompt` | Renders prompt messages from Go templates with artifact, variable, session, and cache-control data. | [Go-template renderer](../../internal/prompt/go_renderer.go) |
|
| `internal/prompt` | Renders prompt messages from Go templates with artifact, variable, session, and cache-control data. | [Go-template renderer](../../internal/prompt/go_renderer.go) |
|
||||||
| `internal/artifact` | Resolves ordinary inline and unrestricted caller-selected file references into copied artifacts with metadata and hashes. | [Internal sources and validation](sources.md) |
|
| `internal/artifact` | Resolves ordinary inline and unrestricted caller-selected file references into copied artifacts with metadata and hashes. | [Internal sources and validation](sources.md) |
|
||||||
| `internal/validate` | Validates basic, JSON, and JSON Schema output using operating-system filesystem or `fs.FS` schema sources. | [Framework formats](../formats.md#schemas), [internal sources and validation](sources.md) |
|
| `internal/validate` | Validates basic, JSON, and JSON Schema output using operating-system filesystem or `fs.FS` schema sources and creates frozen validation plans for prepared execution. | [Framework formats](../formats.md#schemas), [internal sources and validation](sources.md) |
|
||||||
| `internal/llm` | Defines the internal generation boundary and implements outbound OpenAI-compatible chat requests from resolved execution targets, including response decoding, authentication, deadline handling, and ownership of the OpenAI-compatible reserved request-field policy. | [Internal model client](llm.md) |
|
| `internal/llm` | Defines the internal generation boundary and implements outbound OpenAI-compatible chat requests from resolved execution targets, including response decoding, authentication, deadline handling, and ownership of the OpenAI-compatible reserved request-field policy. | [Internal model client](llm.md) |
|
||||||
| `internal/usecase` | Resolves backend, profile, and request settings and coordinates preparation and execution across internal sources, rendering, artifact loading, generation, validation, and optional repair. | [Internal runner](runner.md) |
|
| `internal/usecase` | Resolves prompt definitions and hashes, profiles, backends, and targets for exact inspection and request settings for preparation, and coordinates ordinary execution and one-attempt prepared execution across internal sources, rendering, artifact loading, generation, validation, capacity, and optional repair. | [Internal runner](runner.md), [prepared-execution implementation](../../internal/usecase/prepared_execution.go) |
|
||||||
|
|
||||||
The root package assembles these internal components without exposing their
|
The root package assembles these internal components without exposing their
|
||||||
representations. Consumers depend only on the root facade.
|
representations. Consumers depend only on the root facade.
|
||||||
|
|||||||
@@ -28,6 +28,34 @@ constructor does not enable one.
|
|||||||
Each invocation carries its state in request, prepared-run, and result values.
|
Each invocation carries its state in request, prepared-run, and result values.
|
||||||
The runner has no durable run or session store.
|
The runner has no durable run or session store.
|
||||||
|
|
||||||
|
## Shared Prompt Selection
|
||||||
|
|
||||||
|
The runner uses one prompt-selection and hashing boundary for ordinary
|
||||||
|
preparation and exact prompt inspection. Preparation retains its early
|
||||||
|
request-ID check before direct-session normalization; both operations then use
|
||||||
|
the configured prompt repository to select one definition, load referenced
|
||||||
|
message content, and calculate the same prompt hash.
|
||||||
|
|
||||||
|
Inspection stops after that structural lookup. It does not parse templates or
|
||||||
|
touch profile, artifact, schema, renderer, validator, admission, or model
|
||||||
|
collaborators. The root [`Engine.InspectPrompt`](../../engine.go) GoDoc owns
|
||||||
|
the public operation's exact contract.
|
||||||
|
|
||||||
|
## Shared Profile Selection
|
||||||
|
|
||||||
|
The runner uses one profile-selection and target-resolution boundary for
|
||||||
|
ordinary preparation and exact profile inspection. Preparation first selects a
|
||||||
|
request profile or a prompt default; inspection begins with its required
|
||||||
|
explicit profile ID. Both then apply the ordinary source precedence, resolve a
|
||||||
|
named backend, and construct the effective target from framework, backend, and
|
||||||
|
profile values.
|
||||||
|
|
||||||
|
Inspection stops after the resulting endpoint and model are structurally
|
||||||
|
validated. It does not check credential availability or perform prompt,
|
||||||
|
artifact, schema, rendering, admission, or model-client work. The root
|
||||||
|
[`Engine.InspectProfile`](../../engine.go) GoDoc owns the public operation's
|
||||||
|
exact contract.
|
||||||
|
|
||||||
## Shared Preparation Pipeline
|
## Shared Preparation Pipeline
|
||||||
|
|
||||||
`Prepare` and `Run` share one private preparation pipeline split at the point
|
`Prepare` and `Run` share one private preparation pipeline split at the point
|
||||||
@@ -124,15 +152,17 @@ validation failures. Wrapping preserves the package identities mapped by the
|
|||||||
public facade and retains collaborator identities where they are part of the
|
public facade and retains collaborator identities where they are part of the
|
||||||
internal contract.
|
internal contract.
|
||||||
|
|
||||||
Admission capacity exhaustion retains the internal capacity identity and adds
|
Admission capacity exhaustion retains the internal capacity identity. At the
|
||||||
the selected backend ID as context. It is not recategorized as an invalid
|
use-case boundary, the runner attaches the selected backend ID in an internal
|
||||||
request or generation failure, and no partial result is returned. A context
|
typed error, and the root facade copies that value into the public
|
||||||
already done at admission retains its context identity directly. Cancellation
|
[`CapacityError`](../../capacity_error.go) without parsing diagnostic text. It
|
||||||
while waiting for an active generation permit prevents client invocation when
|
is not recategorized as an invalid request or generation failure, and no
|
||||||
it wins the grant race; the model-client boundary then preserves the context
|
partial result is returned. A context already done at admission retains its
|
||||||
error through the generation-failure category. Deferred release restores the
|
context identity directly. Cancellation while waiting for an active generation
|
||||||
admission lease on preparation, generation, validation, repair, and
|
permit prevents client invocation when it wins the grant race; the model-client
|
||||||
cancellation failures.
|
boundary then preserves the context error through the generation-failure
|
||||||
|
category. Deferred release restores the admission lease on preparation,
|
||||||
|
generation, validation, repair, and cancellation failures.
|
||||||
|
|
||||||
Other context cancellation propagates through the invoked collaborator and is
|
Other context cancellation propagates through the invoked collaborator and is
|
||||||
classified by the owning operation.
|
classified by the owning operation.
|
||||||
|
|||||||
@@ -16,6 +16,11 @@ validation modes, built-in catalog, and source precedence.
|
|||||||
definitions, selects an ID and optional version, and resolves file-backed
|
definitions, selects an ID and optional version, and resolves file-backed
|
||||||
message content within the selected operating-system or `fs.FS` source.
|
message content within the selected operating-system or `fs.FS` source.
|
||||||
|
|
||||||
|
Exact prompt inspection performs one point-in-time lookup through that same
|
||||||
|
repository and validates referenced message content before returning declared
|
||||||
|
metadata. It does not parse templates or read profile, input, or schema
|
||||||
|
sources, and it does not retain the definition for a later execution.
|
||||||
|
|
||||||
Its package tests own prompt selection, strict decoding, definition validation,
|
Its package tests own prompt selection, strict decoding, definition validation,
|
||||||
duplicate detection, and source containment:
|
duplicate detection, and source containment:
|
||||||
[prompt-definition repository tests](../../internal/promptdef/repository_test.go).
|
[prompt-definition repository tests](../../internal/promptdef/repository_test.go).
|
||||||
@@ -23,21 +28,30 @@ duplicate detection, and source containment:
|
|||||||
## Profiles And Built-Ins
|
## Profiles And Built-Ins
|
||||||
|
|
||||||
`internal/profile` loads and validates execution profiles from an
|
`internal/profile` loads and validates execution profiles from an
|
||||||
operating-system filesystem or an `fs.FS`. It supports a primary repository
|
operating-system filesystem or an `fs.FS`. Its overlay repository consults the
|
||||||
with fallback only when the primary reports that a profile is absent. Strict
|
next repository only when the higher-precedence repository reports that a
|
||||||
YAML decoding recognizes the optional `backend` field, trims its value, and
|
profile is absent. Strict YAML decoding recognizes the optional `backend`
|
||||||
requires a model plus at least one non-blank backend or endpoint. Loading does
|
field, trims its value, and requires a model plus at least one non-blank
|
||||||
not check registry membership because the available registry belongs to the
|
backend or endpoint. Loading does not check registry membership because the
|
||||||
assembled engine; the runner checks membership during preparation.
|
available registry belongs to the assembled engine; the runner checks
|
||||||
|
membership during preparation and exact profile inspection.
|
||||||
|
|
||||||
`internal/profile/builtin` embeds the maintained built-in profile catalog and
|
The root engine assembles profile repositories in precedence order: in-memory
|
||||||
can place a caller-selected repository ahead of that catalog. Every embedded
|
profiles, one ordinary configured source, an application fallback source, then
|
||||||
profile selects `openrouter` and inherits its endpoint and credential
|
the embedded built-in catalog. An explicit file or `fs.FS` profile source
|
||||||
environment-variable name from the built-in backend registry rather than
|
replaces `Config.ProfileDir` within the ordinary configured-source category.
|
||||||
repeating those values. Profile behavior is owned by the
|
|
||||||
[profile repository tests](../../internal/profile/repository_test.go), while
|
Exact profile inspection performs one point-in-time lookup through those
|
||||||
catalog completeness, the backend-selection invariant, duplicate IDs, and
|
profile sources and checks the resolved target without reading prompt, input,
|
||||||
overlay behavior are owned by the
|
or schema sources. It does not retain that lookup for a later execution.
|
||||||
|
|
||||||
|
`internal/profile/builtin` embeds the maintained built-in profile catalog.
|
||||||
|
Every embedded profile selects `openrouter` and inherits its endpoint and
|
||||||
|
credential environment-variable name from the built-in backend registry rather
|
||||||
|
than repeating those values. Profile loading and overlay behavior are owned by
|
||||||
|
the [profile repository tests](../../internal/profile/repository_test.go),
|
||||||
|
while catalog completeness, the backend-selection invariant, and duplicate IDs
|
||||||
|
are owned by the
|
||||||
[built-in repository tests](../../internal/profile/builtin/repository_test.go).
|
[built-in repository tests](../../internal/profile/builtin/repository_test.go).
|
||||||
|
|
||||||
## Ordinary Artifacts
|
## Ordinary Artifacts
|
||||||
@@ -68,6 +82,22 @@ filesystem or an `fs.FS`. Invalid generated content is returned as a validation
|
|||||||
result; inability to load, register, or compile a schema is an operational
|
result; inability to load, register, or compile a schema is an operational
|
||||||
error.
|
error.
|
||||||
|
|
||||||
|
For executable preparation, the built-in validators create a frozen validation
|
||||||
|
plan. None, basic, and JSON modes retain the effective output contract without
|
||||||
|
source access. JSON Schema mode loads the root document, resolves and compiles
|
||||||
|
every transitive reference during preparation, and retains the compiled
|
||||||
|
validator. The provider-facing structured-output metadata uses that same
|
||||||
|
captured root document.
|
||||||
|
|
||||||
|
`PrepareExecution` also completes prompt and profile selection, artifact
|
||||||
|
loading and hashing, session and message rendering, and target resolution.
|
||||||
|
`RunPrepared` uses the retained source-derived state and validation plan; it
|
||||||
|
does not reopen prompt, profile, input, or schema sources and does not rerender
|
||||||
|
the request. By contrast, ordinary `Prepare` produces a preparation value only:
|
||||||
|
a later `Run` performs its own source resolution and preparation.
|
||||||
|
|
||||||
The [validator tests](../../internal/validate/standard_validator_test.go) own
|
The [validator tests](../../internal/validate/standard_validator_test.go) own
|
||||||
basic, JSON, JSON Schema, source resolution, schema loading, compilation, and
|
basic, JSON, JSON Schema, source resolution, schema loading, compilation,
|
||||||
content-failure behavior.
|
frozen-reference behavior, and content-failure behavior. Prepared execution
|
||||||
|
orchestration is owned by the
|
||||||
|
[use-case tests](../../internal/usecase/prepared_execution_test.go).
|
||||||
|
|||||||
@@ -90,7 +90,7 @@ engine, err := promptkit.NewEngine(
|
|||||||
```
|
```
|
||||||
|
|
||||||
See the
|
See the
|
||||||
[custom-backend consumer guide](../consumers/pkg-promptkit.md#register-a-custom-backend)
|
[local-endpoint consumer guide](../consumers/pkg-promptkit.md#configure-a-local-openai-compatible-endpoint)
|
||||||
for task-oriented usage. The
|
for task-oriented usage. The
|
||||||
[`Backend` and `WithBackend` GoDoc](../../backends.go) owns exact registration,
|
[`Backend` and `WithBackend` GoDoc](../../backends.go) owns exact registration,
|
||||||
validation, copying, defaulting, and uniqueness semantics. The
|
validation, copying, defaulting, and uniqueness semantics. The
|
||||||
|
|||||||
74
docs/releases/v0.3.0.md
Normal file
74
docs/releases/v0.3.0.md
Normal file
@@ -0,0 +1,74 @@
|
|||||||
|
# Promptkit v0.3.0
|
||||||
|
|
||||||
|
This supplemental changelog summarizes the consumer-facing changes from
|
||||||
|
`v0.2.0` to `v0.3.0`. The annotated `v0.3.0` tag is the authoritative release
|
||||||
|
record. Exact current contracts belong to the linked GoDoc and durable
|
||||||
|
documentation.
|
||||||
|
|
||||||
|
## Summary
|
||||||
|
|
||||||
|
`v0.3.0` adds a concise way to register the common local OpenAI-compatible
|
||||||
|
backend configuration:
|
||||||
|
|
||||||
|
- `BackendLocal` provides the conventional, non-reserved backend ID `"local"`;
|
||||||
|
and
|
||||||
|
- `LocalBackend` constructs an ordinary `Backend` from an endpoint and
|
||||||
|
concurrency limit.
|
||||||
|
|
||||||
|
The helper is explicit and additive. It does not pre-register a backend, read
|
||||||
|
environment variables, select a model, or replace the complete `Backend`
|
||||||
|
configuration interface.
|
||||||
|
|
||||||
|
## Compatibility
|
||||||
|
|
||||||
|
Existing `v0.2.0` consumers require no migration. Endpoint-only profiles,
|
||||||
|
complete custom `Backend` values, the built-in OpenRouter backend, and existing
|
||||||
|
registrations using the literal ID `"local"` continue to work unchanged.
|
||||||
|
|
||||||
|
## Upgrade
|
||||||
|
|
||||||
|
Update the module dependency with:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go get gitea.maximumdirect.net/eric/promptkit@v0.3.0
|
||||||
|
go mod tidy
|
||||||
|
```
|
||||||
|
|
||||||
|
Run the consuming project's ordinary tests and race-enabled tests after the
|
||||||
|
upgrade.
|
||||||
|
|
||||||
|
## Configure A Local Backend
|
||||||
|
|
||||||
|
Register the convenience value through the existing `WithBackend` option and
|
||||||
|
select it from one or more profiles:
|
||||||
|
|
||||||
|
```go
|
||||||
|
engine, err := promptkit.NewEngine(
|
||||||
|
promptkit.Config{PromptDir: "prompts"},
|
||||||
|
promptkit.WithBackend(
|
||||||
|
promptkit.LocalBackend("http://localhost:8000/v1", 2),
|
||||||
|
),
|
||||||
|
promptkit.WithProfiles(promptkit.Profile{
|
||||||
|
ID: "local-summary",
|
||||||
|
BackendID: promptkit.BackendLocal,
|
||||||
|
Model: "example-model",
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
```
|
||||||
|
|
||||||
|
Use an endpoint-only profile when shared backend identity and capacity policy
|
||||||
|
are unnecessary. Continue to use a complete keyed `Backend` value for custom
|
||||||
|
IDs, authentication, extra request parameters, explicit queue capacity, or
|
||||||
|
multiple local endpoints.
|
||||||
|
|
||||||
|
See the
|
||||||
|
[local-endpoint consumer guide](../consumers/pkg-promptkit.md#configure-a-local-openai-compatible-endpoint)
|
||||||
|
for task-oriented configuration choices. The
|
||||||
|
[`BackendLocal`, `LocalBackend`, and `WithBackend` GoDoc](../../backends.go)
|
||||||
|
owns their exact construction, registration, validation, and concurrency
|
||||||
|
semantics.
|
||||||
|
|
||||||
|
## Consumer Action
|
||||||
|
|
||||||
|
None. Adopt the convenience constructor when it simplifies local endpoint
|
||||||
|
configuration.
|
||||||
189
docs/releases/v0.4.0.md
Normal file
189
docs/releases/v0.4.0.md
Normal file
@@ -0,0 +1,189 @@
|
|||||||
|
# Promptkit v0.4.0
|
||||||
|
|
||||||
|
This supplemental changelog and adoption guide summarizes the consumer-facing
|
||||||
|
changes from `v0.3.0` to `v0.4.0`. The annotated `v0.4.0` tag is the
|
||||||
|
authoritative release record. Exact current contracts belong to the linked
|
||||||
|
GoDoc and durable documentation.
|
||||||
|
|
||||||
|
## Summary
|
||||||
|
|
||||||
|
`v0.4.0` adds four complementary capabilities:
|
||||||
|
|
||||||
|
- opaque prepared-execution handles for preparing once, inspecting safe
|
||||||
|
details, and executing the same frozen snapshot;
|
||||||
|
- exact prompt-definition inspection without profile resolution or execution;
|
||||||
|
- exact profile inspection without selecting a prompt or checking credential
|
||||||
|
availability; and
|
||||||
|
- structured backend identity on engine admission-capacity rejection.
|
||||||
|
|
||||||
|
These APIs let consumers perform more precise preflight work and retain useful
|
||||||
|
operational context without reproducing Promptkit's internal resolution logic.
|
||||||
|
|
||||||
|
## Compatibility
|
||||||
|
|
||||||
|
The release is additive for `v0.3.0` consumers. Existing uses of `Prepare`,
|
||||||
|
`Run`, backend registration, endpoint-only profiles, local-backend helpers,
|
||||||
|
runtime overrides, public JSON values, and error sentinels continue to work
|
||||||
|
without migration.
|
||||||
|
|
||||||
|
Capacity rejection now returns a structured error while continuing to match
|
||||||
|
`ErrCapacityExceeded` through `errors.Is`. Error-string wording and direct
|
||||||
|
sentinel equality were not public contracts.
|
||||||
|
|
||||||
|
The new inspection values, capacity error, and prepared-execution handle do not
|
||||||
|
have stable JSON representations. `PreparedExecution.Details` returns the
|
||||||
|
existing stable `PreparedRun` value.
|
||||||
|
|
||||||
|
## Upgrade
|
||||||
|
|
||||||
|
Update the module dependency with:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go get gitea.maximumdirect.net/eric/promptkit@v0.4.0
|
||||||
|
go mod tidy
|
||||||
|
```
|
||||||
|
|
||||||
|
Run the consuming project's ordinary and race-enabled tests after upgrading.
|
||||||
|
No source migration is required.
|
||||||
|
|
||||||
|
## Prepare Once And Execute The Same Snapshot
|
||||||
|
|
||||||
|
Consumers that need to persist preparation details before generation can now
|
||||||
|
prepare an opaque, engine-bound execution:
|
||||||
|
|
||||||
|
```go
|
||||||
|
prepared, err := engine.PrepareExecution(ctx, request)
|
||||||
|
if err != nil {
|
||||||
|
// Handle preparation failure.
|
||||||
|
}
|
||||||
|
defer prepared.Discard()
|
||||||
|
|
||||||
|
details := prepared.Details()
|
||||||
|
// Persist a consumer-selected, appropriately protected preparation record.
|
||||||
|
|
||||||
|
result, err := engine.RunPrepared(ctx, prepared)
|
||||||
|
```
|
||||||
|
|
||||||
|
Preparation freezes the selected sources, rendered messages, effective
|
||||||
|
settings, input content, structured-output metadata, and validation resources
|
||||||
|
needed by execution. `Details` returns a fresh, caller-owned,
|
||||||
|
credential-redacted `PreparedRun`.
|
||||||
|
|
||||||
|
A handle belongs to its creating engine and permits one execution attempt.
|
||||||
|
`RunPrepared` consumes that attempt on success and on operational failure.
|
||||||
|
`Discard` is idempotent and releases an unclaimed handle's execution-only
|
||||||
|
state. Consumers should discard handles they will not execute, particularly
|
||||||
|
when a direct request API key may be retained privately until claim or
|
||||||
|
discard.
|
||||||
|
|
||||||
|
Prepared execution does not reserve backend admission during preparation.
|
||||||
|
Credential availability and backend admission are checked when execution
|
||||||
|
begins. The execution context is independent of the preparation context.
|
||||||
|
|
||||||
|
See the
|
||||||
|
[prepared-execution consumer guide](../consumers/pkg-promptkit.md#prepare-now-and-execute-the-same-snapshot-later),
|
||||||
|
the [`PreparedExecution` GoDoc](../../prepared_execution.go), and the
|
||||||
|
[`Engine.PrepareExecution` and `Engine.RunPrepared` GoDoc](../../engine.go)
|
||||||
|
for the exact lifecycle, ownership, cancellation, capacity, timing, and
|
||||||
|
failure contracts.
|
||||||
|
|
||||||
|
## Inspect A Prompt
|
||||||
|
|
||||||
|
`Engine.InspectPrompt` resolves one prompt ID and optional version through the
|
||||||
|
engine's configured prompt source:
|
||||||
|
|
||||||
|
```go
|
||||||
|
inspection, err := engine.InspectPrompt(ctx, "report.summary", "")
|
||||||
|
```
|
||||||
|
|
||||||
|
The result includes prompt identity, the opaque prompt hash, declared default
|
||||||
|
profile ID, declared input metadata, and normalized output contract. It
|
||||||
|
structurally loads the selected definition and referenced message content but
|
||||||
|
does not resolve a profile, load schemas or artifacts, render templates,
|
||||||
|
reserve capacity, or contact a model.
|
||||||
|
|
||||||
|
Use inspection for exact configuration checks and metadata discovery. Use
|
||||||
|
`PrepareExecution` rather than relying on a prior inspection when later
|
||||||
|
execution must freeze one exact source state, because filesystem-backed
|
||||||
|
inspection is only a point-in-time lookup.
|
||||||
|
|
||||||
|
See the
|
||||||
|
[prompt-inspection consumer guide](../consumers/pkg-promptkit.md#inspect-a-prompt-before-preparation)
|
||||||
|
and [`Engine.InspectPrompt` GoDoc](../../engine.go) for exact selection,
|
||||||
|
ownership, and error behavior.
|
||||||
|
|
||||||
|
## Inspect A Profile
|
||||||
|
|
||||||
|
`Engine.InspectProfile` resolves one explicit profile independently of a
|
||||||
|
prompt:
|
||||||
|
|
||||||
|
```go
|
||||||
|
inspection, err := engine.InspectProfile(ctx, "report-production")
|
||||||
|
```
|
||||||
|
|
||||||
|
The result includes the resolved effective execution target and whether a
|
||||||
|
later request must provide a direct credential. Environment-variable names may
|
||||||
|
be reported, but inspection does not read credential values or require the
|
||||||
|
named variable to be populated.
|
||||||
|
|
||||||
|
Inspection applies the engine's profile source precedence and resolves any
|
||||||
|
selected backend. It does not load a prompt, render content, reserve capacity,
|
||||||
|
or contact a model.
|
||||||
|
|
||||||
|
See the
|
||||||
|
[profile-inspection consumer guide](../consumers/pkg-promptkit.md#inspect-a-profile-before-prompt-work)
|
||||||
|
and [`Engine.InspectProfile` GoDoc](../../engine.go) for the exact resolution,
|
||||||
|
credential, ownership, and error contracts.
|
||||||
|
|
||||||
|
## Identify Capacity-Rejected Backends
|
||||||
|
|
||||||
|
Calls rejected at Promptkit's bounded engine admission boundary continue to
|
||||||
|
match `ErrCapacityExceeded`. Consumers can additionally obtain the selected
|
||||||
|
registered backend ID without parsing diagnostic text:
|
||||||
|
|
||||||
|
```go
|
||||||
|
result, err := engine.Run(ctx, request)
|
||||||
|
if errors.Is(err, promptkit.ErrCapacityExceeded) {
|
||||||
|
var capacityErr *promptkit.CapacityError
|
||||||
|
if errors.As(err, &capacityErr) {
|
||||||
|
// Record capacityErr.BackendID using application-owned diagnostics.
|
||||||
|
}
|
||||||
|
|
||||||
|
// Apply application-owned overload or retry policy.
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
The structured error applies to `Run` and `RunPrepared` admission rejection.
|
||||||
|
It does not represent provider throttling, quota exhaustion, cancellation
|
||||||
|
while waiting for generation capacity, or another model-client failure.
|
||||||
|
Promptkit does not prescribe retry timing or transport status mapping.
|
||||||
|
|
||||||
|
See the
|
||||||
|
[error-handling consumer guide](../consumers/pkg-promptkit.md#handle-errors),
|
||||||
|
the [`CapacityError` GoDoc](../../capacity_error.go), and the
|
||||||
|
[`ErrCapacityExceeded` GoDoc](../../engine.go) for the canonical contracts.
|
||||||
|
|
||||||
|
## Public API Additions
|
||||||
|
|
||||||
|
The release adds:
|
||||||
|
|
||||||
|
- `Engine.PrepareExecution`;
|
||||||
|
- `Engine.RunPrepared`;
|
||||||
|
- `PreparedExecution`, including `Details`, `Discard`, `String`, and
|
||||||
|
`GoString`;
|
||||||
|
- `Engine.InspectPrompt`;
|
||||||
|
- `PromptInspection`;
|
||||||
|
- `PromptInputDefinition`;
|
||||||
|
- `Engine.InspectProfile`;
|
||||||
|
- `ProfileInspection`; and
|
||||||
|
- `CapacityError`.
|
||||||
|
|
||||||
|
No public API was removed.
|
||||||
|
|
||||||
|
## Consumer Action
|
||||||
|
|
||||||
|
None. Existing `v0.3.0` workflows may upgrade without adopting the new APIs.
|
||||||
|
|
||||||
|
Consumers that adopt prepared execution should discard unused handles.
|
||||||
|
Consumers that need backend-specific capacity diagnostics may add an
|
||||||
|
`errors.As` check while retaining their existing `errors.Is` classification.
|
||||||
125
docs/releases/v0.5.0.md
Normal file
125
docs/releases/v0.5.0.md
Normal file
@@ -0,0 +1,125 @@
|
|||||||
|
# Promptkit v0.5.0
|
||||||
|
|
||||||
|
This supplemental changelog and migration guide summarizes the consumer-facing
|
||||||
|
changes from `v0.4.0` to `v0.5.0`. The annotated `v0.5.0` tag is the
|
||||||
|
authoritative release record. Exact current contracts belong to the linked
|
||||||
|
GoDoc and durable documentation.
|
||||||
|
|
||||||
|
## Summary
|
||||||
|
|
||||||
|
`v0.5.0` makes provider requests less prescriptive and adds an application
|
||||||
|
fallback layer for profile definitions:
|
||||||
|
|
||||||
|
- unset optional provider controls are omitted from OpenAI-compatible request
|
||||||
|
bodies instead of being populated with framework values; and
|
||||||
|
- `WithFallbackProfileFS` lets an application package profile defaults that
|
||||||
|
operators can override through the existing ordinary profile sources.
|
||||||
|
|
||||||
|
These changes let compatible providers apply their own model defaults while
|
||||||
|
giving applications stable embedded profile IDs without weakening operator
|
||||||
|
configuration precedence.
|
||||||
|
|
||||||
|
## Compatibility
|
||||||
|
|
||||||
|
The release adds one public function and removes no public declaration.
|
||||||
|
Existing source code should continue to compile.
|
||||||
|
|
||||||
|
There is one intentional behavior change: when no profile or runtime override
|
||||||
|
selects `top_p`, Promptkit no longer sends the former framework value of `1`.
|
||||||
|
It omits `top_p` and lets the provider choose its behavior. Unset
|
||||||
|
`temperature` and `max_tokens` are likewise omitted. Explicit nonzero profile
|
||||||
|
values and runtime values—including explicit runtime zero values—retain their
|
||||||
|
precedence and wire effect.
|
||||||
|
|
||||||
|
Consumers that relied on Promptkit always sending `top_p: 1` should add that
|
||||||
|
value to the relevant profile or runtime override before upgrading. Consumers
|
||||||
|
that did not rely on the implicit sampling value require no migration.
|
||||||
|
|
||||||
|
Application fallback profiles are opt-in. Engines that do not call
|
||||||
|
`WithFallbackProfileFS` retain the previous profile-source behavior.
|
||||||
|
|
||||||
|
## Upgrade
|
||||||
|
|
||||||
|
Update the module dependency with:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go get gitea.maximumdirect.net/eric/promptkit@v0.5.0
|
||||||
|
go mod tidy
|
||||||
|
```
|
||||||
|
|
||||||
|
Run the consuming project's ordinary and race-enabled tests after upgrading.
|
||||||
|
If request payloads or model behavior are asserted in fixtures, review them for
|
||||||
|
the optional-parameter omission described below.
|
||||||
|
|
||||||
|
## Omitted Optional Provider Controls
|
||||||
|
|
||||||
|
The built-in OpenAI-compatible client now includes `temperature`,
|
||||||
|
`max_tokens`, and `top_p` only when a profile or runtime override selects the
|
||||||
|
value. An explicit runtime zero remains present because runtime override
|
||||||
|
pointers distinguish zero from an unspecified value.
|
||||||
|
|
||||||
|
Promptkit's positive generation deadline remains a framework concern and is
|
||||||
|
not a provider request-body default. Required request fields, session IDs,
|
||||||
|
structured output, reasoning selection, credentials, and explicit extra
|
||||||
|
parameters retain their existing behavior.
|
||||||
|
|
||||||
|
See the [framework default and precedence reference](../formats.md#defaults-and-overrides),
|
||||||
|
the [`ExecutionTargetOverride` GoDoc](../../types.go), and the
|
||||||
|
[OpenAI-compatible request-body contract](../integrations/openai-compatible-chat.md#request-body)
|
||||||
|
for current details.
|
||||||
|
|
||||||
|
## Embedded Application Fallback Profiles
|
||||||
|
|
||||||
|
Applications can package ordinary profile YAML in an `fs.FS` and register it
|
||||||
|
as a fallback source:
|
||||||
|
|
||||||
|
```go
|
||||||
|
//go:embed profiles/*.yaml
|
||||||
|
var applicationProfiles embed.FS
|
||||||
|
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{
|
||||||
|
PromptDir: "prompts",
|
||||||
|
ProfileDir: operatorProfileDir,
|
||||||
|
},
|
||||||
|
promptkit.WithFallbackProfileFS(applicationProfiles, "profiles"),
|
||||||
|
)
|
||||||
|
```
|
||||||
|
|
||||||
|
Leave `operatorProfileDir` empty when no operator source is configured. A
|
||||||
|
configured ordinary source is authoritative: a matching definition overrides
|
||||||
|
the application fallback, while a read or validation failure remains an error
|
||||||
|
instead of silently reaching a lower layer.
|
||||||
|
|
||||||
|
Profile definitions resolve in this order:
|
||||||
|
|
||||||
|
1. in-memory profiles supplied with `WithProfiles`;
|
||||||
|
2. the ordinary configured source selected by `WithProfileFile`,
|
||||||
|
`WithProfileFS`, or `Config.ProfileDir`;
|
||||||
|
3. the application source supplied with `WithFallbackProfileFS`; and
|
||||||
|
4. Promptkit's embedded built-in profiles.
|
||||||
|
|
||||||
|
Only an absent profile ID falls through. Sources provide complete profiles and
|
||||||
|
do not merge fields. Loading remains lazy, and the new source uses the existing
|
||||||
|
strict profile YAML and credential rules.
|
||||||
|
|
||||||
|
See the
|
||||||
|
[embedded-default consumer guidance](../consumers/pkg-promptkit.md#supply-embedded-application-defaults),
|
||||||
|
the [`WithFallbackProfileFS` GoDoc](../../engine.go), and the
|
||||||
|
[profile source reference](../formats.md#source-and-profile-precedence) for
|
||||||
|
current details.
|
||||||
|
|
||||||
|
## Public API Changes
|
||||||
|
|
||||||
|
The release adds:
|
||||||
|
|
||||||
|
- `WithFallbackProfileFS`.
|
||||||
|
|
||||||
|
No public declaration was removed or changed.
|
||||||
|
|
||||||
|
## Consumer Action
|
||||||
|
|
||||||
|
- Review any workflow that depended on Promptkit's implicit `top_p: 1` and
|
||||||
|
configure the value explicitly when required.
|
||||||
|
- Optionally adopt `WithFallbackProfileFS` when an application should package
|
||||||
|
overridable profile defaults.
|
||||||
|
- Run consumer tests after updating the module dependency.
|
||||||
@@ -1,236 +0,0 @@
|
|||||||
# Backend-Specific Concurrency Management
|
|
||||||
|
|
||||||
**Status:** Complete.
|
|
||||||
|
|
||||||
## Purpose
|
|
||||||
|
|
||||||
This roadmap defines the scope and target end state for engine-local,
|
|
||||||
backend-specific concurrency management. It records the intended capability,
|
|
||||||
consumer value, and important policy choices.
|
|
||||||
|
|
||||||
This document is planning material, not a description of current behavior.
|
|
||||||
Current exported contracts remain owned by Go declarations and GoDoc, backend
|
|
||||||
registration guidance by the
|
|
||||||
[consumer guide](../consumers/pkg-promptkit.md#register-a-custom-backend), and
|
|
||||||
implemented orchestration by the
|
|
||||||
[internal runner document](../internal/runner.md).
|
|
||||||
|
|
||||||
## Motivation
|
|
||||||
|
|
||||||
Different model backends can sustain very different request loads. A local
|
|
||||||
network endpoint may need a small concurrency limit, while OpenRouter can
|
|
||||||
usually accept substantially more simultaneous work. Requiring every consumer
|
|
||||||
to build its own semaphores and queues would duplicate routing knowledge,
|
|
||||||
create inconsistent cancellation behavior, and make it easy for one caller to
|
|
||||||
bypass the intended backend limit.
|
|
||||||
|
|
||||||
Promptkit should own this coordination because it already resolves each run to
|
|
||||||
an engine-scoped backend identity and owns every model-generation call made by
|
|
||||||
the runner. Consumers should continue submitting ready-to-run requests through
|
|
||||||
the synchronous API, including concurrently from multiple goroutines, without
|
|
||||||
implementing their own backend scheduler.
|
|
||||||
|
|
||||||
The buffered queue is a safety boundary, not an ordinary throughput
|
|
||||||
restriction. Its primary purpose is to prevent a bug or unintended submission
|
|
||||||
loop from creating an unbounded in-memory backlog.
|
|
||||||
|
|
||||||
## Scope
|
|
||||||
|
|
||||||
The feature will add optional concurrency policy to registered backends and
|
|
||||||
coordinate `Run` calls against independent per-backend capacity pools.
|
|
||||||
|
|
||||||
Each policy has two distinct controls:
|
|
||||||
|
|
||||||
- an active-generation limit, which protects the backend from too many
|
|
||||||
simultaneous model requests; and
|
|
||||||
- a bounded waiting capacity, which protects the process from admitting an
|
|
||||||
unbounded backlog.
|
|
||||||
|
|
||||||
Concurrency policy belongs to a backend registration. It is not a profile
|
|
||||||
model parameter and cannot be overridden per run. Profiles select the policy
|
|
||||||
through their backend ID, while a profile or request endpoint override remains
|
|
||||||
in the selected backend's pool.
|
|
||||||
|
|
||||||
`Prepare` does not call a model and will remain outside concurrency admission.
|
|
||||||
|
|
||||||
## Defaults And Configuration
|
|
||||||
|
|
||||||
The built-in OpenRouter backend will use:
|
|
||||||
|
|
||||||
- an active-generation limit of 16; and
|
|
||||||
- a waiting capacity of 1024.
|
|
||||||
|
|
||||||
The waiting default is intentionally generous. Reaching it should indicate
|
|
||||||
abnormal submission pressure rather than normal application behavior.
|
|
||||||
|
|
||||||
Consumer-registered backends will remain unlimited unless the consumer
|
|
||||||
configures an active-generation limit. When a consumer enables a limit and
|
|
||||||
does not specify waiting capacity, the waiting capacity will default to 1024.
|
|
||||||
Consumers may configure a different bounded capacity, including zero when
|
|
||||||
they want no admitted backlog beyond the active-limit-sized run set.
|
|
||||||
|
|
||||||
The public representation must distinguish an omitted waiting capacity from
|
|
||||||
an explicit zero.
|
|
||||||
|
|
||||||
Endpoint-only profiles have no backend registration from which to obtain
|
|
||||||
policy and will remain unlimited. A future engine-wide or endpoint-keyed
|
|
||||||
policy can be considered separately if consumers demonstrate that need.
|
|
||||||
|
|
||||||
Invalid limits or capacities will fail engine construction as invalid
|
|
||||||
configuration. Policy values will be copied into engine-owned immutable state
|
|
||||||
along with the rest of the backend registration.
|
|
||||||
|
|
||||||
## Admission And Execution Behavior
|
|
||||||
|
|
||||||
`Run` remains a synchronous, wait-for-result operation. Concurrent callers may
|
|
||||||
block inside `Run` while waiting for their selected backend, then receive the
|
|
||||||
ordinary result or error from that invocation.
|
|
||||||
|
|
||||||
For a configured pool, the active-generation limit plus the waiting capacity
|
|
||||||
defines the maximum number of concurrent `Run` invocations that Promptkit will
|
|
||||||
accept for that backend. A waiting capacity of zero therefore accepts no more
|
|
||||||
runs than the active limit. Admission is immediate: a call either reserves one
|
|
||||||
of those bounded slots or receives the capacity error. An accepted run may
|
|
||||||
then wait internally for active-generation capacity.
|
|
||||||
|
|
||||||
For a limited backend, Promptkit will bound the number of accepted runs before
|
|
||||||
expensive artifact loading, prompt rendering, and large defensive copies where
|
|
||||||
practical. Lightweight prompt, profile, and backend resolution may occur first
|
|
||||||
when it is required to identify the correct capacity pool. This pre-admission
|
|
||||||
resolution must not become a second execution-precedence path with behavior
|
|
||||||
that can drift from `Prepare`.
|
|
||||||
|
|
||||||
An accepted run retains its admission until it completes or fails. Every
|
|
||||||
actual model-generation call for that run must separately observe the
|
|
||||||
backend's active-generation limit. This includes:
|
|
||||||
|
|
||||||
- the initial generation;
|
|
||||||
- every output-repair generation; and
|
|
||||||
- calls made through either the built-in or an injected model client.
|
|
||||||
|
|
||||||
Preparation and output validation should not hold an active-generation permit.
|
|
||||||
A repair remains part of its already-admitted run, but reacquires active
|
|
||||||
generation capacity so repairs cannot exceed the backend limit. It must not be
|
|
||||||
rejected merely because new runs filled the waiting queue after its initial
|
|
||||||
generation.
|
|
||||||
|
|
||||||
Within one backend pool, waiting generation calls should be served in FIFO
|
|
||||||
order, subject to canceled calls being removed. Different backend pools make
|
|
||||||
progress independently; a saturated local backend must not consume
|
|
||||||
OpenRouter's active or waiting capacity.
|
|
||||||
|
|
||||||
The feature will not promise ordering across backend pools or completion order
|
|
||||||
among admitted runs.
|
|
||||||
|
|
||||||
## Capacity Failure And Cancellation
|
|
||||||
|
|
||||||
When a backend's bounded waiting capacity is full, a new `Run` call will fail
|
|
||||||
promptly rather than waiting outside the bounded admission system. The public
|
|
||||||
API will expose a recognizable capacity-exhaustion error identity distinct
|
|
||||||
from invalid configuration, invalid requests, and model-client failures.
|
|
||||||
Rejected calls return no partial result and do not invoke the model client.
|
|
||||||
|
|
||||||
Waiting within the admitted backlog or for active-generation capacity must
|
|
||||||
honor the caller's context. Cancellation or deadline expiry will:
|
|
||||||
|
|
||||||
- stop waiting promptly;
|
|
||||||
- release any admission or generation capacity held by that invocation;
|
|
||||||
- preserve the applicable context error identity; and
|
|
||||||
- avoid invoking the model client if cancellation wins before generation
|
|
||||||
starts.
|
|
||||||
|
|
||||||
Capacity must also be released after preparation, generation, validation,
|
|
||||||
repair, or collaborator failure. One failed or canceled run must not reduce
|
|
||||||
the backend's future usable capacity.
|
|
||||||
|
|
||||||
Elapsed `Run` timing will include time spent waiting after the call is
|
|
||||||
accepted. `PreparedRun` timing will continue to describe preparation rather
|
|
||||||
than queue waiting.
|
|
||||||
|
|
||||||
## Engine And Client Boundaries
|
|
||||||
|
|
||||||
All pools and queued state belong to one `Engine`. Separate engines do not
|
|
||||||
share capacity, even when they register the same backend ID or endpoint. The
|
|
||||||
feature introduces no process-global scheduler.
|
|
||||||
|
|
||||||
The engine will apply policy consistently to the built-in model client and an
|
|
||||||
injected `LLMClient`. Consumers calling their own client outside Promptkit are
|
|
||||||
outside this boundary. Injected clients remain responsible for their internal
|
|
||||||
thread safety and cancellation behavior.
|
|
||||||
|
|
||||||
Backend policy is keyed by the resolved backend ID rather than endpoint text.
|
|
||||||
This preserves stable routing when a selected backend's endpoint is overridden
|
|
||||||
and avoids accidentally combining unrelated registrations that happen to use
|
|
||||||
the same URL.
|
|
||||||
|
|
||||||
## Queue Lifetime And Observability
|
|
||||||
|
|
||||||
Admission state is buffered, ephemeral, and in-process. It is not persisted
|
|
||||||
and has no survival guarantee across engine disposal or process termination.
|
|
||||||
Promptkit will not introduce background job ownership or require consumers to
|
|
||||||
start or stop workers.
|
|
||||||
|
|
||||||
The initial feature does not require public queue-depth metrics, callbacks, or
|
|
||||||
inspection APIs. Capacity errors and ordinary call timing provide the
|
|
||||||
consumer-visible behavior. Operational observability can be added later
|
|
||||||
without coupling the scheduling mechanism to an application logging or
|
|
||||||
metrics system.
|
|
||||||
|
|
||||||
## Compatibility
|
|
||||||
|
|
||||||
Consumer-registered backends and endpoint-only profiles remain unlimited
|
|
||||||
unless concurrency is explicitly configured, preserving their existing
|
|
||||||
behavior.
|
|
||||||
|
|
||||||
The built-in OpenRouter backend will change from unlimited concurrency to a
|
|
||||||
limit of 16 with a bounded waiting capacity of 1024. Ordinary synchronous
|
|
||||||
calls remain unchanged, while unusually high concurrent use may now wait or
|
|
||||||
return the capacity error. This behavioral change must be identified in the
|
|
||||||
release notes for the version that publishes it.
|
|
||||||
|
|
||||||
Adding backend policy fields and a public capacity error is otherwise
|
|
||||||
additive. The change will use a pre-`v1` minor release under Promptkit's
|
|
||||||
[release policy](../release.md#release-model).
|
|
||||||
|
|
||||||
## Non-Goals
|
|
||||||
|
|
||||||
This scope does not include:
|
|
||||||
|
|
||||||
- asynchronous job handles, polling, or detached result delivery;
|
|
||||||
- durable or cross-process queues;
|
|
||||||
- persistence or recovery across engine or process shutdown;
|
|
||||||
- priorities, scheduling weights, or consumer-defined fairness classes;
|
|
||||||
- automatic retries, backoff, rate-limit interpretation, or provider quota
|
|
||||||
discovery;
|
|
||||||
- token-per-minute or request-per-minute rate limiting;
|
|
||||||
- dynamic reconfiguration after engine construction;
|
|
||||||
- per-profile or per-run concurrency overrides;
|
|
||||||
- endpoint-keyed pooling for profiles without a backend ID;
|
|
||||||
- process-global coordination across engines;
|
|
||||||
- application worker lifecycle, logging, tracing, or metrics policy; or
|
|
||||||
- changes to prompt, profile, schema, or model-provider wire formats.
|
|
||||||
|
|
||||||
## Target End State
|
|
||||||
|
|
||||||
This roadmap reaches its target end state when:
|
|
||||||
|
|
||||||
- each engine independently coordinates configured backend capacity;
|
|
||||||
- the built-in OpenRouter backend allows 16 active generations and up to 1024
|
|
||||||
waiting runs;
|
|
||||||
- consumer backends can opt into their own active and waiting limits while
|
|
||||||
remaining unlimited by default;
|
|
||||||
- endpoint overrides retain the selected backend's capacity pool and
|
|
||||||
endpoint-only profiles remain unlimited;
|
|
||||||
- synchronous `Run` callers wait for and receive their ordinary result;
|
|
||||||
- admission is bounded before expensive preparation work where practical;
|
|
||||||
- every initial and repair generation observes the backend's active limit
|
|
||||||
without serializing preparation or validation;
|
|
||||||
- a full waiting queue returns a recognizable capacity error without invoking
|
|
||||||
the model client;
|
|
||||||
- cancellation and all failure paths promptly release capacity and preserve
|
|
||||||
context error identity;
|
|
||||||
- built-in and injected model clients receive the same scheduling behavior;
|
|
||||||
- pools remain ephemeral, engine-scoped, and independent across backend IDs;
|
|
||||||
and
|
|
||||||
- current-state GoDoc, consumer, internal, and release documentation describe
|
|
||||||
the implemented behavior once it lands.
|
|
||||||
82
docs/roadmap/deferred.md
Normal file
82
docs/roadmap/deferred.md
Normal file
@@ -0,0 +1,82 @@
|
|||||||
|
# Deferred Feature Ideas
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
|
||||||
|
This document catalogs feature ideas that remain potentially useful but have
|
||||||
|
been deliberately postponed. These ideas are not awaiting ordinary selection
|
||||||
|
from the [future feature catalog](future.md); each has a stated reason to wait
|
||||||
|
and should be reconsidered only when its trigger becomes relevant.
|
||||||
|
|
||||||
|
Deferred entries are not commitments, schedules, active implementation plans,
|
||||||
|
or descriptions of current behavior. When an entry is reactivated, move it to
|
||||||
|
`future.md` for evaluation or directly into a focused roadmap after its open
|
||||||
|
design dependencies have been resolved.
|
||||||
|
|
||||||
|
## Deferred Ideas
|
||||||
|
|
||||||
|
### Semantic Execution-Target Fingerprints
|
||||||
|
|
||||||
|
**Reason for deferral:** A stable digest requires a deliberate semantic-
|
||||||
|
equality and versioning design. Notarius can safely use conservative source
|
||||||
|
hashes and a Promptkit release marker today, while Weatherreporter does not
|
||||||
|
currently reuse LLM-dependent checkpoints.
|
||||||
|
|
||||||
|
Promptkit could expose an opaque equality value for a resolved profile and its
|
||||||
|
effective generation target. This would let checkpointing consumers detect
|
||||||
|
generation-affecting configuration changes without hashing YAML presentation
|
||||||
|
or depending on Promptkit's built-in catalog layout.
|
||||||
|
|
||||||
|
The digest should change with semantically relevant state such as the resolved
|
||||||
|
model, endpoint, backend routing identity, request defaults, extra parameters,
|
||||||
|
profile generation settings, and selected built-in profile semantics. It
|
||||||
|
should exclude credential values, concurrency and queue policy, source paths,
|
||||||
|
comments, formatting, and other representation-only changes. Whether a
|
||||||
|
credential environment-variable name affects equality must be decided
|
||||||
|
explicitly. The encoding should remain opaque and internally versioned so
|
||||||
|
Promptkit can deliberately invalidate earlier digests when its resolution
|
||||||
|
semantics change.
|
||||||
|
|
||||||
|
Reconsider this idea when a downstream consumer needs Promptkit-owned
|
||||||
|
checkpoint equality or when a broader semantic identity design is selected.
|
||||||
|
|
||||||
|
### Eager Source Validation
|
||||||
|
|
||||||
|
**Reason for deferral:** Exact prompt and profile inspection may already
|
||||||
|
provide a sufficiently small validation surface. Experience from downstream
|
||||||
|
adoption should establish whether an engine-wide operation would add enough
|
||||||
|
value to justify its broader contract.
|
||||||
|
|
||||||
|
Promptkit could provide an explicit offline operation that discovers and
|
||||||
|
structurally validates configured prompt, profile, and schema sources without
|
||||||
|
model generation. The normal `NewEngine` path would remain lazy.
|
||||||
|
|
||||||
|
An eager operation would need coherent handling for duplicate prompt IDs and
|
||||||
|
versions, strict YAML decoding, referenced content files, profile/backend
|
||||||
|
membership, schema syntax and transitive references, context cancellation,
|
||||||
|
and source-specific public errors. Credential declarations must remain
|
||||||
|
separate from credential values; checking current environment availability,
|
||||||
|
if supported at all, should be an explicit option and must not expose secrets.
|
||||||
|
|
||||||
|
Reconsider this idea after downstream use of `InspectPrompt`,
|
||||||
|
`InspectProfile`, and fixture-based preparation demonstrates a concrete gap.
|
||||||
|
|
||||||
|
### Structured Generation Errors
|
||||||
|
|
||||||
|
**Reason for deferral:** Existing `ErrLLMGenerate` classification, preserved
|
||||||
|
injected-client errors, and prepared execution details currently provide the
|
||||||
|
necessary failure boundary. A typed error should wait for stronger downstream
|
||||||
|
demand and a transport-neutral field design.
|
||||||
|
|
||||||
|
Promptkit could expose safe structured generation context through
|
||||||
|
`errors.As` while preserving `errors.Is(err, ErrLLMGenerate)`. Potential
|
||||||
|
fields include the selected backend ID and model plus an optional HTTP status
|
||||||
|
when the built-in OpenAI-compatible transport supplies one.
|
||||||
|
|
||||||
|
The design must not expose provider response bodies, endpoints, credential
|
||||||
|
environment names or values, request content, or generated content. It should
|
||||||
|
not duplicate prompt and profile provenance already available from a prepared
|
||||||
|
execution, and it must preserve the identity of errors returned by injected
|
||||||
|
clients. Retry and backoff policy remains a consumer responsibility.
|
||||||
|
|
||||||
|
Reconsider this idea when consumers need structured generation diagnostics
|
||||||
|
beyond the existing sentinel, wrapped client error, and preparation record.
|
||||||
@@ -12,6 +12,9 @@ consumer value, and important scope boundaries. Defer API design,
|
|||||||
implementation details, sequencing, and acceptance criteria until an idea is
|
implementation details, sequencing, and acceptance criteria until an idea is
|
||||||
selected.
|
selected.
|
||||||
|
|
||||||
|
Ideas that have been deliberately postponed rather than left available for
|
||||||
|
ordinary selection belong in the [deferred catalog](deferred.md).
|
||||||
|
|
||||||
## Using This Catalog
|
## Using This Catalog
|
||||||
|
|
||||||
- Add an idea when its purpose and likely value can be stated clearly.
|
- Add an idea when its purpose and likely value can be stated clearly.
|
||||||
@@ -23,6 +26,8 @@ selected.
|
|||||||
- When an idea is selected, move its active planning to a focused roadmap or,
|
- When an idea is selected, move its active planning to a focused roadmap or,
|
||||||
when it requires a durable architectural decision, an ADR. Update
|
when it requires a durable architectural decision, an ADR. Update
|
||||||
current-state documentation only when implementation lands.
|
current-state documentation only when implementation lands.
|
||||||
|
- Move an idea to `deferred.md` when maintainers decide to retain it but wait
|
||||||
|
for a stated design dependency, demand signal, or reconsideration trigger.
|
||||||
- Remove ideas that are no longer relevant. Retain a rejected idea only when
|
- Remove ideas that are no longer relevant. Retain a rejected idea only when
|
||||||
its rationale is likely to prevent repeated reconsideration.
|
its rationale is likely to prevent repeated reconsideration.
|
||||||
|
|
||||||
@@ -33,9 +38,7 @@ consumers.
|
|||||||
|
|
||||||
## Ideas
|
## Ideas
|
||||||
|
|
||||||
No ideas are currently cataloged. Backend-specific concurrency management has
|
No ideas currently await selection.
|
||||||
been selected for active planning in the
|
|
||||||
[focused concurrency roadmap](concurrency.md).
|
|
||||||
|
|
||||||
## Entry Format
|
## Entry Format
|
||||||
|
|
||||||
|
|||||||
@@ -1,841 +0,0 @@
|
|||||||
# Backend-Specific Concurrency Management Implementation Plan
|
|
||||||
|
|
||||||
**Status:** Complete.
|
|
||||||
|
|
||||||
## Purpose
|
|
||||||
|
|
||||||
This document is the decision-complete implementation plan for
|
|
||||||
[backend-specific concurrency management](concurrency.md). It is written for a
|
|
||||||
coding agent that will implement each stage in order.
|
|
||||||
|
|
||||||
The feature roadmap owns the intended capability, consumer value, policy
|
|
||||||
choices, compatibility decision, and target end state. This document owns the
|
|
||||||
concrete API, internal representation, scheduling architecture, implementation
|
|
||||||
sequence, test ownership, documentation updates, and completion gates.
|
|
||||||
|
|
||||||
## Implementation Rules
|
|
||||||
|
|
||||||
- Complete the stages in order. Keep the repository compiling and the focused
|
|
||||||
tests passing at every stage boundary.
|
|
||||||
- Preserve unrelated working-tree changes. In particular, `concurrency.md` and
|
|
||||||
the removal of its source idea from `future.md` may already be uncommitted
|
|
||||||
when implementation begins; retain both.
|
|
||||||
- Follow every policy under `docs/policy/`, the task-specific reading guide in
|
|
||||||
`docs/development.md`, and the target behavior in `concurrency.md`.
|
|
||||||
- Keep the public API in the root `promptkit` package and implementation
|
|
||||||
details under `internal/`. Do not expose scheduler types or create another
|
|
||||||
public package.
|
|
||||||
- Use only the Go standard library for scheduling. Do not add a queue,
|
|
||||||
semaphore, worker-pool, or metrics dependency.
|
|
||||||
- Preserve synchronous, wait-for-result `Run`, unrestricted `Prepare`,
|
|
||||||
engine-local state, endpoint-only profiles, backend-selected profiles,
|
|
||||||
backend identity through endpoint overrides, and injected `LLMClient`
|
|
||||||
behavior.
|
|
||||||
- Do not broaden the work into asynchronous jobs, durable queues, retries,
|
|
||||||
rate limiting, dynamic configuration, priorities, worker lifecycle,
|
|
||||||
endpoint-keyed pools, or public queue observability.
|
|
||||||
- Keep all tests deterministic, bounded, offline, and race-safe. Coordinate
|
|
||||||
concurrent tests with channels and barriers rather than timing assumptions
|
|
||||||
or live providers.
|
|
||||||
- Update exact GoDoc with each exported declaration change. Update durable
|
|
||||||
current-state documents only after the corresponding behavior is
|
|
||||||
implemented.
|
|
||||||
- Test configurable mechanisms with small test-owned limits. Assert the exact
|
|
||||||
OpenRouter `16` and default queue `1024` values only at the registry contract
|
|
||||||
that owns those operational defaults.
|
|
||||||
- Do not create a release, change a module version, or tag a commit. The final
|
|
||||||
implementation handoff must identify the built-in OpenRouter behavior change
|
|
||||||
for the next pre-`v1` minor release.
|
|
||||||
|
|
||||||
## Fixed Design
|
|
||||||
|
|
||||||
### Public Backend Configuration
|
|
||||||
|
|
||||||
Append these fields to the existing root `Backend` type in `backends.go`:
|
|
||||||
|
|
||||||
```go
|
|
||||||
type Backend struct {
|
|
||||||
// Existing fields remain unchanged and in their current order.
|
|
||||||
|
|
||||||
ConcurrencyLimit int
|
|
||||||
QueueCapacity *int
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
Use these exact semantics:
|
|
||||||
|
|
||||||
| Public values | Meaning |
|
|
||||||
| --- | --- |
|
|
||||||
| `ConcurrencyLimit == 0`, `QueueCapacity == nil` | Unlimited backend; preserve current behavior. |
|
|
||||||
| `ConcurrencyLimit > 0`, `QueueCapacity == nil` | Limit active generations and use the default waiting capacity of 1024. |
|
|
||||||
| `ConcurrencyLimit > 0`, `QueueCapacity != nil` | Limit active generations and use the pointed-to capacity exactly, including zero. |
|
|
||||||
| `ConcurrencyLimit < 0` | Invalid engine configuration. |
|
|
||||||
| `QueueCapacity != nil` and `*QueueCapacity < 0` | Invalid engine configuration. |
|
|
||||||
| `ConcurrencyLimit == 0` and `QueueCapacity != nil` | Invalid engine configuration because a queue without an active limit has no defined consumer value. |
|
|
||||||
|
|
||||||
`ConcurrencyLimit` counts simultaneous calls to the engine-owned internal
|
|
||||||
model-client boundary for this backend. `QueueCapacity` controls additional
|
|
||||||
accepted `Run` invocations beyond that limit. The maximum admitted runs for a
|
|
||||||
limited backend is therefore:
|
|
||||||
|
|
||||||
```text
|
|
||||||
ConcurrencyLimit + effective QueueCapacity
|
|
||||||
```
|
|
||||||
|
|
||||||
Guard that addition against integer overflow during backend validation.
|
|
||||||
Do not impose an arbitrary upper bound beyond non-negativity and overflow
|
|
||||||
safety.
|
|
||||||
|
|
||||||
The `QueueCapacity` pointer exists only to distinguish omission from explicit
|
|
||||||
zero. `WithBackend` and `NewEngine` must not retain the caller's pointer.
|
|
||||||
`Backend` continues to have no stable JSON representation, and consumers
|
|
||||||
remain directed to keyed literals.
|
|
||||||
|
|
||||||
Do not add concurrency fields to `Profile`, `ExecutionTarget`,
|
|
||||||
`ExecutionTargetOverride`, `RunRequest`, prompt or profile files, or stable
|
|
||||||
prepared/result JSON.
|
|
||||||
|
|
||||||
### Built-In And Custom Defaults
|
|
||||||
|
|
||||||
The backend registry owns these exact operational defaults:
|
|
||||||
|
|
||||||
```go
|
|
||||||
const (
|
|
||||||
openRouterConcurrencyLimit = 16
|
|
||||||
defaultQueueCapacity = 1024
|
|
||||||
)
|
|
||||||
```
|
|
||||||
|
|
||||||
The built-in `openrouter` definition has a normalized concurrency limit of 16
|
|
||||||
and queue capacity of 1024.
|
|
||||||
|
|
||||||
Consumer registrations remain unlimited when concurrency is omitted. For a
|
|
||||||
consumer backend with a positive limit and omitted queue capacity, normalize
|
|
||||||
the queue capacity to 1024. Preserve an explicitly configured zero.
|
|
||||||
|
|
||||||
Consumers still cannot replace the reserved `openrouter` registration.
|
|
||||||
Endpoint-only profiles have no backend policy and remain unlimited. A selected
|
|
||||||
backend retains its pool when a profile or request overrides only its endpoint.
|
|
||||||
|
|
||||||
### Internal Backend Representation
|
|
||||||
|
|
||||||
Extend `internal/domain.Backend` with scalar policy values and explicit
|
|
||||||
presence rather than retaining a pointer:
|
|
||||||
|
|
||||||
```go
|
|
||||||
type Backend struct {
|
|
||||||
// Existing fields...
|
|
||||||
ConcurrencyLimit int
|
|
||||||
QueueCapacity int
|
|
||||||
QueueCapacitySet bool
|
|
||||||
}
|
|
||||||
|
|
||||||
type BackendCapacityPolicy struct {
|
|
||||||
ConcurrencyLimit int
|
|
||||||
QueueCapacity int
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
`WithBackend` converts the public pointer into `QueueCapacity` plus
|
|
||||||
`QueueCapacitySet`. Registry normalization validates the combinations above,
|
|
||||||
fills the default, and leaves every limited stored backend with
|
|
||||||
`QueueCapacitySet == true`. Unlimited stored backends retain zero values and
|
|
||||||
`QueueCapacitySet == false`.
|
|
||||||
|
|
||||||
Add this internal registry method:
|
|
||||||
|
|
||||||
```go
|
|
||||||
func (r *Registry) CapacityPolicies() map[string]domain.BackendCapacityPolicy
|
|
||||||
```
|
|
||||||
|
|
||||||
It returns a newly allocated map containing only limited backends. Values are
|
|
||||||
scalars, so callers cannot mutate registry state. The built-in OpenRouter
|
|
||||||
policy is included. `GetBackend` continues returning a defensive backend copy,
|
|
||||||
now including normalized scalar capacity metadata.
|
|
||||||
|
|
||||||
Capacity policy is operational registry metadata. Do not merge it into an
|
|
||||||
execution target or expose it to injected model clients.
|
|
||||||
|
|
||||||
### Public Capacity Error
|
|
||||||
|
|
||||||
Add this root sentinel beside the other run errors in `engine.go`:
|
|
||||||
|
|
||||||
```go
|
|
||||||
var ErrCapacityExceeded = errors.New("backend capacity exceeded")
|
|
||||||
```
|
|
||||||
|
|
||||||
Its GoDoc must state that it identifies a `Run` rejected because the selected
|
|
||||||
backend has already admitted `ConcurrencyLimit + QueueCapacity` runs. It is
|
|
||||||
not an invalid request, an LLM/provider rate-limit response, or an
|
|
||||||
`ErrLLMGenerate` failure.
|
|
||||||
|
|
||||||
The internal capacity component owns a corresponding internal
|
|
||||||
`ErrCapacityExceeded`. Add its mapping in `publicErrorFor` before the broader
|
|
||||||
generation and invalid-request cases. The public error must preserve the
|
|
||||||
internal error through wrapping while matching `ErrCapacityExceeded` with
|
|
||||||
`errors.Is`.
|
|
||||||
|
|
||||||
A capacity rejection returns no partial result and must not invoke the
|
|
||||||
artifact reader, renderer, schema loader, validator, or model client. Prompt,
|
|
||||||
profile, and backend loading needed to select the pool may already have
|
|
||||||
occurred.
|
|
||||||
|
|
||||||
### Internal Capacity Component
|
|
||||||
|
|
||||||
Add `internal/capacity` as the single owner of engine-local run admission and
|
|
||||||
active-generation permits.
|
|
||||||
|
|
||||||
Use these package-level boundaries:
|
|
||||||
|
|
||||||
```go
|
|
||||||
var ErrCapacityExceeded error
|
|
||||||
|
|
||||||
type Manager struct {
|
|
||||||
// Private immutable pool map.
|
|
||||||
}
|
|
||||||
|
|
||||||
func NewManager(
|
|
||||||
policies map[string]domain.BackendCapacityPolicy,
|
|
||||||
) (*Manager, error)
|
|
||||||
|
|
||||||
func (m *Manager) Admit(
|
|
||||||
ctx context.Context,
|
|
||||||
backendID string,
|
|
||||||
) (release func(), err error)
|
|
||||||
|
|
||||||
func NewClient(m *Manager, next llm.Client) llm.Client
|
|
||||||
```
|
|
||||||
|
|
||||||
`NewManager` copies the supplied map and creates one independent pool per
|
|
||||||
limited backend. Defensively reject blank IDs, non-positive concurrency
|
|
||||||
limits, negative queue capacities, or total-capacity overflow even though the
|
|
||||||
registry normally supplies normalized values. Construction creates no worker
|
|
||||||
goroutines.
|
|
||||||
|
|
||||||
An absent manager, blank backend ID, or ID absent from the policy map is
|
|
||||||
unlimited:
|
|
||||||
|
|
||||||
- `Admit` succeeds with a non-nil no-op release function; and
|
|
||||||
- the client wrapper calls the next client directly.
|
|
||||||
|
|
||||||
For a limited pool, `Admit` is immediate and context-aware:
|
|
||||||
|
|
||||||
1. return `ctx.Err()` if the context is already done;
|
|
||||||
2. under the pool lock, compare admitted runs with
|
|
||||||
`ConcurrencyLimit + QueueCapacity`;
|
|
||||||
3. return an error matching internal `ErrCapacityExceeded` when full; or
|
|
||||||
4. increment admitted runs and return an idempotent release function.
|
|
||||||
|
|
||||||
The release function decrements admission exactly once, even if accidentally
|
|
||||||
called more than once. It does not release an active-generation permit; those
|
|
||||||
permits have their own lifetime.
|
|
||||||
|
|
||||||
### FIFO Generation Permits
|
|
||||||
|
|
||||||
`NewClient` returns an internal `llm.Client` wrapper around either the built-in
|
|
||||||
client or the public-client adapter. It must preserve requests, successful
|
|
||||||
responses, nil responses, and collaborator error identities exactly.
|
|
||||||
`next` must be non-nil; `NewEngine` and internal runner construction maintain
|
|
||||||
that invariant. A nil manager returns `next` unchanged.
|
|
||||||
|
|
||||||
For a configured backend ID, the wrapper:
|
|
||||||
|
|
||||||
1. acquires one active-generation permit from the matching pool;
|
|
||||||
2. waits in FIFO order when the active count equals `ConcurrencyLimit`;
|
|
||||||
3. removes a canceled waiter and returns `ctx.Err()` when cancellation wins
|
|
||||||
before the permit is granted;
|
|
||||||
4. invokes the next client only after a permit is granted; and
|
|
||||||
5. releases the permit with `defer` after every success, nil response,
|
|
||||||
collaborator error, panic unwinding, or context outcome.
|
|
||||||
|
|
||||||
Implement FIFO and cancellation explicitly with a mutex and an ordered waiter
|
|
||||||
list. A channel used only as a counting semaphore is insufficient because it
|
|
||||||
does not define FIFO ordering or safe removal of canceled waiters.
|
|
||||||
|
|
||||||
Permit grant and cancellation must have one lock-protected linearization
|
|
||||||
point. If cancellation removes the waiter first, do not invoke the next
|
|
||||||
client. If grant wins first, invoke the next client with the caller's context;
|
|
||||||
the next client may then observe cancellation normally. Never lose or
|
|
||||||
double-release a permit in this race.
|
|
||||||
|
|
||||||
Releasing a permit transfers it to the oldest non-canceled waiter before
|
|
||||||
making it generally available. Different backend pools never share admission
|
|
||||||
or active counts.
|
|
||||||
|
|
||||||
The active wrapper enforces its limit even if an internal caller invokes it
|
|
||||||
without a run admission lease. Bounded backlog is guaranteed for ordinary
|
|
||||||
engine `Run` calls by the runner admission path; no public API exposes the
|
|
||||||
wrapped internal client directly.
|
|
||||||
|
|
||||||
### Engine Assembly
|
|
||||||
|
|
||||||
In `NewEngine`, after constructing the validated backend registry:
|
|
||||||
|
|
||||||
1. obtain `backendRegistry.CapacityPolicies()`;
|
|
||||||
2. construct one `capacity.Manager`;
|
|
||||||
3. construct the selected base internal LLM client exactly as today;
|
|
||||||
4. wrap that base client with `capacity.NewClient`; and
|
|
||||||
5. pass both the wrapped client and manager-as-admitter to the runner.
|
|
||||||
|
|
||||||
Every `NewEngine` call constructs a distinct manager. Do not cache managers,
|
|
||||||
pools, or policies in package globals. The wrapper must be applied after a
|
|
||||||
public injected client is adapted to `internal/llm.Client`, so built-in and
|
|
||||||
injected clients receive identical scheduling behavior.
|
|
||||||
|
|
||||||
If `NewManager` reports a defensive configuration error, make `NewEngine`
|
|
||||||
return an error matching `ErrInvalidConfig`.
|
|
||||||
|
|
||||||
`Prepare` does not use the manager. An injected client remains required to be
|
|
||||||
safe for concurrent calls because different backend pools and unlimited
|
|
||||||
backends may still invoke it concurrently.
|
|
||||||
|
|
||||||
### Shared Two-Phase Preparation
|
|
||||||
|
|
||||||
Refactor `internal/usecase.Runner` so `Prepare` and `Run` share one preparation
|
|
||||||
pipeline with two private phases. Do not duplicate prompt/profile/backend
|
|
||||||
selection or execution precedence.
|
|
||||||
|
|
||||||
The first phase resolves only the state required before admission:
|
|
||||||
|
|
||||||
1. validate `PromptID`;
|
|
||||||
2. normalize the direct session ID;
|
|
||||||
3. load the prompt definition;
|
|
||||||
4. hash the original prompt definition at its existing error-order position;
|
|
||||||
5. select and load the execution profile;
|
|
||||||
6. resolve the selected backend;
|
|
||||||
7. resolve and validate the effective execution target and credentials; and
|
|
||||||
8. resolve the effective output contract without loading its schema.
|
|
||||||
|
|
||||||
Return a private state value containing the loaded definition, normalized
|
|
||||||
direct session, prompt-definition hash, selected profile ID, effective target,
|
|
||||||
numeric-presence metadata, effective output contract, and preparation start
|
|
||||||
time. Keep this value private to `internal/usecase`.
|
|
||||||
|
|
||||||
The second phase consumes that state and performs:
|
|
||||||
|
|
||||||
1. structured-output schema loading;
|
|
||||||
2. artifact loading and input hashing;
|
|
||||||
3. message and prompt-session rendering;
|
|
||||||
4. direct-session application;
|
|
||||||
5. rendered-prompt hashing; and
|
|
||||||
6. `PreparedRun` construction and timing.
|
|
||||||
|
|
||||||
Preserve every existing precedence rule, error identity, direct-session
|
|
||||||
template bypass, hash input, selected identity, copy guarantee, and timing
|
|
||||||
field. Do not reload the prompt, profile, or backend between phases.
|
|
||||||
|
|
||||||
`Runner.Prepare` records its start time, runs both phases consecutively, and
|
|
||||||
never calls admission. Its behavior and error ordering remain unchanged.
|
|
||||||
|
|
||||||
`Runner.Run` records its existing run start time, runs the first preparation
|
|
||||||
phase, and then calls:
|
|
||||||
|
|
||||||
```go
|
|
||||||
release, err := r.admitter.Admit(ctx, effectiveBackendID)
|
|
||||||
```
|
|
||||||
|
|
||||||
Use a narrow use-case-owned interface with the same signature:
|
|
||||||
|
|
||||||
```go
|
|
||||||
type RunAdmitter interface {
|
|
||||||
Admit(context.Context, string) (func(), error)
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
A nil admitter means unlimited behavior for internal constructors and tests.
|
|
||||||
On successful admission, immediately `defer release()` around the remainder of
|
|
||||||
the run. Then run the second preparation phase, initial generation,
|
|
||||||
validation, and all repair attempts.
|
|
||||||
|
|
||||||
If admission returns internal `capacity.ErrCapacityExceeded`, add useful
|
|
||||||
backend context without changing its identity. If it returns `ctx.Err()`,
|
|
||||||
preserve that identity directly rather than recategorizing it as invalid
|
|
||||||
request or generation failure.
|
|
||||||
|
|
||||||
This refactor intentionally replaces the current literal `Run`-calls-`Prepare`
|
|
||||||
implementation with shared private phases. Update current-state documentation
|
|
||||||
to describe one shared pipeline rather than retaining an inaccurate call-graph
|
|
||||||
claim.
|
|
||||||
|
|
||||||
### Generation And Repair Lifetime
|
|
||||||
|
|
||||||
The admission lease covers the entire accepted run:
|
|
||||||
|
|
||||||
- second-phase preparation;
|
|
||||||
- initial generation;
|
|
||||||
- validation;
|
|
||||||
- every repair; and
|
|
||||||
- all failure and cancellation exits.
|
|
||||||
|
|
||||||
Preparation and validation do not hold an active-generation permit. The
|
|
||||||
wrapped client acquires a permit only around each actual `Generate` call.
|
|
||||||
|
|
||||||
The runner's initial generation already carries the effective backend ID in
|
|
||||||
`GenerateRequest.Target`. Preserve that value. `RepairRequest.Target` and the
|
|
||||||
default repairer's generated request must continue carrying the same backend
|
|
||||||
ID, allowing each repair to reacquire the same pool's active permit.
|
|
||||||
|
|
||||||
When testing or constructing `NewRunnerWithRepairer`, pass the same wrapped
|
|
||||||
client to both the runner and `NewDefaultOutputRepairer`. Do not add capacity
|
|
||||||
state to `RepairRequest`, `ExecutionTarget`, or public generation values.
|
|
||||||
|
|
||||||
A repair remains within its existing admission lease. It waits for a FIFO
|
|
||||||
active permit but never performs a second bounded admission and therefore
|
|
||||||
cannot fail merely because later runs filled the admission capacity.
|
|
||||||
|
|
||||||
### Error And Cancellation Semantics
|
|
||||||
|
|
||||||
The required public outcomes are:
|
|
||||||
|
|
||||||
| Situation | Required error identity |
|
|
||||||
| --- | --- |
|
|
||||||
| Admission capacity is full | `ErrCapacityExceeded` only; not `ErrInvalidRequest` or `ErrLLMGenerate`. |
|
|
||||||
| Context is done before admission succeeds | Preserve `ctx.Err()`; do not return capacity exhaustion. |
|
|
||||||
| Context cancels while waiting for an active permit | Preserve `ctx.Err()` through the existing `ErrLLMGenerate` generation category. |
|
|
||||||
| Wrapped client fails after permit acquisition | Preserve existing `ErrLLMGenerate` and collaborator identities. |
|
|
||||||
| Preparation or validation fails after admission | Preserve its existing category and release admission. |
|
|
||||||
|
|
||||||
Maintain the existing rule that `Run` returns no partial result on any
|
|
||||||
operational error. Do not add queue status to errors or results.
|
|
||||||
|
|
||||||
`RunResult.Duration` continues to start at runner entry and therefore includes
|
|
||||||
pre-admission resolution, accepted preparation, and active-permit waiting.
|
|
||||||
`PreparedRun.DurationMS` continues to cover only its shared preparation phases;
|
|
||||||
it does not include later generation waiting. Capacity-rejected calls have no
|
|
||||||
result or timing value.
|
|
||||||
|
|
||||||
### Ownership And Concurrency Safety
|
|
||||||
|
|
||||||
The registry, capacity policy map, pool map, and per-pool limits are immutable
|
|
||||||
after engine construction. Only admission counts, active counts, and waiter
|
|
||||||
lists are mutable and must be protected by the owning pool mutex.
|
|
||||||
|
|
||||||
Do not retain public queue pointers, caller request values, contexts, or
|
|
||||||
generation requests after their call completes. A canceled waiter must be
|
|
||||||
unlinked so its context and request cannot remain reachable from the pool.
|
|
||||||
|
|
||||||
Do not hold a pool mutex while:
|
|
||||||
|
|
||||||
- loading or rendering prompts;
|
|
||||||
- reading artifacts or schemas;
|
|
||||||
- invoking a model client;
|
|
||||||
- validating output;
|
|
||||||
- closing a waiter notification channel if the implementation could re-enter
|
|
||||||
pool code; or
|
|
||||||
- calling consumer code.
|
|
||||||
|
|
||||||
No scheduler operation may spawn a goroutine whose lifetime outlasts the
|
|
||||||
calling `Run`. The zero steady-state goroutine count is part of the
|
|
||||||
in-process/no-worker-lifecycle design.
|
|
||||||
|
|
||||||
## Test Ownership
|
|
||||||
|
|
||||||
Use this ownership split and avoid repeating the full policy matrix at every
|
|
||||||
layer:
|
|
||||||
|
|
||||||
- `internal/backend/registry_test.go` owns normalization, validation, the exact
|
|
||||||
OpenRouter policy, the custom default queue, explicit zero, unlimited
|
|
||||||
omission, and policy-map copying.
|
|
||||||
- `internal/capacity/manager_test.go` owns admission bounds, idempotent release,
|
|
||||||
FIFO active permits, cancellation races, capacity recovery, independent
|
|
||||||
pools, unlimited IDs, and observed peak concurrency.
|
|
||||||
- `internal/capacity/client_test.go` owns wrapper request/response/error
|
|
||||||
transparency and the rule that cancellation before grant does not invoke the
|
|
||||||
next client. Combine these with manager tests if one coherent package test
|
|
||||||
expresses the behavior more clearly.
|
|
||||||
- `internal/usecase/runner_test.go` owns two-phase preparation parity, pool
|
|
||||||
selection, admission before expensive work, admission release across run
|
|
||||||
exits, `Prepare` bypass, and repair reuse of the admitted backend.
|
|
||||||
- Root external-package tests own public configuration conversion, assembled
|
|
||||||
engine-local behavior, endpoint-override routing, injected-client limiting,
|
|
||||||
and public capacity/context error identities.
|
|
||||||
- Existing model-client HTTP tests remain unchanged because scheduling does
|
|
||||||
not alter the OpenAI-compatible wire contract.
|
|
||||||
|
|
||||||
Concurrency tests must use test-owned limits such as one or two and
|
|
||||||
channel-controlled blocking clients. Record observed active and peak counts
|
|
||||||
under a mutex or atomics. Do not use `time.Sleep` to infer queue state.
|
|
||||||
Package-internal tests may inspect a waiter list under its mutex through a
|
|
||||||
small test helper when necessary to establish deterministic FIFO ordering; do
|
|
||||||
not add production metrics or hooks solely for tests.
|
|
||||||
|
|
||||||
Do not add separate tests for trivial scalar copies when registry or assembled
|
|
||||||
behavior already protects them.
|
|
||||||
|
|
||||||
## Stage 1 — Backend Policy And Public Configuration
|
|
||||||
|
|
||||||
**Status:** Complete.
|
|
||||||
|
|
||||||
### Goal
|
|
||||||
|
|
||||||
Add the public and internal backend policy representation, normalize all
|
|
||||||
configured states, and expose immutable normalized policies without changing
|
|
||||||
runtime scheduling yet.
|
|
||||||
|
|
||||||
### Work
|
|
||||||
|
|
||||||
1. Add `ConcurrencyLimit` and `QueueCapacity` to `Backend` in `backends.go`
|
|
||||||
with exact GoDoc for unlimited, defaulted, explicit-zero, invalid, and
|
|
||||||
engine-scoped behavior.
|
|
||||||
2. Convert the public queue pointer into scalar value plus presence in
|
|
||||||
`WithBackend`; do not retain the pointer.
|
|
||||||
3. Add the internal backend policy fields and
|
|
||||||
`BackendCapacityPolicy` to `internal/domain/domain.go`.
|
|
||||||
4. Add the two registry-owned constants and configure the built-in OpenRouter
|
|
||||||
definition with 16 and 1024.
|
|
||||||
5. Extend `normalizeBackend` with the fixed validation, defaulting, explicit
|
|
||||||
zero, and overflow rules.
|
|
||||||
6. Add `Registry.CapacityPolicies`, returning only limited policies in a fresh
|
|
||||||
map.
|
|
||||||
7. Update existing backend composite literals and assertions only where the
|
|
||||||
new fields are relevant. Continue using keyed literals.
|
|
||||||
|
|
||||||
### Tests
|
|
||||||
|
|
||||||
1. Extend the exact built-in registry test with the OpenRouter limit and queue.
|
|
||||||
2. Add one coherent table covering unlimited omission, default queue,
|
|
||||||
explicit-zero queue, negative values, queue-without-limit, and total
|
|
||||||
overflow.
|
|
||||||
3. Extend the registry copy/normalization test to prove returned policy maps
|
|
||||||
cannot mutate registry state.
|
|
||||||
4. Add root coverage only if needed to prove the public pointer/presence
|
|
||||||
conversion; do not reproduce registry validation cases at the facade.
|
|
||||||
|
|
||||||
### Focused Validation
|
|
||||||
|
|
||||||
Run:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
gofmt -w backends.go internal/domain/domain.go \
|
|
||||||
internal/backend/registry.go internal/backend/registry_test.go
|
|
||||||
go test . ./internal/backend
|
|
||||||
go vet . ./internal/backend
|
|
||||||
git diff --check
|
|
||||||
```
|
|
||||||
|
|
||||||
Include another touched Go test file in `gofmt` only if it actually changed.
|
|
||||||
|
|
||||||
### Completion Gate
|
|
||||||
|
|
||||||
This stage is complete when every public configuration state has one normalized
|
|
||||||
internal meaning, OpenRouter exposes exactly 16/1024, custom backends remain
|
|
||||||
unlimited by omission, and no runtime call is scheduled yet.
|
|
||||||
|
|
||||||
## Stage 2 — Engine-Local Capacity Manager
|
|
||||||
|
|
||||||
**Status:** Complete.
|
|
||||||
|
|
||||||
### Goal
|
|
||||||
|
|
||||||
Implement and prove the bounded admission mechanism and FIFO active-generation
|
|
||||||
client wrapper independently of runner orchestration.
|
|
||||||
|
|
||||||
### Work
|
|
||||||
|
|
||||||
1. Add `internal/capacity/manager.go` with the manager, immutable policy copy,
|
|
||||||
per-backend pools, internal error, immediate admission, idempotent release,
|
|
||||||
and FIFO context-aware active permits.
|
|
||||||
2. Add `internal/capacity/client.go` with the transparent `llm.Client` wrapper.
|
|
||||||
3. Use mutex-protected waiter state and an ordered list; explicitly resolve
|
|
||||||
grant-versus-cancel races.
|
|
||||||
4. Ensure unlimited and independent-pool fast paths avoid queue allocation.
|
|
||||||
5. Do not start workers, timers, cleanup goroutines, or process-global state.
|
|
||||||
|
|
||||||
### Tests
|
|
||||||
|
|
||||||
1. Add a compact constructor-validation table for blank IDs, non-positive
|
|
||||||
limits, negative queues, and total-capacity overflow.
|
|
||||||
2. With a small configured policy, prove that exactly
|
|
||||||
`limit + queueCapacity` admissions succeed, the next matches
|
|
||||||
`ErrCapacityExceeded`, and a release permits another admission.
|
|
||||||
3. Prove release is idempotent.
|
|
||||||
4. Drive more blocked client calls than the active limit and assert observed
|
|
||||||
peak concurrency never exceeds that limit.
|
|
||||||
5. Prove FIFO order with deterministic queue-entry synchronization.
|
|
||||||
6. Cancel the first and a middle waiter and prove they are removed, never call
|
|
||||||
the wrapped client, and do not block later waiters.
|
|
||||||
7. Exercise the grant/cancel race repeatedly under `go test -race`, asserting
|
|
||||||
no permit leak or double invocation.
|
|
||||||
8. Prove different backend IDs proceed independently and blank, unknown, or
|
|
||||||
nil-manager paths remain unlimited.
|
|
||||||
9. Prove request values, successful and nil responses, and collaborator errors
|
|
||||||
pass through unchanged after permit acquisition.
|
|
||||||
|
|
||||||
### Focused Validation
|
|
||||||
|
|
||||||
Run:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
gofmt -w internal/capacity/manager.go \
|
|
||||||
internal/capacity/manager_test.go \
|
|
||||||
internal/capacity/client.go \
|
|
||||||
internal/capacity/client_test.go
|
|
||||||
go test ./internal/capacity
|
|
||||||
go test -race ./internal/capacity
|
|
||||||
go vet ./internal/capacity
|
|
||||||
git diff --check
|
|
||||||
```
|
|
||||||
|
|
||||||
If tests are combined into one file, omit the nonexistent file from `gofmt`.
|
|
||||||
|
|
||||||
### Completion Gate
|
|
||||||
|
|
||||||
This stage is complete when the standalone component enforces relational
|
|
||||||
admission and active limits, FIFO cancellation is race-safe, separate pools
|
|
||||||
are independent, and the wrapper is transparent apart from waiting.
|
|
||||||
|
|
||||||
## Stage 3 — Shared Preparation And Early Run Admission
|
|
||||||
|
|
||||||
**Status:** Complete.
|
|
||||||
|
|
||||||
### Goal
|
|
||||||
|
|
||||||
Refactor runner preparation into one shared two-phase pipeline and place
|
|
||||||
bounded admission after backend resolution but before schema, artifact, and
|
|
||||||
rendering work.
|
|
||||||
|
|
||||||
### Work
|
|
||||||
|
|
||||||
1. Add the private pre-admission preparation state and split the existing
|
|
||||||
`Prepare` logic according to the fixed design.
|
|
||||||
2. Make `Runner.Prepare` call both phases without an admitter.
|
|
||||||
3. Add the `RunAdmitter` interface and runner field.
|
|
||||||
4. Update `NewRunner` and `NewRunnerWithRepairer` to accept the optional
|
|
||||||
admitter; update internal call sites with nil until root assembly is wired.
|
|
||||||
5. Change `Runner.Run` to use the first phase, admit by effective backend ID,
|
|
||||||
defer the returned release, and then use the second phase.
|
|
||||||
6. Preserve all existing error precedence, target resolution, hashes,
|
|
||||||
metadata, session behavior, and timing.
|
|
||||||
7. Return capacity and context errors with the fixed identities. Do not invoke
|
|
||||||
later collaborators after rejection.
|
|
||||||
|
|
||||||
### Tests
|
|
||||||
|
|
||||||
1. Keep the existing `Run`/`Prepare` parity coverage passing to prove the
|
|
||||||
shared phases do not drift.
|
|
||||||
2. Add a fake admitter that records backend IDs and release calls.
|
|
||||||
3. Prove a backend-selected run admits with the selected ID even when the
|
|
||||||
endpoint is overridden.
|
|
||||||
4. Prove an endpoint-only run uses the unlimited/blank identity and that
|
|
||||||
`Prepare` never calls admission.
|
|
||||||
5. Reject admission and assert schema, artifact, renderer, validator, repairer,
|
|
||||||
and LLM collaborators are not invoked.
|
|
||||||
6. Prove admission is released after one successful run and representative
|
|
||||||
second-phase, generation, and validation errors. Prefer a small table around
|
|
||||||
the single `defer` invariant rather than duplicating every error test.
|
|
||||||
7. Retain direct-session, backend precedence, credential, hashing, and repair
|
|
||||||
tests unchanged except for constructor arguments.
|
|
||||||
|
|
||||||
### Focused Validation
|
|
||||||
|
|
||||||
Run:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
gofmt -w internal/usecase/runner.go \
|
|
||||||
internal/usecase/runner_test.go
|
|
||||||
go test ./internal/usecase
|
|
||||||
go test -race ./internal/usecase
|
|
||||||
go vet ./internal/usecase
|
|
||||||
git diff --check
|
|
||||||
```
|
|
||||||
|
|
||||||
### Completion Gate
|
|
||||||
|
|
||||||
This stage is complete when `Prepare` remains unrestricted, `Run` admits after
|
|
||||||
one canonical routing phase and before expensive completion work, every exit
|
|
||||||
releases admission, and existing preparation semantics remain unchanged.
|
|
||||||
|
|
||||||
## Stage 4 — Engine Assembly And Public Runtime Contract
|
|
||||||
|
|
||||||
**Status:** Complete.
|
|
||||||
|
|
||||||
### Goal
|
|
||||||
|
|
||||||
Wire one manager into each engine, schedule built-in and injected clients,
|
|
||||||
expose the capacity error, and prove assembled runtime behavior.
|
|
||||||
|
|
||||||
### Work
|
|
||||||
|
|
||||||
1. Add public `ErrCapacityExceeded` and its exact GoDoc in `engine.go`.
|
|
||||||
2. Map internal capacity exhaustion in `errors.go`.
|
|
||||||
3. Construct the manager from the registry policy snapshot in `NewEngine`.
|
|
||||||
4. Wrap the selected internal client after built-in or injected-client
|
|
||||||
selection and pass the manager and wrapped client to the runner.
|
|
||||||
5. Update `Engine`, `NewEngine`, `Run`, `WithLLMClient`, and `LLMClient` GoDoc
|
|
||||||
only where concurrency, capacity, or cancellation statements change.
|
|
||||||
6. Ensure manager-construction errors match `ErrInvalidConfig`.
|
|
||||||
7. For internal repair coverage, construct the default repairer with the same
|
|
||||||
wrapped client used by its runner and confirm repair target backend identity
|
|
||||||
remains intact.
|
|
||||||
|
|
||||||
### Tests
|
|
||||||
|
|
||||||
1. Add an external-package assembled test with a small custom limit and a
|
|
||||||
blocking injected client; assert peak generation equals or remains below
|
|
||||||
the configured limit.
|
|
||||||
2. With queue capacity zero, block one accepted run before generation and
|
|
||||||
assert the next matching-backend run returns `ErrCapacityExceeded`, does not
|
|
||||||
match `ErrInvalidRequest` or `ErrLLMGenerate`, returns no result, and never
|
|
||||||
reaches expensive collaborators or the client.
|
|
||||||
3. In the same or another focused workflow, prove an endpoint override remains
|
|
||||||
in the selected backend's pool.
|
|
||||||
4. Prove two engines with the same backend ID have independent capacity.
|
|
||||||
5. Prove an unlimited custom backend and an endpoint-only profile preserve
|
|
||||||
concurrent behavior.
|
|
||||||
6. Cancel a call waiting for an active permit; assert it matches both
|
|
||||||
`context.Canceled` and `ErrLLMGenerate`, never invokes the injected client,
|
|
||||||
and leaves capacity reusable.
|
|
||||||
7. Add one internal repair workflow with concurrent runs or controlled permits
|
|
||||||
showing initial and repair generations never exceed the same backend limit
|
|
||||||
and repairs do not perform a second admission.
|
|
||||||
8. Extend the public error sentinel contract test with
|
|
||||||
`ErrCapacityExceeded`.
|
|
||||||
|
|
||||||
Avoid a second HTTP-level concurrency suite: the capacity client tests and one
|
|
||||||
assembled injected-client workflow already protect the shared wrapper used by
|
|
||||||
the built-in client.
|
|
||||||
|
|
||||||
### Focused Validation
|
|
||||||
|
|
||||||
Run:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
gofmt -w engine.go errors.go backends.go \
|
|
||||||
internal/usecase/runner.go internal/usecase/runner_test.go \
|
|
||||||
engine_test.go public_contract_test.go
|
|
||||||
go test . ./internal/backend ./internal/capacity ./internal/usecase
|
|
||||||
go test -race . ./internal/capacity ./internal/usecase
|
|
||||||
go vet . ./internal/backend ./internal/capacity ./internal/usecase
|
|
||||||
git diff --check
|
|
||||||
```
|
|
||||||
|
|
||||||
Add any newly created capacity files to `gofmt` when they changed in this
|
|
||||||
stage.
|
|
||||||
|
|
||||||
### Completion Gate
|
|
||||||
|
|
||||||
This stage is complete when every engine has independent pools, limited runs
|
|
||||||
are bounded and FIFO at generation, endpoint routing is correct, capacity and
|
|
||||||
context errors are stable, repairs reuse admission, and both client kinds pass
|
|
||||||
through the same wrapper.
|
|
||||||
|
|
||||||
## Stage 5 — Durable Documentation And Final Validation
|
|
||||||
|
|
||||||
**Status:** Complete.
|
|
||||||
|
|
||||||
### Goal
|
|
||||||
|
|
||||||
Move implemented contracts into their durable owners, record compatibility
|
|
||||||
impact, and validate the complete repository.
|
|
||||||
|
|
||||||
### Work
|
|
||||||
|
|
||||||
1. Review every changed exported declaration. Ensure GoDoc is the canonical
|
|
||||||
owner of exact field types, nil/zero semantics, defaulting, error identity,
|
|
||||||
engine scope, concurrency safety, cancellation, and source compatibility.
|
|
||||||
2. Update `doc.go` so its concurrency summary acknowledges backend scheduling
|
|
||||||
while continuing to require injected collaborators to be concurrency-safe.
|
|
||||||
3. Update `docs/consumers/pkg-promptkit.md` with task-oriented examples for:
|
|
||||||
- a limited local backend;
|
|
||||||
- omitted queue capacity selecting 1024;
|
|
||||||
- explicit zero queue capacity; and
|
|
||||||
- handling `ErrCapacityExceeded`.
|
|
||||||
Keep exact field semantics in GoDoc rather than duplicating a full table.
|
|
||||||
4. Add `docs/internal/capacity.md` as the durable owner of pool lifecycle,
|
|
||||||
admission, FIFO active permits, cancellation, client wrapping, and test
|
|
||||||
ownership.
|
|
||||||
5. Add `internal/capacity` to `docs/internal/overview.md`.
|
|
||||||
6. Update `docs/policy/architecture.md` to include the implemented component
|
|
||||||
and root assembly dependency without turning policy into an API reference.
|
|
||||||
7. Update `docs/internal/runner.md` to describe the shared two-phase
|
|
||||||
preparation pipeline, early bounded admission, lease lifetime, generation
|
|
||||||
permits, repairs, capacity failures, and cancellation.
|
|
||||||
8. Review `docs/formats.md`; add only a concise link or clarification if needed
|
|
||||||
to explain that endpoint overrides preserve backend capacity identity.
|
|
||||||
Do not add concurrency fields to YAML.
|
|
||||||
9. Do not change the OpenAI-compatible integration contract or
|
|
||||||
`docs/internal/llm.md` unless implementation changes their current
|
|
||||||
statements; scheduling is outside the provider wire contract and concrete
|
|
||||||
model-client implementation.
|
|
||||||
10. Record in the implementation handoff that built-in OpenRouter now limits
|
|
||||||
active generations to 16 with queue capacity 1024 and that the release
|
|
||||||
must be a pre-`v1` minor release. Do not edit the release procedure or
|
|
||||||
create a tag.
|
|
||||||
11. After every check passes, set `concurrency.md`, this implementation plan,
|
|
||||||
and each stage status to `Complete`. Do not remove the roadmaps in the
|
|
||||||
implementation change; lifecycle retirement follows review.
|
|
||||||
|
|
||||||
### Full Validation
|
|
||||||
|
|
||||||
Run the complete sequence from `docs/development.md`:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
go test ./...
|
|
||||||
go test -race ./...
|
|
||||||
go vet ./...
|
|
||||||
go build ./...
|
|
||||||
go run ./examples/go-library/prepare
|
|
||||||
gofmt -l $(git ls-files '*.go')
|
|
||||||
git diff --check
|
|
||||||
```
|
|
||||||
|
|
||||||
The formatting command must produce no paths. Follow every added or changed
|
|
||||||
Markdown link and confirm its target and heading exist.
|
|
||||||
|
|
||||||
Also inspect:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
git status --short
|
|
||||||
git diff --stat
|
|
||||||
git diff
|
|
||||||
```
|
|
||||||
|
|
||||||
Confirm that:
|
|
||||||
|
|
||||||
- only intended backend, capacity, runner, facade, test, documentation, and
|
|
||||||
roadmap files changed;
|
|
||||||
- no `go.work`, `go.work.sum`, local module replacement, credential,
|
|
||||||
generated binary, coverage output, or unrelated change was introduced;
|
|
||||||
- the built-in OpenRouter policy is exactly 16/1024;
|
|
||||||
- custom and endpoint-only backends remain unlimited by omission;
|
|
||||||
- explicit queue zero is distinguishable from omission;
|
|
||||||
- no capacity value enters execution targets, generated requests, stable JSON,
|
|
||||||
prompt/profile YAML, or provider payloads;
|
|
||||||
- every engine owns distinct pools with no package-global mutable state;
|
|
||||||
- every initial and repair generation uses the active permit wrapper;
|
|
||||||
- capacity and waiter state is released on success, error, panic unwinding,
|
|
||||||
and cancellation;
|
|
||||||
- concurrency tests use deterministic coordination rather than sleeps;
|
|
||||||
- current-state documentation describes implemented behavior rather than
|
|
||||||
referring readers to the roadmaps; and
|
|
||||||
- the feature and implementation roadmaps contain no unresolved work marked
|
|
||||||
complete.
|
|
||||||
|
|
||||||
### Completion Gate
|
|
||||||
|
|
||||||
The implementation is complete only when every target-end-state item in
|
|
||||||
`concurrency.md` is implemented, race-enabled tests demonstrate the configured
|
|
||||||
limits and cancellation safety, durable contracts no longer depend on roadmap
|
|
||||||
prose, and the OpenRouter compatibility change is clearly reported for the
|
|
||||||
next minor release.
|
|
||||||
|
|
||||||
## Implementation Handoff
|
|
||||||
|
|
||||||
Backend-specific capacity management is implemented and has passed the complete
|
|
||||||
repository validation sequence. The built-in OpenRouter backend now permits 16
|
|
||||||
active generations and a waiting capacity of 1024. Custom backends remain
|
|
||||||
unlimited when their limit is omitted, and endpoint-only profiles remain
|
|
||||||
unlimited.
|
|
||||||
|
|
||||||
Publishing this behavior requires a pre-`v1` minor release. Its release notes
|
|
||||||
must identify that unusually high concurrent OpenRouter use can now wait or
|
|
||||||
return `ErrCapacityExceeded`. This implementation does not change a module
|
|
||||||
version or create a tag.
|
|
||||||
|
|
||||||
## Open Questions
|
|
||||||
|
|
||||||
None. The feature roadmap and this plan fix the public representation,
|
|
||||||
registry defaults, admission bound, FIFO generation behavior, early-routing
|
|
||||||
refactor, cancellation races, error identities, engine and repair lifetimes,
|
|
||||||
test ownership, compatibility treatment, and non-goals required for
|
|
||||||
implementation.
|
|
||||||
322
engine.go
322
engine.go
@@ -63,9 +63,10 @@ var (
|
|||||||
// ErrPromptRender identifies a failure to render prompt messages or the
|
// ErrPromptRender identifies a failure to render prompt messages or the
|
||||||
// session ID from the resolved inputs and variables.
|
// session ID from the resolved inputs and variables.
|
||||||
ErrPromptRender = errors.New("failed to render prompt")
|
ErrPromptRender = errors.New("failed to render prompt")
|
||||||
// ErrCapacityExceeded identifies a Run rejected because the selected backend
|
// ErrCapacityExceeded identifies a Run or RunPrepared rejected because the
|
||||||
// already admitted ConcurrencyLimit + QueueCapacity calls. It is not an
|
// selected backend already admitted ConcurrencyLimit + QueueCapacity calls.
|
||||||
// invalid request, an LLM or provider rate-limit response, or ErrLLMGenerate.
|
// A [CapacityError] reports the selected backend ID. It is not an invalid
|
||||||
|
// request, an LLM or provider rate-limit response, or ErrLLMGenerate.
|
||||||
ErrCapacityExceeded = errors.New("backend capacity exceeded")
|
ErrCapacityExceeded = errors.New("backend capacity exceeded")
|
||||||
// ErrLLMGenerate identifies a model-client failure or a nil successful
|
// ErrLLMGenerate identifies a model-client failure or a nil successful
|
||||||
// response. Errors returned by an injected LLMClient remain available
|
// response. Errors returned by an injected LLMClient remain available
|
||||||
@@ -77,12 +78,15 @@ var (
|
|||||||
ErrValidation = errors.New("failed to validate output")
|
ErrValidation = errors.New("failed to validate output")
|
||||||
)
|
)
|
||||||
|
|
||||||
// Engine prepares and runs Promptkit prompt requests.
|
// Engine inspects prompts and profiles and prepares and runs Promptkit prompt
|
||||||
|
// requests.
|
||||||
//
|
//
|
||||||
// An Engine is safe for concurrent calls to [Engine.Prepare] and [Engine.Run].
|
// An Engine is safe for concurrent calls to [Engine.InspectPrompt],
|
||||||
// Each Engine owns independent backend-capacity pools that coordinate Run
|
// [Engine.InspectProfile], [Engine.Prepare], [Engine.PrepareExecution],
|
||||||
// admission and model generation. Injected collaborators may still be invoked
|
// [Engine.Run], and [Engine.RunPrepared]. Each Engine owns independent
|
||||||
// concurrently across different backend pools or for unlimited backends.
|
// backend-capacity pools that coordinate Run and RunPrepared admission and
|
||||||
|
// model generation. Injected collaborators may still be invoked concurrently
|
||||||
|
// across different backend pools or for unlimited backends.
|
||||||
type Engine struct {
|
type Engine struct {
|
||||||
runner *usecase.Runner
|
runner *usecase.Runner
|
||||||
}
|
}
|
||||||
@@ -94,9 +98,10 @@ type Config struct {
|
|||||||
// It is required unless a WithPromptFS or WithPromptFile option supplies the
|
// It is required unless a WithPromptFS or WithPromptFile option supplies the
|
||||||
// prompt source.
|
// prompt source.
|
||||||
PromptDir string
|
PromptDir string
|
||||||
// ProfileDir is an optional directory whose profiles take precedence over
|
// ProfileDir is an optional ordinary configured source whose profiles take
|
||||||
// embedded built-in profiles. An empty value selects only built-ins unless
|
// precedence over application fallback and embedded built-in profiles. An
|
||||||
// profile options are also supplied.
|
// empty value selects the lower-precedence sources unless a profile-source
|
||||||
|
// option supplies the ordinary source.
|
||||||
ProfileDir string
|
ProfileDir string
|
||||||
// SchemaDir is the root for JSON Schema files. An empty value uses the
|
// SchemaDir is the root for JSON Schema files. An empty value uses the
|
||||||
// current directory. WithSchemaFS or WithSchemaFile replaces this source.
|
// current directory. WithSchemaFS or WithSchemaFile replaces this source.
|
||||||
@@ -115,12 +120,12 @@ type Config struct {
|
|||||||
// Option customizes engine construction.
|
// Option customizes engine construction.
|
||||||
//
|
//
|
||||||
// NewEngine applies options in argument order and ignores nil options. Within
|
// NewEngine applies options in argument order and ignores nil options. Within
|
||||||
// each prompt-source, profile-source, in-memory-profile, schema-source,
|
// each prompt-source, ordinary-profile-source, fallback-profile-source,
|
||||||
// model-client, and artifact-reader category, the last non-nil valid option
|
// in-memory-profile, schema-source, model-client, and artifact-reader
|
||||||
// replaces earlier options in that category. WithBackend is the additive
|
// category, the last non-nil valid option replaces earlier options in that
|
||||||
// exception: unique registrations accumulate, and a repeated backend ID is an
|
// category. WithBackend is the additive exception: unique registrations
|
||||||
// error rather than a replacement. An invalid option fails construction even
|
// accumulate, and a repeated backend ID is an error rather than a replacement.
|
||||||
// if a later option would replace it.
|
// An invalid option fails construction even if a later option would replace it.
|
||||||
type Option interface {
|
type Option interface {
|
||||||
apply(*engineOptions) error
|
apply(*engineOptions) error
|
||||||
}
|
}
|
||||||
@@ -132,26 +137,30 @@ func (f optionFunc) apply(options *engineOptions) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type engineOptions struct {
|
type engineOptions struct {
|
||||||
llmClient llm.Client
|
llmClient llm.Client
|
||||||
artifactReader artifactadapter.Reader
|
artifactReader artifactadapter.Reader
|
||||||
promptDefs promptdef.Repository
|
promptDefs promptdef.Repository
|
||||||
profiles profile.Repository
|
profiles profile.Repository
|
||||||
memoryProfiles profile.Repository
|
fallbackProfiles profile.Repository
|
||||||
backends []domain.Backend
|
memoryProfiles profile.Repository
|
||||||
validator validate.Validator
|
backends []domain.Backend
|
||||||
promptSource bool
|
validator validate.Validator
|
||||||
profileSource bool
|
promptSource bool
|
||||||
memorySource bool
|
profileSource bool
|
||||||
validatorSource bool
|
fallbackProfileSource bool
|
||||||
artifactSource bool
|
memorySource bool
|
||||||
|
validatorSource bool
|
||||||
|
artifactSource bool
|
||||||
}
|
}
|
||||||
|
|
||||||
// WithLLMClient replaces the built-in model client used by [Engine.Run].
|
// WithLLMClient replaces the built-in model client used by [Engine.Run] and
|
||||||
|
// [Engine.RunPrepared].
|
||||||
//
|
//
|
||||||
// A nil client makes NewEngine fail with ErrInvalidConfig. The Engine schedules
|
// A nil client makes NewEngine fail with ErrInvalidConfig. The Engine schedules
|
||||||
// Generate calls according to the selected backend's capacity policy, but the
|
// Generate calls according to the selected backend's capacity policy, but the
|
||||||
// client may still be called concurrently across different backend pools or for
|
// client may still be called concurrently across different backend pools or for
|
||||||
// unlimited backends. The client is not used by [Engine.Prepare].
|
// unlimited backends. The client is not used by [Engine.Prepare] or
|
||||||
|
// [Engine.PrepareExecution].
|
||||||
func WithLLMClient(client LLMClient) Option {
|
func WithLLMClient(client LLMClient) Option {
|
||||||
return optionFunc(func(options *engineOptions) error {
|
return optionFunc(func(options *engineOptions) error {
|
||||||
if client == nil {
|
if client == nil {
|
||||||
@@ -218,12 +227,12 @@ func WithPromptFile(path string) Option {
|
|||||||
|
|
||||||
// WithProfileFS loads execution profiles from fsys under root.
|
// WithProfileFS loads execution profiles from fsys under root.
|
||||||
//
|
//
|
||||||
// Profiles from this source overlay built-in profiles. Profile YAML must use
|
// Profiles from this ordinary configured source take precedence over
|
||||||
// api_key_env for environment-based credentials; raw API keys are rejected.
|
// application fallback and built-in profiles. Profile YAML must use api_key_env
|
||||||
// fsys must be non-nil and root must be non-empty; otherwise NewEngine fails
|
// for environment-based credentials; raw API keys are rejected. fsys must be
|
||||||
// with ErrInvalidConfig. This option replaces Config.ProfileDir and earlier
|
// non-nil and root must be non-empty; otherwise NewEngine fails with
|
||||||
// file or FS profile-source options, but remains below WithProfiles in
|
// ErrInvalidConfig. This option replaces Config.ProfileDir and earlier file or
|
||||||
// precedence.
|
// FS profile-source options, but remains below WithProfiles in precedence.
|
||||||
func WithProfileFS(fsys fs.FS, root string) Option {
|
func WithProfileFS(fsys fs.FS, root string) Option {
|
||||||
return optionFunc(func(options *engineOptions) error {
|
return optionFunc(func(options *engineOptions) error {
|
||||||
if fsys == nil {
|
if fsys == nil {
|
||||||
@@ -240,11 +249,12 @@ func WithProfileFS(fsys fs.FS, root string) Option {
|
|||||||
|
|
||||||
// WithProfileFile loads execution profiles from the single profile file at path.
|
// WithProfileFile loads execution profiles from the single profile file at path.
|
||||||
//
|
//
|
||||||
// The profile overlays built-in profiles. Profile YAML must use api_key_env for
|
// The profile takes precedence over application fallback and built-in profiles.
|
||||||
// environment-based credentials; raw API keys are rejected. path must name an
|
// Profile YAML must use api_key_env for environment-based credentials; raw API
|
||||||
// existing non-directory file when NewEngine applies the option. This option
|
// keys are rejected. path must name an existing non-directory file when
|
||||||
// replaces Config.ProfileDir and earlier file or FS profile-source options,
|
// NewEngine applies the option. This option replaces Config.ProfileDir and
|
||||||
// but remains below WithProfiles in precedence.
|
// earlier file or FS profile-source options, but remains below WithProfiles in
|
||||||
|
// precedence.
|
||||||
func WithProfileFile(path string) Option {
|
func WithProfileFile(path string) Option {
|
||||||
return optionFunc(func(options *engineOptions) error {
|
return optionFunc(func(options *engineOptions) error {
|
||||||
fsys, root, err := fileSource(path)
|
fsys, root, err := fileSource(path)
|
||||||
@@ -257,8 +267,41 @@ func WithProfileFile(path string) Option {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// WithFallbackProfileFS supplies application-owned fallback profile
|
||||||
|
// definitions from fsys under root.
|
||||||
|
//
|
||||||
|
// Profile lookup checks, in order, profiles supplied by WithProfiles; the
|
||||||
|
// ordinary configured source selected by WithProfileFile, WithProfileFS, or
|
||||||
|
// Config.ProfileDir; this fallback source; and Promptkit's embedded built-in
|
||||||
|
// profiles. Each source supplies a complete profile definition; profile fields
|
||||||
|
// are not merged between sources. Only an absent profile ID proceeds to the
|
||||||
|
// next source. A matching read, parse, duplicate, validation, or credential
|
||||||
|
// format failure stops resolution.
|
||||||
|
//
|
||||||
|
// Files use the ordinary strict profile YAML and api_key_env credential rules.
|
||||||
|
// Loading and validation are lazy: NewEngine validates this option's arguments
|
||||||
|
// but does not read profile files. fsys must be non-nil and root must be
|
||||||
|
// nonblank; otherwise NewEngine returns an error matching ErrInvalidConfig.
|
||||||
|
// Repeating this option replaces the earlier valid fallback source.
|
||||||
|
//
|
||||||
|
// This option controls profile-definition lookup, not provider or generation
|
||||||
|
// failover.
|
||||||
|
func WithFallbackProfileFS(fsys fs.FS, root string) Option {
|
||||||
|
return optionFunc(func(options *engineOptions) error {
|
||||||
|
if fsys == nil {
|
||||||
|
return ErrInvalidConfig
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(root) == "" {
|
||||||
|
return ErrInvalidConfig
|
||||||
|
}
|
||||||
|
options.fallbackProfiles = profile.NewFSRepository(fsys, root)
|
||||||
|
options.fallbackProfileSource = true
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
// WithProfiles configures in-memory profiles that take precedence over
|
// WithProfiles configures in-memory profiles that take precedence over
|
||||||
// configured profile files and built-in profiles.
|
// ordinary configured, application fallback, and built-in profiles.
|
||||||
//
|
//
|
||||||
// NewEngine validates and copies every profile. IDs must be unique within one
|
// NewEngine validates and copies every profile. IDs must be unique within one
|
||||||
// call. An invalid profile, duplicate ID, or unsupported ExtraParams value
|
// call. An invalid profile, duplicate ID, or unsupported ExtraParams value
|
||||||
@@ -344,13 +387,7 @@ func NewEngine(cfg Config, opts ...Option) (*Engine, error) {
|
|||||||
promptDefs = promptdef.NewFilesystemRepository(cfg.PromptDir)
|
promptDefs = promptdef.NewFilesystemRepository(cfg.PromptDir)
|
||||||
}
|
}
|
||||||
|
|
||||||
profiles := builtin.NewRepositoryWithDirectory(cfg.ProfileDir)
|
profiles := newProfileRepository(cfg.ProfileDir, options)
|
||||||
if options.profileSource {
|
|
||||||
profiles = builtin.NewRepositoryWithPrimary(options.profiles)
|
|
||||||
}
|
|
||||||
if options.memorySource {
|
|
||||||
profiles = profile.NewOverlayRepository(options.memoryProfiles, profiles)
|
|
||||||
}
|
|
||||||
|
|
||||||
backendRegistry, err := backend.NewRegistry(options.backends)
|
backendRegistry, err := backend.NewRegistry(options.backends)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -403,6 +440,26 @@ func NewEngine(cfg Config, opts ...Option) (*Engine, error) {
|
|||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func newProfileRepository(profileDir string, options engineOptions) profile.Repository {
|
||||||
|
repository := builtin.NewRepository()
|
||||||
|
|
||||||
|
if options.fallbackProfileSource {
|
||||||
|
repository = profile.NewOverlayRepository(options.fallbackProfiles, repository)
|
||||||
|
}
|
||||||
|
|
||||||
|
if options.profileSource {
|
||||||
|
repository = profile.NewOverlayRepository(options.profiles, repository)
|
||||||
|
} else if strings.TrimSpace(profileDir) != "" {
|
||||||
|
repository = profile.NewOverlayRepository(profile.NewFilesystemRepository(profileDir), repository)
|
||||||
|
}
|
||||||
|
|
||||||
|
if options.memorySource {
|
||||||
|
repository = profile.NewOverlayRepository(options.memoryProfiles, repository)
|
||||||
|
}
|
||||||
|
|
||||||
|
return repository
|
||||||
|
}
|
||||||
|
|
||||||
func fileSource(name string) (fs.FS, string, error) {
|
func fileSource(name string) (fs.FS, string, error) {
|
||||||
cleanName := strings.TrimSpace(name)
|
cleanName := strings.TrimSpace(name)
|
||||||
if cleanName == "" {
|
if cleanName == "" {
|
||||||
@@ -423,6 +480,93 @@ func fileSource(name string) (fs.FS, string, error) {
|
|||||||
return os.DirFS(dir), filepath.ToSlash(base), nil
|
return os.DirFS(dir), filepath.ToSlash(base), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// InspectPrompt resolves one explicit prompt definition without selecting a
|
||||||
|
// profile or starting execution work.
|
||||||
|
//
|
||||||
|
// InspectPrompt requires a nonblank promptID. It passes nonblank promptID and
|
||||||
|
// promptVersion values unchanged to the engine's ordinary, case-sensitive
|
||||||
|
// prompt selection. An empty version succeeds only when that source has one
|
||||||
|
// selected ID; a nonempty version selects one exact ID/version pair. The
|
||||||
|
// configured prompt source is used without merging, fallback, or enumeration.
|
||||||
|
//
|
||||||
|
// A successful result proves that the selected definition and any referenced
|
||||||
|
// message content files were structurally loaded. Inputs are returned in
|
||||||
|
// definition order. DefaultProfileID is declared metadata only and is not
|
||||||
|
// resolved. OutputContract is the normalized declared contract, with a JSON
|
||||||
|
// Schema path when declared but without loading or compiling that schema.
|
||||||
|
// PromptHash is the same opaque equality value as PreparedRun.PromptHash for
|
||||||
|
// the selected definition and observed source state; its spelling, length,
|
||||||
|
// encoding, algorithm, and security properties are not contracts.
|
||||||
|
//
|
||||||
|
// This method does not return prompt bodies, templates, source paths, schemas,
|
||||||
|
// rendered messages, or execution settings. It does not resolve a profile or
|
||||||
|
// credential, read artifacts or schemas, render, validate, admit capacity,
|
||||||
|
// contact a provider, or generate model output. The returned PromptInspection
|
||||||
|
// and its input slice are caller-owned. Filesystem-backed inspection is a
|
||||||
|
// point-in-time lookup and does not freeze a definition for later execution.
|
||||||
|
//
|
||||||
|
// A nil Engine returns an error matching ErrInvalidConfig. A blank prompt ID
|
||||||
|
// matches ErrInvalidRequest. An absent exact ID or version matches
|
||||||
|
// ErrPromptNotFound and not ErrPromptLoad. Malformed, unreadable, duplicate,
|
||||||
|
// ambiguous, referenced-content, or hashing failures match ErrPromptLoad.
|
||||||
|
// Cancellation during lookup matches ErrPromptLoad while preserving the
|
||||||
|
// context error. InspectPrompt returns no partial result on error.
|
||||||
|
func (e *Engine) InspectPrompt(
|
||||||
|
ctx context.Context,
|
||||||
|
promptID string,
|
||||||
|
promptVersion string,
|
||||||
|
) (*PromptInspection, error) {
|
||||||
|
if e == nil || e.runner == nil {
|
||||||
|
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
|
||||||
|
}
|
||||||
|
|
||||||
|
inspection, err := e.runner.InspectPrompt(ctx, promptID, promptVersion)
|
||||||
|
if err != nil {
|
||||||
|
return nil, mapPublicError(err)
|
||||||
|
}
|
||||||
|
return fromDomainPromptInspection(inspection), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// InspectProfile resolves one explicit profile without selecting a prompt or
|
||||||
|
// starting execution work.
|
||||||
|
//
|
||||||
|
// InspectProfile trims surrounding whitespace from profileID and looks up the
|
||||||
|
// resulting nonblank ID exactly and case-sensitively through the engine's
|
||||||
|
// in-memory, ordinary configured-source, application fallback, and built-in
|
||||||
|
// profile precedence. It applies the framework timeout baseline, selected
|
||||||
|
// backend, and then selected profile to EffectiveModelParams without a request
|
||||||
|
// override. BackendID is empty for an endpoint-only profile.
|
||||||
|
//
|
||||||
|
// APIKeyEnv in the returned target is an environment-variable name, never its
|
||||||
|
// value. APIKeyRequired instead reports a direct credential requirement and is
|
||||||
|
// mutually exclusive with a nonblank APIKeyEnv. InspectProfile neither derives
|
||||||
|
// an ID from a prompt default_profile nor checks credential availability, so an
|
||||||
|
// absent or blank named environment variable is not an error.
|
||||||
|
//
|
||||||
|
// The returned ProfileInspection and all nested mutable values are
|
||||||
|
// caller-owned. Filesystem-backed inspection is a point-in-time lookup and
|
||||||
|
// does not freeze the profile for a later execution. This method does not load
|
||||||
|
// a prompt, render, read artifacts or schemas, admit backend capacity, contact
|
||||||
|
// a provider, or generate model output.
|
||||||
|
//
|
||||||
|
// A nil Engine returns an error matching ErrInvalidConfig. A blank profile ID
|
||||||
|
// matches ErrInvalidRequest. An absent exact ID matches ErrProfileNotFound and
|
||||||
|
// not ErrProfileLoad. Malformed or unreadable profile data, an unknown backend,
|
||||||
|
// or an invalid resolved target matches ErrProfileLoad. Cancellation during
|
||||||
|
// profile loading matches ErrProfileLoad while preserving the context error.
|
||||||
|
// InspectProfile returns no partial result on error.
|
||||||
|
func (e *Engine) InspectProfile(ctx context.Context, profileID string) (*ProfileInspection, error) {
|
||||||
|
if e == nil || e.runner == nil {
|
||||||
|
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
|
||||||
|
}
|
||||||
|
|
||||||
|
inspection, err := e.runner.InspectProfile(ctx, profileID)
|
||||||
|
if err != nil {
|
||||||
|
return nil, mapPublicError(err)
|
||||||
|
}
|
||||||
|
return fromDomainProfileInspection(inspection), nil
|
||||||
|
}
|
||||||
|
|
||||||
// Prepare resolves and renders a prompt request without calling an LLM.
|
// Prepare resolves and renders a prompt request without calling an LLM.
|
||||||
//
|
//
|
||||||
// Prepare selects the prompt and profile, resolves any selected backend and
|
// Prepare selects the prompt and profile, resolves any selected backend and
|
||||||
@@ -457,6 +601,38 @@ func (e *Engine) Prepare(ctx context.Context, req RunRequest) (*PreparedRun, err
|
|||||||
return fromDomainPreparedRun(prepared), nil
|
return fromDomainPreparedRun(prepared), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// PrepareExecution completely prepares a prompt request without calling the
|
||||||
|
// configured LLMClient or reserving backend admission capacity.
|
||||||
|
//
|
||||||
|
// The returned opaque handle is bound to this Engine and permits one
|
||||||
|
// [Engine.RunPrepared] invocation. Preparation freezes the selected sources,
|
||||||
|
// rendered messages, effective settings, inputs, provider structured-output
|
||||||
|
// metadata, and validation resources needed by that invocation. The handle
|
||||||
|
// retains a direct RunRequest.APIKey only in private execution state;
|
||||||
|
// [PreparedExecution.Details] is credential-redacted.
|
||||||
|
//
|
||||||
|
// The context governs preparation only. Cancellation after this method
|
||||||
|
// returns does not invalidate the handle or propagate to RunPrepared.
|
||||||
|
// PrepareExecution returns the same error categories as [Engine.Prepare] and
|
||||||
|
// returns no handle on error. A nil Engine returns an error matching
|
||||||
|
// ErrInvalidConfig.
|
||||||
|
func (e *Engine) PrepareExecution(ctx context.Context, req RunRequest) (*PreparedExecution, error) {
|
||||||
|
if e == nil || e.runner == nil {
|
||||||
|
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
|
||||||
|
}
|
||||||
|
|
||||||
|
domainReq, err := toDomainRunRequest(req)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("%w: %v", ErrInvalidRequest, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
prepared, err := e.runner.PrepareExecution(ctx, domainReq)
|
||||||
|
if err != nil {
|
||||||
|
return nil, mapPublicError(err)
|
||||||
|
}
|
||||||
|
return &PreparedExecution{internal: prepared}, nil
|
||||||
|
}
|
||||||
|
|
||||||
// Run prepares a request, invokes the configured LLMClient, and validates the
|
// Run prepares a request, invokes the configured LLMClient, and validates the
|
||||||
// generated output.
|
// generated output.
|
||||||
//
|
//
|
||||||
@@ -467,9 +643,10 @@ func (e *Engine) Prepare(ctx context.Context, req RunRequest) (*PreparedRun, err
|
|||||||
// single-pass even when OutputContract.RepairAttempts is positive.
|
// single-pass even when OutputContract.RepairAttempts is positive.
|
||||||
//
|
//
|
||||||
// Run can return every error category documented by [Engine.Prepare], plus
|
// Run can return every error category documented by [Engine.Prepare], plus
|
||||||
// ErrCapacityExceeded and ErrLLMGenerate. ErrCapacityExceeded identifies
|
// ErrCapacityExceeded and ErrLLMGenerate. An engine admission rejection is
|
||||||
// rejection before artifacts, schemas, rendering, or model generation because
|
// discoverable as [CapacityError] and still matches ErrCapacityExceeded. It
|
||||||
// the selected backend's admission capacity is full; it does not match
|
// occurs before artifacts, schemas, rendering, or model generation because the
|
||||||
|
// selected backend's admission capacity is full; it does not match
|
||||||
// ErrInvalidRequest or ErrLLMGenerate. Errors from injected clients remain
|
// ErrInvalidRequest or ErrLLMGenerate. Errors from injected clients remain
|
||||||
// available through errors.Is. Cancellation while waiting for model-generation
|
// available through errors.Is. Cancellation while waiting for model-generation
|
||||||
// capacity matches both ErrLLMGenerate and the context error. Cancellation
|
// capacity matches both ErrLLMGenerate and the context error. Cancellation
|
||||||
@@ -491,3 +668,42 @@ func (e *Engine) Run(ctx context.Context, req RunRequest) (*RunResult, error) {
|
|||||||
}
|
}
|
||||||
return fromDomainRunResult(result), nil
|
return fromDomainRunResult(result), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// RunPrepared atomically claims and executes a handle created by
|
||||||
|
// [Engine.PrepareExecution].
|
||||||
|
//
|
||||||
|
// A valid owning-Engine invocation consumes the handle's one attempt before
|
||||||
|
// credential revalidation, backend admission, generation, or validation.
|
||||||
|
// Cancellation, capacity rejection, generation failure, operational
|
||||||
|
// validation failure, and success all leave the handle unusable. A nil,
|
||||||
|
// zero-value, foreign-Engine, discarded, claimed, or used handle returns an
|
||||||
|
// error matching ErrInvalidRequest; a nil Engine returns ErrInvalidConfig and
|
||||||
|
// does not claim the handle.
|
||||||
|
//
|
||||||
|
// The supplied context governs this execution attempt independently of the
|
||||||
|
// preparation context. It covers credential revalidation, admission,
|
||||||
|
// generation, validation, and any internal repair. Result timing begins after
|
||||||
|
// the claim and excludes preparation and consumer-held delay.
|
||||||
|
//
|
||||||
|
// RunPrepared can return ErrInvalidRequest, ErrAPIKeyEnvMissing,
|
||||||
|
// ErrCapacityExceeded, ErrLLMGenerate, or ErrValidation as applicable while
|
||||||
|
// preserving documented collaborator and context identities. An engine
|
||||||
|
// admission rejection is discoverable as [CapacityError] and still matches
|
||||||
|
// ErrCapacityExceeded. A completed content-validation rejection is returned
|
||||||
|
// in RunResult, not as an operational error. An operational error returns no
|
||||||
|
// partial RunResult.
|
||||||
|
func (e *Engine) RunPrepared(ctx context.Context, prepared *PreparedExecution) (*RunResult, error) {
|
||||||
|
if e == nil || e.runner == nil {
|
||||||
|
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
|
||||||
|
}
|
||||||
|
|
||||||
|
var internal *usecase.PreparedExecution
|
||||||
|
if prepared != nil {
|
||||||
|
internal = prepared.internal
|
||||||
|
}
|
||||||
|
result, err := e.runner.RunPrepared(ctx, internal)
|
||||||
|
if err != nil {
|
||||||
|
return nil, mapPublicError(err)
|
||||||
|
}
|
||||||
|
return fromDomainRunResult(result), nil
|
||||||
|
}
|
||||||
|
|||||||
@@ -360,14 +360,14 @@ func TestEngineExecutionSettingPrecedence(t *testing.T) {
|
|||||||
wantPresence promptkit.ExecutionTargetPresence
|
wantPresence promptkit.ExecutionTargetPresence
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
name: "framework defaults fill zero-valued profile settings",
|
name: "unspecified provider controls retain framework timeout",
|
||||||
profile: defaultsProfile,
|
profile: defaultsProfile,
|
||||||
want: promptkit.ExecutionTarget{
|
want: promptkit.ExecutionTarget{
|
||||||
Endpoint: defaultsProfile.endpoint,
|
Endpoint: defaultsProfile.endpoint,
|
||||||
Model: defaultsProfile.model,
|
Model: defaultsProfile.model,
|
||||||
Temperature: 0,
|
Temperature: 0,
|
||||||
MaxTokens: 0,
|
MaxTokens: 0,
|
||||||
TopP: 1,
|
TopP: 0,
|
||||||
TimeoutSeconds: 600,
|
TimeoutSeconds: 600,
|
||||||
ServiceTier: defaultsProfile.serviceTier,
|
ServiceTier: defaultsProfile.serviceTier,
|
||||||
ReasoningEffort: defaultsProfile.reasoningEffort,
|
ReasoningEffort: defaultsProfile.reasoningEffort,
|
||||||
@@ -2301,6 +2301,8 @@ func TestSourceOptionsRejectInvalidInputs(t *testing.T) {
|
|||||||
{name: "profile fs nil", opt: promptkit.WithProfileFS(nil, "profiles")},
|
{name: "profile fs nil", opt: promptkit.WithProfileFS(nil, "profiles")},
|
||||||
{name: "profile fs empty root", opt: promptkit.WithProfileFS(fstest.MapFS{}, "")},
|
{name: "profile fs empty root", opt: promptkit.WithProfileFS(fstest.MapFS{}, "")},
|
||||||
{name: "profile file empty", opt: promptkit.WithProfileFile("")},
|
{name: "profile file empty", opt: promptkit.WithProfileFile("")},
|
||||||
|
{name: "fallback profile fs nil", opt: promptkit.WithFallbackProfileFS(nil, "profiles")},
|
||||||
|
{name: "fallback profile fs empty root", opt: promptkit.WithFallbackProfileFS(fstest.MapFS{}, "")},
|
||||||
{name: "schema fs nil", opt: promptkit.WithSchemaFS(nil, "schemas")},
|
{name: "schema fs nil", opt: promptkit.WithSchemaFS(nil, "schemas")},
|
||||||
{name: "schema fs empty root", opt: promptkit.WithSchemaFS(fstest.MapFS{}, "")},
|
{name: "schema fs empty root", opt: promptkit.WithSchemaFS(fstest.MapFS{}, "")},
|
||||||
{name: "schema file empty", opt: promptkit.WithSchemaFile("")},
|
{name: "schema file empty", opt: promptkit.WithSchemaFile("")},
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ package promptkit
|
|||||||
import (
|
import (
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/capacity"
|
"gitea.maximumdirect.net/eric/promptkit/internal/capacity"
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/profile"
|
"gitea.maximumdirect.net/eric/promptkit/internal/profile"
|
||||||
@@ -14,6 +15,11 @@ func mapPublicError(err error) error {
|
|||||||
if err == nil {
|
if err == nil {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
var internalCapacityError *usecase.CapacityError
|
||||||
|
if errors.As(err, &internalCapacityError) && internalCapacityError != nil &&
|
||||||
|
strings.TrimSpace(internalCapacityError.BackendID) != "" {
|
||||||
|
return &CapacityError{BackendID: internalCapacityError.BackendID}
|
||||||
|
}
|
||||||
publicErr := publicErrorFor(err)
|
publicErr := publicErrorFor(err)
|
||||||
if publicErr == nil {
|
if publicErr == nil {
|
||||||
return err
|
return err
|
||||||
|
|||||||
@@ -20,3 +20,31 @@ func TestMapPublicErrorPreservesGenerationCancellation(t *testing.T) {
|
|||||||
t.Fatalf("mapped error=%v, want context.Canceled", err)
|
t.Fatalf("mapped error=%v, want context.Canceled", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestMapPublicErrorTranslatesCapacityError(t *testing.T) {
|
||||||
|
internalErr := &usecase.CapacityError{BackendID: "limited"}
|
||||||
|
|
||||||
|
err := mapPublicError(internalErr)
|
||||||
|
var publicErr *CapacityError
|
||||||
|
if !errors.As(err, &publicErr) || publicErr == nil {
|
||||||
|
t.Fatalf("mapped error=%v, want public CapacityError", err)
|
||||||
|
}
|
||||||
|
if publicErr.BackendID != "limited" {
|
||||||
|
t.Fatalf("mapped backend ID=%q, want limited", publicErr.BackendID)
|
||||||
|
}
|
||||||
|
if !errors.Is(err, ErrCapacityExceeded) {
|
||||||
|
t.Fatalf("mapped error=%v, want ErrCapacityExceeded", err)
|
||||||
|
}
|
||||||
|
if errors.Is(err, ErrInvalidRequest) || errors.Is(err, ErrLLMGenerate) {
|
||||||
|
t.Fatalf("mapped capacity error has an unrelated category: %v", err)
|
||||||
|
}
|
||||||
|
var leakedInternalErr *usecase.CapacityError
|
||||||
|
if errors.As(err, &leakedInternalErr) {
|
||||||
|
t.Fatalf("mapped error exposes internal CapacityError: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
internalErr.BackendID = "changed"
|
||||||
|
if publicErr.BackendID != "limited" {
|
||||||
|
t.Fatalf("mapped backend ID changed with source error: %q", publicErr.BackendID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -124,6 +124,11 @@ func TestManagerAdmissionHonorsContextAndUnlimitedBackends(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("construct manager: %v", err)
|
t.Fatalf("construct manager: %v", err)
|
||||||
}
|
}
|
||||||
|
release, err := manager.Admit(context.Background(), "limited")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("fill limited pool: %v", err)
|
||||||
|
}
|
||||||
|
defer release()
|
||||||
|
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
cancel()
|
cancel()
|
||||||
|
|||||||
@@ -14,9 +14,6 @@ const (
|
|||||||
ContentTypeApplicationJSON = "application/json"
|
ContentTypeApplicationJSON = "application/json"
|
||||||
OpenAIChatCompletionsPath = "/chat/completions"
|
OpenAIChatCompletionsPath = "/chat/completions"
|
||||||
|
|
||||||
ExecutionDefaultTemperature = 0.0
|
|
||||||
ExecutionDefaultMaxTokens = 0
|
|
||||||
ExecutionDefaultTopP = 1.0
|
|
||||||
ExecutionDefaultTimeoutSeconds = 600
|
ExecutionDefaultTimeoutSeconds = 600
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -26,9 +23,6 @@ var (
|
|||||||
|
|
||||||
func ExecutionTargetDefault() domain.ExecutionTarget {
|
func ExecutionTargetDefault() domain.ExecutionTarget {
|
||||||
return domain.ExecutionTarget{
|
return domain.ExecutionTarget{
|
||||||
Temperature: ExecutionDefaultTemperature,
|
|
||||||
MaxTokens: ExecutionDefaultMaxTokens,
|
|
||||||
TopP: ExecutionDefaultTopP,
|
|
||||||
TimeoutSeconds: ExecutionDefaultTimeoutSeconds,
|
TimeoutSeconds: ExecutionDefaultTimeoutSeconds,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -145,6 +145,16 @@ type PromptDefinition struct {
|
|||||||
Validation OutputContract `yaml:"validation"`
|
Validation OutputContract `yaml:"validation"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// PromptInspection is the resolved result of exact prompt inspection.
|
||||||
|
type PromptInspection struct {
|
||||||
|
PromptID string
|
||||||
|
PromptVersion string
|
||||||
|
PromptHash string
|
||||||
|
DefaultProfileID string
|
||||||
|
Inputs []PromptInput
|
||||||
|
OutputContract OutputContract
|
||||||
|
}
|
||||||
|
|
||||||
// PromptInput describes one named input expected by a prompt definition.
|
// PromptInput describes one named input expected by a prompt definition.
|
||||||
type PromptInput struct {
|
type PromptInput struct {
|
||||||
Name string `yaml:"name"`
|
Name string `yaml:"name"`
|
||||||
@@ -236,6 +246,13 @@ type ExecutionTarget struct {
|
|||||||
ExtraParams map[string]any `yaml:"extra_params" json:"extra_params"`
|
ExtraParams map[string]any `yaml:"extra_params" json:"extra_params"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ProfileInspection is the resolved result of exact profile inspection.
|
||||||
|
type ProfileInspection struct {
|
||||||
|
ProfileID string
|
||||||
|
EffectiveModelParams ExecutionTarget
|
||||||
|
APIKeyRequired bool
|
||||||
|
}
|
||||||
|
|
||||||
// OutputContract defines the requirements for the output artifact.
|
// OutputContract defines the requirements for the output artifact.
|
||||||
type OutputContract struct {
|
type OutputContract struct {
|
||||||
Format OutputFormat `yaml:"format"`
|
Format OutputFormat `yaml:"format"`
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
// Package jsonvalue validates and defensively copies JSON-compatible value
|
// Package jsonvalue validates and defensively copies JSON-compatible value
|
||||||
// trees used by public configuration and request boundaries.
|
// trees used by configuration, request, and prepared-state boundaries.
|
||||||
package jsonvalue
|
package jsonvalue
|
||||||
|
|
||||||
import (
|
import (
|
||||||
@@ -18,13 +18,19 @@ type visit struct {
|
|||||||
ptr uintptr
|
ptr uintptr
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Copy validates and deeply copies a JSON-compatible value while preserving
|
||||||
|
// compatible concrete map, slice, array, scalar, and number types.
|
||||||
|
func Copy(src any) (any, error) {
|
||||||
|
return copyValue(reflect.ValueOf(src), "value", make(map[visit]struct{}), true)
|
||||||
|
}
|
||||||
|
|
||||||
// CopyMap validates and deeply copies an extra-parameter map while preserving
|
// CopyMap validates and deeply copies an extra-parameter map while preserving
|
||||||
// compatible concrete map, slice, array, scalar, and number types.
|
// compatible concrete map, slice, array, scalar, and number types.
|
||||||
func CopyMap(src map[string]any) (map[string]any, error) {
|
func CopyMap(src map[string]any) (map[string]any, error) {
|
||||||
if src == nil {
|
if src == nil {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
copied, err := copyValue(reflect.ValueOf(src), "extra_params", make(map[visit]struct{}))
|
copied, err := copyValue(reflect.ValueOf(src), "extra_params", make(map[visit]struct{}), false)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -35,7 +41,12 @@ func CopyMap(src map[string]any) (map[string]any, error) {
|
|||||||
return out, nil
|
return out, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func copyValue(value reflect.Value, path string, seen map[visit]struct{}) (any, error) {
|
func copyValue(
|
||||||
|
value reflect.Value,
|
||||||
|
path string,
|
||||||
|
seen map[visit]struct{},
|
||||||
|
allowEmptyMapKeys bool,
|
||||||
|
) (any, error) {
|
||||||
if !value.IsValid() {
|
if !value.IsValid() {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
@@ -43,7 +54,7 @@ func copyValue(value reflect.Value, path string, seen map[visit]struct{}) (any,
|
|||||||
if value.IsNil() {
|
if value.IsNil() {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
return copyValue(value.Elem(), path, seen)
|
return copyValue(value.Elem(), path, seen, allowEmptyMapKeys)
|
||||||
}
|
}
|
||||||
if !value.CanInterface() {
|
if !value.CanInterface() {
|
||||||
return nil, fmt.Errorf("%s: value cannot be copied", path)
|
return nil, fmt.Errorf("%s: value cannot be copied", path)
|
||||||
@@ -88,22 +99,27 @@ func copyValue(value reflect.Value, path string, seen map[visit]struct{}) (any,
|
|||||||
}
|
}
|
||||||
seen[current] = struct{}{}
|
seen[current] = struct{}{}
|
||||||
defer delete(seen, current)
|
defer delete(seen, current)
|
||||||
return copyValue(value.Elem(), path, seen)
|
return copyValue(value.Elem(), path, seen, allowEmptyMapKeys)
|
||||||
case reflect.Map:
|
case reflect.Map:
|
||||||
return copyMapValue(value, path, seen)
|
return copyMapValue(value, path, seen, allowEmptyMapKeys)
|
||||||
case reflect.Slice:
|
case reflect.Slice:
|
||||||
if value.IsNil() {
|
if value.IsNil() {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
return copySequenceValue(value, path, seen)
|
return copySequenceValue(value, path, seen, allowEmptyMapKeys)
|
||||||
case reflect.Array:
|
case reflect.Array:
|
||||||
return copySequenceValue(value, path, seen)
|
return copySequenceValue(value, path, seen, allowEmptyMapKeys)
|
||||||
default:
|
default:
|
||||||
return nil, fmt.Errorf("%s: unsupported JSON value type %s", path, value.Type())
|
return nil, fmt.Errorf("%s: unsupported JSON value type %s", path, value.Type())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func copyMapValue(value reflect.Value, path string, seen map[visit]struct{}) (any, error) {
|
func copyMapValue(
|
||||||
|
value reflect.Value,
|
||||||
|
path string,
|
||||||
|
seen map[visit]struct{},
|
||||||
|
allowEmptyMapKeys bool,
|
||||||
|
) (any, error) {
|
||||||
if value.IsNil() {
|
if value.IsNil() {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
@@ -133,10 +149,10 @@ func copyMapValue(value reflect.Value, path string, seen map[visit]struct{}) (an
|
|||||||
elementType := value.Type().Elem()
|
elementType := value.Type().Elem()
|
||||||
for _, key := range keys {
|
for _, key := range keys {
|
||||||
name := key.String()
|
name := key.String()
|
||||||
if name == "" {
|
if name == "" && !allowEmptyMapKeys {
|
||||||
return nil, fmt.Errorf("%s: map key must not be empty", path)
|
return nil, fmt.Errorf("%s: map key must not be empty", path)
|
||||||
}
|
}
|
||||||
copied, err := copyValue(value.MapIndex(key), path+"."+name, seen)
|
copied, err := copyValue(value.MapIndex(key), path+"."+name, seen, allowEmptyMapKeys)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -171,7 +187,12 @@ func copyMapValue(value reflect.Value, path string, seen map[visit]struct{}) (an
|
|||||||
return out, nil
|
return out, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func copySequenceValue(value reflect.Value, path string, seen map[visit]struct{}) (any, error) {
|
func copySequenceValue(
|
||||||
|
value reflect.Value,
|
||||||
|
path string,
|
||||||
|
seen map[visit]struct{},
|
||||||
|
allowEmptyMapKeys bool,
|
||||||
|
) (any, error) {
|
||||||
var current visit
|
var current visit
|
||||||
if value.Kind() == reflect.Slice {
|
if value.Kind() == reflect.Slice {
|
||||||
current = visit{typ: value.Type(), ptr: value.Pointer()}
|
current = visit{typ: value.Type(), ptr: value.Pointer()}
|
||||||
@@ -186,7 +207,12 @@ func copySequenceValue(value reflect.Value, path string, seen map[visit]struct{}
|
|||||||
preserveType := true
|
preserveType := true
|
||||||
elementType := value.Type().Elem()
|
elementType := value.Type().Elem()
|
||||||
for i := 0; i < value.Len(); i++ {
|
for i := 0; i < value.Len(); i++ {
|
||||||
copied, err := copyValue(value.Index(i), fmt.Sprintf("%s[%d]", path, i), seen)
|
copied, err := copyValue(
|
||||||
|
value.Index(i),
|
||||||
|
fmt.Sprintf("%s[%d]", path, i),
|
||||||
|
seen,
|
||||||
|
allowEmptyMapKeys,
|
||||||
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -44,6 +44,21 @@ func TestCopyMapPreservesTypesAndIsolatesMutations(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestCopyAllowsEmptyObjectKeysAndIsolatesMutations(t *testing.T) {
|
||||||
|
nested := map[string]any{"": []any{"original"}}
|
||||||
|
|
||||||
|
copiedValue, err := jsonvalue.Copy(nested)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("copy value: %v", err)
|
||||||
|
}
|
||||||
|
nested[""].([]any)[0] = "changed"
|
||||||
|
|
||||||
|
copied := copiedValue.(map[string]any)
|
||||||
|
if got := copied[""].([]any)[0]; got != "original" {
|
||||||
|
t.Fatalf("copied value was not isolated: %v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestCopyMapRejectsInvalidValues(t *testing.T) {
|
func TestCopyMapRejectsInvalidValues(t *testing.T) {
|
||||||
cyclicMap := map[string]any{}
|
cyclicMap := map[string]any{}
|
||||||
cyclicMap["self"] = cyclicMap
|
cyclicMap["self"] = cyclicMap
|
||||||
|
|||||||
@@ -2,7 +2,6 @@ package builtin
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"embed"
|
"embed"
|
||||||
"strings"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/profile"
|
"gitea.maximumdirect.net/eric/promptkit/internal/profile"
|
||||||
)
|
)
|
||||||
@@ -15,17 +14,3 @@ var assets embed.FS
|
|||||||
func NewRepository() profile.Repository {
|
func NewRepository() profile.Repository {
|
||||||
return profile.NewFSRepository(assets, assetRoot)
|
return profile.NewFSRepository(assets, assetRoot)
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewRepositoryWithPrimary(primary profile.Repository) profile.Repository {
|
|
||||||
if primary == nil {
|
|
||||||
return NewRepository()
|
|
||||||
}
|
|
||||||
return profile.NewOverlayRepository(primary, NewRepository())
|
|
||||||
}
|
|
||||||
|
|
||||||
func NewRepositoryWithDirectory(dir string) profile.Repository {
|
|
||||||
if strings.TrimSpace(dir) == "" {
|
|
||||||
return NewRepository()
|
|
||||||
}
|
|
||||||
return NewRepositoryWithPrimary(profile.NewFilesystemRepository(dir))
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -2,14 +2,11 @@ package builtin
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
|
||||||
"io/fs"
|
"io/fs"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/backend"
|
"gitea.maximumdirect.net/eric/promptkit/internal/backend"
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/profile"
|
|
||||||
"gopkg.in/yaml.v3"
|
"gopkg.in/yaml.v3"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -91,53 +88,3 @@ func loadBuiltInProfileIDs(t *testing.T) map[string]string {
|
|||||||
}
|
}
|
||||||
return ids
|
return ids
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRepositoryWithPrimaryUsesPrimaryBeforeBuiltIns(t *testing.T) {
|
|
||||||
repo := NewRepositoryWithPrimary(staticProfileRepo{
|
|
||||||
profiles: map[string]string{"mistral-small-3": "custom-model"},
|
|
||||||
})
|
|
||||||
|
|
||||||
p, err := repo.GetProfile(context.Background(), "mistral-small-3")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("expected profile to load, got %v", err)
|
|
||||||
}
|
|
||||||
if p.Model != "custom-model" {
|
|
||||||
t.Fatalf("expected primary profile to override built-in, got %+v", p)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRepositoryWithPrimaryFallsBackToBuiltIns(t *testing.T) {
|
|
||||||
repo := NewRepositoryWithPrimary(staticProfileRepo{})
|
|
||||||
|
|
||||||
p, err := repo.GetProfile(context.Background(), "mistral-small-3")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("expected built-in profile to load, got %v", err)
|
|
||||||
}
|
|
||||||
if p.ID != "mistral-small-3" {
|
|
||||||
t.Fatalf("unexpected profile: %+v", p)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRepositoryWithPrimaryDoesNotFallBackAfterPrimaryError(t *testing.T) {
|
|
||||||
repo := NewRepositoryWithPrimary(staticProfileRepo{err: profile.ErrInvalidProfile})
|
|
||||||
|
|
||||||
_, err := repo.GetProfile(context.Background(), "mistral-small-3")
|
|
||||||
if !errors.Is(err, profile.ErrInvalidProfile) {
|
|
||||||
t.Fatalf("expected primary error, got %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
type staticProfileRepo struct {
|
|
||||||
profiles map[string]string
|
|
||||||
err error
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r staticProfileRepo) GetProfile(_ context.Context, id string) (*domain.ExecutionProfile, error) {
|
|
||||||
if r.err != nil {
|
|
||||||
return nil, r.err
|
|
||||||
}
|
|
||||||
if model, ok := r.profiles[id]; ok {
|
|
||||||
return &domain.ExecutionProfile{ID: id, Endpoint: "http://primary/v1", Model: model}, nil
|
|
||||||
}
|
|
||||||
return nil, profile.ErrProfileNotFound
|
|
||||||
}
|
|
||||||
|
|||||||
24
internal/usecase/capacity_error.go
Normal file
24
internal/usecase/capacity_error.go
Normal file
@@ -0,0 +1,24 @@
|
|||||||
|
package usecase
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/capacity"
|
||||||
|
)
|
||||||
|
|
||||||
|
// CapacityError identifies bounded admission rejected for one selected backend.
|
||||||
|
type CapacityError struct {
|
||||||
|
BackendID string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *CapacityError) Error() string {
|
||||||
|
if e == nil || strings.TrimSpace(e.BackendID) == "" {
|
||||||
|
return capacity.ErrCapacityExceeded.Error()
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("backend %q admission: %v", e.BackendID, capacity.ErrCapacityExceeded)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *CapacityError) Unwrap() error {
|
||||||
|
return capacity.ErrCapacityExceeded
|
||||||
|
}
|
||||||
298
internal/usecase/prepared_execution.go
Normal file
298
internal/usecase/prepared_execution.go
Normal file
@@ -0,0 +1,298 @@
|
|||||||
|
package usecase
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/jsonvalue"
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/validate"
|
||||||
|
)
|
||||||
|
|
||||||
|
type preparedExecutionState uint8
|
||||||
|
|
||||||
|
const (
|
||||||
|
preparedExecutionReady preparedExecutionState = iota
|
||||||
|
preparedExecutionClaimed
|
||||||
|
preparedExecutionDiscarded
|
||||||
|
)
|
||||||
|
|
||||||
|
// PreparedExecution owns one frozen, single-use runner execution.
|
||||||
|
type PreparedExecution struct {
|
||||||
|
owner *Runner
|
||||||
|
mu sync.Mutex
|
||||||
|
state preparedExecutionState
|
||||||
|
details *domain.PreparedRun
|
||||||
|
payload *preparedExecutionPayload
|
||||||
|
}
|
||||||
|
|
||||||
|
type preparedExecutionPayload struct {
|
||||||
|
prepared *domain.PreparedRun
|
||||||
|
validation validate.PreparedValidation
|
||||||
|
directKey string
|
||||||
|
}
|
||||||
|
|
||||||
|
// PrepareExecution completes preparation without generation or admission and
|
||||||
|
// returns a runner-bound, single-use execution.
|
||||||
|
func (r *Runner) PrepareExecution(ctx context.Context, req domain.RunRequest) (*PreparedExecution, error) {
|
||||||
|
state, err := r.resolvePreparation(ctx, req, time.Now().UTC())
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
validationPlan, err := r.prepareValidation(ctx, state.effectiveContract)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
structuredOutput, err := r.structuredOutputFromValidationPlan(
|
||||||
|
state.definition,
|
||||||
|
state.effectiveContract,
|
||||||
|
validationPlan,
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
prepared, err := r.completePreparationWithStructuredOutput(ctx, req, state, structuredOutput)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
executionSnapshot, err := clonePreparedRun(prepared)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("%w: failed to copy prepared execution: %v", ErrInvalidRequest, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
details, err := clonePreparedRun(executionSnapshot)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("%w: failed to copy prepared execution details: %v", ErrInvalidRequest, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return &PreparedExecution{
|
||||||
|
owner: r,
|
||||||
|
state: preparedExecutionReady,
|
||||||
|
details: details,
|
||||||
|
payload: &preparedExecutionPayload{
|
||||||
|
prepared: executionSnapshot,
|
||||||
|
validation: validationPlan,
|
||||||
|
directKey: state.effectiveModel.APIKey,
|
||||||
|
},
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *Runner) prepareValidation(
|
||||||
|
ctx context.Context,
|
||||||
|
contract domain.OutputContract,
|
||||||
|
) (validate.PreparedValidation, error) {
|
||||||
|
if r.validator == nil {
|
||||||
|
return noOpPreparedValidation{contract: contract}, nil
|
||||||
|
}
|
||||||
|
preparer, ok := r.validator.(validate.ValidationPreparer)
|
||||||
|
if !ok {
|
||||||
|
return nil, fmt.Errorf("%w: validator does not support prepared validation", ErrValidation)
|
||||||
|
}
|
||||||
|
plan, err := preparer.PrepareValidation(ctx, contract)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("%w: %w", ErrValidation, err)
|
||||||
|
}
|
||||||
|
if plan == nil {
|
||||||
|
return nil, fmt.Errorf("%w: validator returned nil prepared validation", ErrValidation)
|
||||||
|
}
|
||||||
|
return plan, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *Runner) structuredOutputFromValidationPlan(
|
||||||
|
def *domain.PromptDefinition,
|
||||||
|
contract domain.OutputContract,
|
||||||
|
plan validate.PreparedValidation,
|
||||||
|
) (*domain.StructuredOutputSpec, error) {
|
||||||
|
if contract.ValidationMode != domain.ValidationJSONSchema {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
schemaDocument := plan.SchemaDocument()
|
||||||
|
if schemaDocument == nil {
|
||||||
|
if r.validator == nil {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
return nil, fmt.Errorf("%w: prepared json_schema validation has no schema document", ErrValidation)
|
||||||
|
}
|
||||||
|
return structuredOutputSpec(def, schemaDocument), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Details returns a fresh credential-redacted copy of the prepared run.
|
||||||
|
func (p *PreparedExecution) Details() *domain.PreparedRun {
|
||||||
|
if p == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
p.mu.Lock()
|
||||||
|
detailsSnapshot := p.details
|
||||||
|
p.mu.Unlock()
|
||||||
|
|
||||||
|
details, err := clonePreparedRun(detailsSnapshot)
|
||||||
|
if err != nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return details
|
||||||
|
}
|
||||||
|
|
||||||
|
// Discard invalidates an unclaimed execution and drops its private payload.
|
||||||
|
func (p *PreparedExecution) Discard() {
|
||||||
|
if p == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
p.mu.Lock()
|
||||||
|
if p.state != preparedExecutionReady {
|
||||||
|
p.mu.Unlock()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
p.state = preparedExecutionDiscarded
|
||||||
|
payload := p.payload
|
||||||
|
p.payload = nil
|
||||||
|
p.mu.Unlock()
|
||||||
|
|
||||||
|
payload.clear()
|
||||||
|
}
|
||||||
|
|
||||||
|
// RunPrepared claims and executes one prepared execution owned by this runner.
|
||||||
|
func (r *Runner) RunPrepared(ctx context.Context, prepared *PreparedExecution) (*domain.RunResult, error) {
|
||||||
|
payload, err := prepared.claim(r)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer payload.clear()
|
||||||
|
|
||||||
|
runID, err := newRunID()
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to create run id: %w", err)
|
||||||
|
}
|
||||||
|
start := time.Now().UTC()
|
||||||
|
|
||||||
|
target := payload.prepared.EffectiveModelParams
|
||||||
|
if err := validateAPIKey(target.APIKeyEnv, payload.directKey, target.APIKeyRequired); err != nil {
|
||||||
|
return nil, fmt.Errorf("%w: %w", ErrInvalidRequest, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
release, err := r.admitRun(ctx, target.BackendID)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer release()
|
||||||
|
|
||||||
|
return r.executePreparedRun(ctx, payload.prepared, payload.directKey, runID, start, func(
|
||||||
|
ctx context.Context,
|
||||||
|
artifact *domain.Artifact,
|
||||||
|
attemptsUsed int,
|
||||||
|
) (domain.ValidationResult, error) {
|
||||||
|
result, validationErr := payload.validation.Validate(ctx, artifact)
|
||||||
|
if validationErr != nil {
|
||||||
|
return domain.ValidationResult{}, validationErr
|
||||||
|
}
|
||||||
|
result.RepairAttempts = attemptsUsed
|
||||||
|
return result, nil
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *PreparedExecution) claim(owner *Runner) (*preparedExecutionPayload, error) {
|
||||||
|
if p == nil || owner == nil || p.owner != owner {
|
||||||
|
return nil, fmt.Errorf("%w: prepared execution does not belong to this runner", ErrInvalidRequest)
|
||||||
|
}
|
||||||
|
|
||||||
|
p.mu.Lock()
|
||||||
|
defer p.mu.Unlock()
|
||||||
|
if p.state != preparedExecutionReady || p.payload == nil {
|
||||||
|
return nil, fmt.Errorf("%w: prepared execution is not ready", ErrInvalidRequest)
|
||||||
|
}
|
||||||
|
p.state = preparedExecutionClaimed
|
||||||
|
payload := p.payload
|
||||||
|
p.payload = nil
|
||||||
|
return payload, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *preparedExecutionPayload) clear() {
|
||||||
|
if p == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if p.prepared != nil {
|
||||||
|
p.prepared.EffectiveModelParams.APIKey = ""
|
||||||
|
}
|
||||||
|
p.prepared = nil
|
||||||
|
p.validation = nil
|
||||||
|
p.directKey = ""
|
||||||
|
}
|
||||||
|
|
||||||
|
type noOpPreparedValidation struct {
|
||||||
|
contract domain.OutputContract
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p noOpPreparedValidation) Validate(
|
||||||
|
ctx context.Context,
|
||||||
|
_ *domain.Artifact,
|
||||||
|
) (domain.ValidationResult, error) {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return domain.ValidationResult{}, err
|
||||||
|
}
|
||||||
|
return domain.ValidationResult{
|
||||||
|
Status: domain.ValidationSkipped,
|
||||||
|
Mode: p.contract.ValidationMode,
|
||||||
|
SchemaPath: p.contract.SchemaPath,
|
||||||
|
RepairAttempts: p.contract.RepairAttempts,
|
||||||
|
IsValid: true,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (noOpPreparedValidation) SchemaDocument() any {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func clonePreparedRun(source *domain.PreparedRun) (*domain.PreparedRun, error) {
|
||||||
|
if source == nil {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
copied := *source
|
||||||
|
extraParams, err := jsonvalue.CopyMap(source.EffectiveModelParams.ExtraParams)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
copied.EffectiveModelParams.ExtraParams = extraParams
|
||||||
|
|
||||||
|
if source.InputHashes != nil {
|
||||||
|
copied.InputHashes = make(map[string]string, len(source.InputHashes))
|
||||||
|
for name, hash := range source.InputHashes {
|
||||||
|
copied.InputHashes[name] = hash
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if source.Messages != nil {
|
||||||
|
copied.Messages = make([]domain.RenderedMessage, len(source.Messages))
|
||||||
|
for i, message := range source.Messages {
|
||||||
|
copied.Messages[i] = message
|
||||||
|
if message.CacheControl != nil {
|
||||||
|
cacheControl := *message.CacheControl
|
||||||
|
copied.Messages[i].CacheControl = &cacheControl
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if source.StructuredOutput != nil {
|
||||||
|
structuredOutput := *source.StructuredOutput
|
||||||
|
copied.StructuredOutput = &structuredOutput
|
||||||
|
if source.StructuredOutput.JSONSchema != nil {
|
||||||
|
jsonSchema := *source.StructuredOutput.JSONSchema
|
||||||
|
copied.StructuredOutput.JSONSchema = &jsonSchema
|
||||||
|
schema, copyErr := cloneJSONValue(source.StructuredOutput.JSONSchema.Schema)
|
||||||
|
if copyErr != nil {
|
||||||
|
return nil, copyErr
|
||||||
|
}
|
||||||
|
copied.StructuredOutput.JSONSchema.Schema = schema
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return &copied, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func cloneJSONValue(source any) (any, error) {
|
||||||
|
return jsonvalue.Copy(source)
|
||||||
|
}
|
||||||
510
internal/usecase/prepared_execution_test.go
Normal file
510
internal/usecase/prepared_execution_test.go
Normal file
@@ -0,0 +1,510 @@
|
|||||||
|
package usecase
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"os"
|
||||||
|
"reflect"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/validate"
|
||||||
|
)
|
||||||
|
|
||||||
|
type recordingPreparedValidation struct {
|
||||||
|
contract domain.OutputContract
|
||||||
|
schemaDocument any
|
||||||
|
results []domain.ValidationResult
|
||||||
|
errs []error
|
||||||
|
artifacts []string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *recordingPreparedValidation) Validate(
|
||||||
|
_ context.Context,
|
||||||
|
artifact *domain.Artifact,
|
||||||
|
) (domain.ValidationResult, error) {
|
||||||
|
p.artifacts = append(p.artifacts, string(artifact.Body))
|
||||||
|
index := len(p.artifacts) - 1
|
||||||
|
if index < len(p.errs) && p.errs[index] != nil {
|
||||||
|
return domain.ValidationResult{}, p.errs[index]
|
||||||
|
}
|
||||||
|
if len(p.results) == 0 {
|
||||||
|
return domain.ValidationResult{
|
||||||
|
Status: domain.ValidationPassed,
|
||||||
|
Mode: p.contract.ValidationMode,
|
||||||
|
IsValid: true,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
if index >= len(p.results) {
|
||||||
|
index = len(p.results) - 1
|
||||||
|
}
|
||||||
|
return p.results[index], nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *recordingPreparedValidation) SchemaDocument() any {
|
||||||
|
return p.schemaDocument
|
||||||
|
}
|
||||||
|
|
||||||
|
type recordingValidationPreparer struct {
|
||||||
|
plan *recordingPreparedValidation
|
||||||
|
prepareErr error
|
||||||
|
prepareCalls int
|
||||||
|
directValidateCalls int
|
||||||
|
}
|
||||||
|
|
||||||
|
func (v *recordingValidationPreparer) Validate(
|
||||||
|
context.Context,
|
||||||
|
*domain.Artifact,
|
||||||
|
domain.OutputContract,
|
||||||
|
) (domain.ValidationResult, error) {
|
||||||
|
v.directValidateCalls++
|
||||||
|
return domain.ValidationResult{}, errors.New("live validation must not be used")
|
||||||
|
}
|
||||||
|
|
||||||
|
func (v *recordingValidationPreparer) PrepareValidation(
|
||||||
|
_ context.Context,
|
||||||
|
contract domain.OutputContract,
|
||||||
|
) (validate.PreparedValidation, error) {
|
||||||
|
v.prepareCalls++
|
||||||
|
if v.prepareErr != nil {
|
||||||
|
return nil, v.prepareErr
|
||||||
|
}
|
||||||
|
v.plan.contract = contract
|
||||||
|
return v.plan, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunnerPrepareExecutionCompletesWithoutAdmissionOrGeneration(t *testing.T) {
|
||||||
|
schemaDocument := map[string]any{
|
||||||
|
"type": "object",
|
||||||
|
"properties": map[string]any{
|
||||||
|
"value": map[string]any{"type": "string"},
|
||||||
|
"": map[string]any{"type": "boolean"},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
def := promptDef(domain.FormatJSON, domain.ValidationJSONSchema, 0)
|
||||||
|
def.Validation.SchemaPath = "schema.json"
|
||||||
|
reader := defaultArtifactReader()
|
||||||
|
renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{
|
||||||
|
SessionID: "prepared-session",
|
||||||
|
Messages: []domain.RenderedMessage{{
|
||||||
|
Role: "user",
|
||||||
|
Content: "original message",
|
||||||
|
}},
|
||||||
|
}}
|
||||||
|
llmClient := &fakeLLM{forbid: true}
|
||||||
|
validator := &recordingValidationPreparer{
|
||||||
|
plan: &recordingPreparedValidation{schemaDocument: schemaDocument},
|
||||||
|
}
|
||||||
|
admitter := &fakeRunAdmitter{}
|
||||||
|
profile := defaultExecutionProfile()
|
||||||
|
profile.ExtraParams = map[string]any{
|
||||||
|
"metadata": map[string]any{"source": "original"},
|
||||||
|
}
|
||||||
|
runner := NewRunner(
|
||||||
|
&fakePromptRepo{def: def},
|
||||||
|
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": profile}},
|
||||||
|
nil,
|
||||||
|
reader,
|
||||||
|
renderer,
|
||||||
|
llmClient,
|
||||||
|
validator,
|
||||||
|
admitter,
|
||||||
|
)
|
||||||
|
|
||||||
|
prepared, err := runner.PrepareExecution(context.Background(), domain.RunRequest{
|
||||||
|
PromptID: "p",
|
||||||
|
ProfileID: "exec",
|
||||||
|
APIKey: "direct-test-key",
|
||||||
|
Inputs: singleInputRef(),
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("prepare execution: %v", err)
|
||||||
|
}
|
||||||
|
defer prepared.Discard()
|
||||||
|
|
||||||
|
if validator.prepareCalls != 1 || validator.directValidateCalls != 0 {
|
||||||
|
t.Fatalf(
|
||||||
|
"validation calls=(prepare=%d direct=%d), want (1, 0)",
|
||||||
|
validator.prepareCalls,
|
||||||
|
validator.directValidateCalls,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
if reader.calls != 1 || renderer.calls != 1 {
|
||||||
|
t.Fatalf("completion calls=(artifact=%d render=%d), want (1, 1)", reader.calls, renderer.calls)
|
||||||
|
}
|
||||||
|
if len(admitter.backendIDs) != 0 || llmClient.calls != 0 {
|
||||||
|
t.Fatalf("prepare invoked execution collaborators: admission=%v generation=%d", admitter.backendIDs, llmClient.calls)
|
||||||
|
}
|
||||||
|
|
||||||
|
first := prepared.Details()
|
||||||
|
if first == nil {
|
||||||
|
t.Fatal("prepared details are nil")
|
||||||
|
}
|
||||||
|
if first.EffectiveModelParams.APIKey != "" {
|
||||||
|
t.Fatal("prepared details retained the direct API key")
|
||||||
|
}
|
||||||
|
if first.StructuredOutput == nil ||
|
||||||
|
first.StructuredOutput.JSONSchema == nil ||
|
||||||
|
!reflect.DeepEqual(first.StructuredOutput.JSONSchema.Schema, schemaDocument) {
|
||||||
|
t.Fatalf("prepared details have unexpected structured output: %#v", first.StructuredOutput)
|
||||||
|
}
|
||||||
|
|
||||||
|
first.Messages[0].Content = "caller mutation"
|
||||||
|
first.InputHashes["input"] = "caller mutation"
|
||||||
|
first.EffectiveModelParams.ExtraParams["metadata"].(map[string]any)["source"] = "caller mutation"
|
||||||
|
first.StructuredOutput.JSONSchema.Schema.(map[string]any)["type"] = "string"
|
||||||
|
renderer.rendered.Messages[0].Content = "source mutation"
|
||||||
|
|
||||||
|
second := prepared.Details()
|
||||||
|
if second.Messages[0].Content != "original message" ||
|
||||||
|
second.InputHashes["input"] == "caller mutation" ||
|
||||||
|
second.EffectiveModelParams.ExtraParams["metadata"].(map[string]any)["source"] != "original" ||
|
||||||
|
second.StructuredOutput.JSONSchema.Schema.(map[string]any)["type"] != "object" {
|
||||||
|
t.Fatalf("details did not preserve an independent snapshot: %#v", second)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunnerRunPreparedRechecksEnvironmentCredentialBeforeAdmission(t *testing.T) {
|
||||||
|
const environmentName = "PROMPTKIT_PREPARED_EXECUTION_TEST_KEY"
|
||||||
|
t.Setenv(environmentName, "available-during-preparation")
|
||||||
|
|
||||||
|
profile := defaultExecutionProfile()
|
||||||
|
profile.APIKeyEnv = environmentName
|
||||||
|
validator := &recordingValidationPreparer{plan: &recordingPreparedValidation{}}
|
||||||
|
admitter := &fakeRunAdmitter{}
|
||||||
|
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "unexpected"}}
|
||||||
|
runner := NewRunner(
|
||||||
|
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
|
||||||
|
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": profile}},
|
||||||
|
nil,
|
||||||
|
defaultArtifactReader(),
|
||||||
|
defaultRenderer(),
|
||||||
|
llmClient,
|
||||||
|
validator,
|
||||||
|
admitter,
|
||||||
|
)
|
||||||
|
prepared, err := runner.PrepareExecution(context.Background(), domain.RunRequest{
|
||||||
|
PromptID: "p",
|
||||||
|
ProfileID: "exec",
|
||||||
|
Inputs: singleInputRef(),
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("prepare execution: %v", err)
|
||||||
|
}
|
||||||
|
if err := os.Unsetenv(environmentName); err != nil {
|
||||||
|
t.Fatalf("unset credential environment: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
result, err := runner.RunPrepared(context.Background(), prepared)
|
||||||
|
if result != nil {
|
||||||
|
t.Fatalf("credential failure returned partial result: %+v", result)
|
||||||
|
}
|
||||||
|
if !errors.Is(err, ErrInvalidRequest) || !errors.Is(err, ErrAPIKeyEnvMissing) {
|
||||||
|
t.Fatalf("credential error identities are missing: %v", err)
|
||||||
|
}
|
||||||
|
if len(admitter.backendIDs) != 0 || llmClient.calls != 0 {
|
||||||
|
t.Fatalf("credential failure reached admission or generation: admission=%v generation=%d", admitter.backendIDs, llmClient.calls)
|
||||||
|
}
|
||||||
|
if _, err := runner.RunPrepared(context.Background(), prepared); !errors.Is(err, ErrInvalidRequest) {
|
||||||
|
t.Fatalf("credential failure did not consume execution: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunnerRunPreparedKeepsDirectCredentialOutOfMetadata(t *testing.T) {
|
||||||
|
const directKey = "direct-prepared-test-key"
|
||||||
|
|
||||||
|
profile := defaultExecutionProfile()
|
||||||
|
profile.APIKeyRequired = true
|
||||||
|
client := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
|
||||||
|
runner := NewRunner(
|
||||||
|
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
|
||||||
|
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": profile}},
|
||||||
|
nil,
|
||||||
|
defaultArtifactReader(),
|
||||||
|
defaultRenderer(),
|
||||||
|
client,
|
||||||
|
&recordingValidationPreparer{plan: &recordingPreparedValidation{}},
|
||||||
|
nil,
|
||||||
|
)
|
||||||
|
|
||||||
|
prepared, err := runner.PrepareExecution(context.Background(), domain.RunRequest{
|
||||||
|
PromptID: "p",
|
||||||
|
ProfileID: "exec",
|
||||||
|
APIKey: directKey,
|
||||||
|
Inputs: singleInputRef(),
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("prepare execution: %v", err)
|
||||||
|
}
|
||||||
|
if details := prepared.Details(); details.EffectiveModelParams.APIKey != "" {
|
||||||
|
t.Fatal("prepared details retained direct credential")
|
||||||
|
}
|
||||||
|
|
||||||
|
result, err := runner.RunPrepared(context.Background(), prepared)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("run prepared: %v", err)
|
||||||
|
}
|
||||||
|
if client.lastReq.Target.APIKey != directKey {
|
||||||
|
t.Fatal("generation did not receive direct credential")
|
||||||
|
}
|
||||||
|
if result.EffectiveModelParams.APIKey != "" {
|
||||||
|
t.Fatal("run result retained direct credential")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunnerRunPreparedUsesFrozenValidationForInitialAndRepairOutputs(t *testing.T) {
|
||||||
|
validator := &recordingValidationPreparer{
|
||||||
|
plan: &recordingPreparedValidation{
|
||||||
|
results: []domain.ValidationResult{
|
||||||
|
{
|
||||||
|
Status: domain.ValidationFailed,
|
||||||
|
Mode: domain.ValidationJSON,
|
||||||
|
Errors: []string{"invalid"},
|
||||||
|
IsValid: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Status: domain.ValidationPassed,
|
||||||
|
Mode: domain.ValidationJSON,
|
||||||
|
IsValid: true,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
repairer := &fakeRepairer{
|
||||||
|
responses: []*domain.GenerateResponse{{Content: `{"repaired":true}`}},
|
||||||
|
}
|
||||||
|
admitter := &fakeRunAdmitter{}
|
||||||
|
reader := defaultArtifactReader()
|
||||||
|
renderer := defaultRenderer()
|
||||||
|
runner := NewRunnerWithRepairer(
|
||||||
|
&fakePromptRepo{def: promptDef(domain.FormatJSON, domain.ValidationJSON, 1)},
|
||||||
|
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
||||||
|
nil,
|
||||||
|
reader,
|
||||||
|
renderer,
|
||||||
|
&fakeLLM{resp: &domain.GenerateResponse{Content: `{"broken":true}`}},
|
||||||
|
validator,
|
||||||
|
repairer,
|
||||||
|
admitter,
|
||||||
|
)
|
||||||
|
prepared, err := runner.PrepareExecution(context.Background(), domain.RunRequest{
|
||||||
|
PromptID: "p",
|
||||||
|
ProfileID: "exec",
|
||||||
|
Inputs: singleInputRef(),
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("prepare execution: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
result, err := runner.RunPrepared(context.Background(), prepared)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("run prepared: %v", err)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(validator.plan.artifacts, []string{`{"broken":true}`, `{"repaired":true}`}) {
|
||||||
|
t.Fatalf("prepared validation artifacts=%#v", validator.plan.artifacts)
|
||||||
|
}
|
||||||
|
if validator.directValidateCalls != 0 || repairer.calls != 1 {
|
||||||
|
t.Fatalf("validation/repair calls=(direct=%d repair=%d), want (0, 1)", validator.directValidateCalls, repairer.calls)
|
||||||
|
}
|
||||||
|
if result.Validation.Status != domain.ValidationPassed || result.Validation.RepairAttempts != 1 {
|
||||||
|
t.Fatalf("unexpected repaired validation result: %+v", result.Validation)
|
||||||
|
}
|
||||||
|
if admitter.releaseCalls != 1 {
|
||||||
|
t.Fatalf("admission releases=%d, want 1", admitter.releaseCalls)
|
||||||
|
}
|
||||||
|
if reader.calls != 1 || renderer.calls != 1 {
|
||||||
|
t.Fatalf("execution reopened preparation sources: artifact=%d render=%d", reader.calls, renderer.calls)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunnerRunPreparedReleasesAdmissionAcrossExecutionErrors(t *testing.T) {
|
||||||
|
generationFailure := errors.New("generation failed")
|
||||||
|
validationFailure := errors.New("validation failed")
|
||||||
|
repairFailure := errors.New("repair failed")
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
generationErr error
|
||||||
|
validation *recordingPreparedValidation
|
||||||
|
repairer *fakeRepairer
|
||||||
|
wantError error
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "generation failure",
|
||||||
|
generationErr: generationFailure,
|
||||||
|
validation: &recordingPreparedValidation{},
|
||||||
|
wantError: ErrLLMGenerate,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "validation failure",
|
||||||
|
validation: &recordingPreparedValidation{
|
||||||
|
errs: []error{validationFailure},
|
||||||
|
},
|
||||||
|
wantError: ErrValidation,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "repair failure",
|
||||||
|
validation: &recordingPreparedValidation{
|
||||||
|
results: []domain.ValidationResult{{
|
||||||
|
Status: domain.ValidationFailed,
|
||||||
|
Mode: domain.ValidationJSON,
|
||||||
|
Errors: []string{"invalid"},
|
||||||
|
IsValid: false,
|
||||||
|
}},
|
||||||
|
},
|
||||||
|
repairer: &fakeRepairer{err: repairFailure},
|
||||||
|
wantError: ErrValidation,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, test := range tests {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
def := promptDef(domain.FormatJSON, domain.ValidationJSON, 1)
|
||||||
|
if test.repairer == nil {
|
||||||
|
def.Validation.RepairAttempts = 0
|
||||||
|
}
|
||||||
|
validator := &recordingValidationPreparer{plan: test.validation}
|
||||||
|
admitter := &fakeRunAdmitter{}
|
||||||
|
runner := NewRunnerWithRepairer(
|
||||||
|
&fakePromptRepo{def: def},
|
||||||
|
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
||||||
|
nil,
|
||||||
|
defaultArtifactReader(),
|
||||||
|
defaultRenderer(),
|
||||||
|
&fakeLLM{
|
||||||
|
resp: &domain.GenerateResponse{Content: `{"value":true}`},
|
||||||
|
err: test.generationErr,
|
||||||
|
},
|
||||||
|
validator,
|
||||||
|
test.repairer,
|
||||||
|
admitter,
|
||||||
|
)
|
||||||
|
prepared, err := runner.PrepareExecution(context.Background(), domain.RunRequest{
|
||||||
|
PromptID: "p",
|
||||||
|
ProfileID: "exec",
|
||||||
|
Inputs: singleInputRef(),
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("prepare execution: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
result, err := runner.RunPrepared(context.Background(), prepared)
|
||||||
|
if result != nil || !errors.Is(err, test.wantError) {
|
||||||
|
t.Fatalf("run prepared=(%+v, %v), want %v", result, err, test.wantError)
|
||||||
|
}
|
||||||
|
if len(admitter.backendIDs) != 1 || admitter.releaseCalls != 1 {
|
||||||
|
t.Fatalf(
|
||||||
|
"admission calls=%#v releases=%d, want one each",
|
||||||
|
admitter.backendIDs,
|
||||||
|
admitter.releaseCalls,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunnerPreparedExecutionOwnershipUseAndDiscard(t *testing.T) {
|
||||||
|
newRunner := func() *Runner {
|
||||||
|
return NewRunner(
|
||||||
|
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
|
||||||
|
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
||||||
|
nil,
|
||||||
|
defaultArtifactReader(),
|
||||||
|
defaultRenderer(),
|
||||||
|
&fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}},
|
||||||
|
&recordingValidationPreparer{plan: &recordingPreparedValidation{}},
|
||||||
|
nil,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
request := domain.RunRequest{
|
||||||
|
PromptID: "p",
|
||||||
|
ProfileID: "exec",
|
||||||
|
Inputs: singleInputRef(),
|
||||||
|
}
|
||||||
|
|
||||||
|
owner := newRunner()
|
||||||
|
prepared, err := owner.PrepareExecution(context.Background(), request)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("prepare execution: %v", err)
|
||||||
|
}
|
||||||
|
if _, err := newRunner().RunPrepared(context.Background(), prepared); !errors.Is(err, ErrInvalidRequest) {
|
||||||
|
t.Fatalf("foreign runner error=%v, want ErrInvalidRequest", err)
|
||||||
|
}
|
||||||
|
if _, err := owner.RunPrepared(context.Background(), prepared); err != nil {
|
||||||
|
t.Fatalf("owner run prepared: %v", err)
|
||||||
|
}
|
||||||
|
if _, err := owner.RunPrepared(context.Background(), prepared); !errors.Is(err, ErrInvalidRequest) {
|
||||||
|
t.Fatalf("second owner run error=%v, want ErrInvalidRequest", err)
|
||||||
|
}
|
||||||
|
if prepared.Details() == nil {
|
||||||
|
t.Fatal("details unavailable after execution")
|
||||||
|
}
|
||||||
|
|
||||||
|
discarded, err := owner.PrepareExecution(context.Background(), request)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("prepare discarded execution: %v", err)
|
||||||
|
}
|
||||||
|
discarded.Discard()
|
||||||
|
discarded.Discard()
|
||||||
|
if _, err := owner.RunPrepared(context.Background(), discarded); !errors.Is(err, ErrInvalidRequest) {
|
||||||
|
t.Fatalf("discarded execution error=%v, want ErrInvalidRequest", err)
|
||||||
|
}
|
||||||
|
if discarded.Details() == nil {
|
||||||
|
t.Fatal("details unavailable after discard")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunnerPreparedExecutionWithoutValidatorSkipsValidation(t *testing.T) {
|
||||||
|
runner := NewRunner(
|
||||||
|
&fakePromptRepo{def: promptDef(domain.FormatJSON, domain.ValidationJSON, 0)},
|
||||||
|
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
||||||
|
nil,
|
||||||
|
defaultArtifactReader(),
|
||||||
|
defaultRenderer(),
|
||||||
|
&fakeLLM{resp: &domain.GenerateResponse{Content: `{}`}},
|
||||||
|
nil,
|
||||||
|
nil,
|
||||||
|
)
|
||||||
|
prepared, err := runner.PrepareExecution(context.Background(), domain.RunRequest{
|
||||||
|
PromptID: "p",
|
||||||
|
ProfileID: "exec",
|
||||||
|
Inputs: singleInputRef(),
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("prepare execution: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
result, err := runner.RunPrepared(context.Background(), prepared)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("run prepared: %v", err)
|
||||||
|
}
|
||||||
|
if result.Validation.Status != domain.ValidationSkipped || !result.Validation.IsValid {
|
||||||
|
t.Fatalf("unexpected no-validator result: %+v", result.Validation)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunnerPrepareExecutionRequiresValidationPreparer(t *testing.T) {
|
||||||
|
reader := defaultArtifactReader()
|
||||||
|
runner := NewRunner(
|
||||||
|
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationBasic, 0)},
|
||||||
|
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
||||||
|
nil,
|
||||||
|
reader,
|
||||||
|
defaultRenderer(),
|
||||||
|
&fakeLLM{forbid: true},
|
||||||
|
&fakeValidator{},
|
||||||
|
nil,
|
||||||
|
)
|
||||||
|
|
||||||
|
prepared, err := runner.PrepareExecution(context.Background(), domain.RunRequest{
|
||||||
|
PromptID: "p",
|
||||||
|
ProfileID: "exec",
|
||||||
|
Inputs: singleInputRef(),
|
||||||
|
})
|
||||||
|
if prepared != nil || !errors.Is(err, ErrValidation) {
|
||||||
|
t.Fatalf("prepare execution=(%+v, %v), want ErrValidation", prepared, err)
|
||||||
|
}
|
||||||
|
if reader.calls != 0 {
|
||||||
|
t.Fatalf("unsupported validator allowed completion, artifact calls=%d", reader.calls)
|
||||||
|
}
|
||||||
|
}
|
||||||
103
internal/usecase/profile_inspection.go
Normal file
103
internal/usecase/profile_inspection.go
Normal file
@@ -0,0 +1,103 @@
|
|||||||
|
package usecase
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||||
|
)
|
||||||
|
|
||||||
|
type resolvedProfileSelection struct {
|
||||||
|
id string
|
||||||
|
profile *domain.ExecutionProfile
|
||||||
|
backend *domain.Backend
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *Runner) resolveProfileSelection(
|
||||||
|
ctx context.Context,
|
||||||
|
profileID string,
|
||||||
|
) (*resolvedProfileSelection, error) {
|
||||||
|
normalizedID := strings.TrimSpace(profileID)
|
||||||
|
if normalizedID == "" {
|
||||||
|
return nil, fmt.Errorf("%w: profile id is required", ErrInvalidRequest)
|
||||||
|
}
|
||||||
|
if r == nil || r.profiles == nil {
|
||||||
|
return nil, fmt.Errorf("%w: profile repository is not configured", ErrProfileLoad)
|
||||||
|
}
|
||||||
|
|
||||||
|
selectedProfile, err := r.profiles.GetProfile(ctx, normalizedID)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("%w: %w", ErrProfileLoad, err)
|
||||||
|
}
|
||||||
|
if selectedProfile == nil {
|
||||||
|
return nil, fmt.Errorf("%w: profile repository returned nil profile", ErrProfileLoad)
|
||||||
|
}
|
||||||
|
|
||||||
|
profileValue := *selectedProfile
|
||||||
|
profileValue.BackendID = strings.TrimSpace(profileValue.BackendID)
|
||||||
|
|
||||||
|
var selectedBackend *domain.Backend
|
||||||
|
if profileValue.BackendID != "" {
|
||||||
|
if r.backends == nil {
|
||||||
|
return nil, fmt.Errorf("%w: backend %q cannot be resolved", ErrProfileLoad, profileValue.BackendID)
|
||||||
|
}
|
||||||
|
backendValue, err := r.backends.GetBackend(profileValue.BackendID)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("%w: backend %q: %w", ErrProfileLoad, profileValue.BackendID, err)
|
||||||
|
}
|
||||||
|
selectedBackend = &backendValue
|
||||||
|
}
|
||||||
|
|
||||||
|
return &resolvedProfileSelection{
|
||||||
|
id: normalizedID,
|
||||||
|
profile: &profileValue,
|
||||||
|
backend: selectedBackend,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateResolvedExecutionTarget(target domain.ExecutionTarget) error {
|
||||||
|
if strings.TrimSpace(target.Endpoint) == "" {
|
||||||
|
return errors.New("execution endpoint is required")
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(target.Model) == "" {
|
||||||
|
return errors.New("execution model is required")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// InspectProfile resolves one explicit profile without prompt or execution work.
|
||||||
|
func (r *Runner) InspectProfile(
|
||||||
|
ctx context.Context,
|
||||||
|
profileID string,
|
||||||
|
) (*domain.ProfileInspection, error) {
|
||||||
|
normalizedID := strings.TrimSpace(profileID)
|
||||||
|
if normalizedID == "" {
|
||||||
|
return nil, fmt.Errorf("%w: profile id is required", ErrInvalidRequest)
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return nil, fmt.Errorf("%w: %w", ErrProfileLoad, ctx.Err())
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
|
||||||
|
selection, err := r.resolveProfileSelection(ctx, normalizedID)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
target, _, err := resolveExecutionTarget(selection.backend, selection.profile, nil)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("%w: %w", ErrProfileLoad, err)
|
||||||
|
}
|
||||||
|
if err := validateResolvedExecutionTarget(target); err != nil {
|
||||||
|
return nil, fmt.Errorf("%w: %w", ErrProfileLoad, err)
|
||||||
|
}
|
||||||
|
target.APIKey = ""
|
||||||
|
|
||||||
|
return &domain.ProfileInspection{
|
||||||
|
ProfileID: selection.id,
|
||||||
|
EffectiveModelParams: target,
|
||||||
|
APIKeyRequired: target.APIKeyRequired,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
213
internal/usecase/profile_inspection_test.go
Normal file
213
internal/usecase/profile_inspection_test.go
Normal file
@@ -0,0 +1,213 @@
|
|||||||
|
package usecase
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"reflect"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/defaults"
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/profile"
|
||||||
|
)
|
||||||
|
|
||||||
|
type inspectionProfileRepository struct {
|
||||||
|
profile *domain.ExecutionProfile
|
||||||
|
err error
|
||||||
|
calls int
|
||||||
|
id string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *inspectionProfileRepository) GetProfile(
|
||||||
|
_ context.Context,
|
||||||
|
id string,
|
||||||
|
) (*domain.ExecutionProfile, error) {
|
||||||
|
r.calls++
|
||||||
|
r.id = id
|
||||||
|
if r.err != nil {
|
||||||
|
return nil, r.err
|
||||||
|
}
|
||||||
|
return r.profile, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type inspectionBackendResolver struct {
|
||||||
|
backend domain.Backend
|
||||||
|
err error
|
||||||
|
calls int
|
||||||
|
id string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *inspectionBackendResolver) GetBackend(id string) (domain.Backend, error) {
|
||||||
|
r.calls++
|
||||||
|
r.id = id
|
||||||
|
if r.err != nil {
|
||||||
|
return domain.Backend{}, r.err
|
||||||
|
}
|
||||||
|
return r.backend, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunnerInspectProfileResolvesProfileAndBackendOnce(t *testing.T) {
|
||||||
|
profiles := &inspectionProfileRepository{profile: &domain.ExecutionProfile{
|
||||||
|
ID: "profile",
|
||||||
|
BackendID: " backend ",
|
||||||
|
Model: "profile-model",
|
||||||
|
Temperature: 0.4,
|
||||||
|
MaxTokens: 32,
|
||||||
|
TimeoutSeconds: 45,
|
||||||
|
ServiceTier: "priority",
|
||||||
|
ReasoningEffort: "high",
|
||||||
|
ExtraParams: map[string]any{
|
||||||
|
"profile": "value",
|
||||||
|
},
|
||||||
|
}}
|
||||||
|
backends := &inspectionBackendResolver{backend: domain.Backend{
|
||||||
|
ID: "backend",
|
||||||
|
Endpoint: "https://backend.example/v1",
|
||||||
|
APIKeyEnv: "BACKEND_KEY",
|
||||||
|
ExtraParams: map[string]any{
|
||||||
|
"backend": "value",
|
||||||
|
},
|
||||||
|
}}
|
||||||
|
runner := &Runner{profiles: profiles, backends: backends}
|
||||||
|
|
||||||
|
inspection, err := runner.InspectProfile(context.Background(), " profile ")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("inspect profile: %v", err)
|
||||||
|
}
|
||||||
|
if profiles.calls != 1 || profiles.id != "profile" {
|
||||||
|
t.Fatalf("profile lookup=(calls=%d id=%q), want one exact lookup", profiles.calls, profiles.id)
|
||||||
|
}
|
||||||
|
if backends.calls != 1 || backends.id != "backend" {
|
||||||
|
t.Fatalf("backend lookup=(calls=%d id=%q), want one exact lookup", backends.calls, backends.id)
|
||||||
|
}
|
||||||
|
if profiles.profile.BackendID != " backend " {
|
||||||
|
t.Fatalf("inspection mutated repository profile backend: %q", profiles.profile.BackendID)
|
||||||
|
}
|
||||||
|
|
||||||
|
wantTarget := domain.ExecutionTarget{
|
||||||
|
BackendID: "backend",
|
||||||
|
Endpoint: "https://backend.example/v1",
|
||||||
|
Model: "profile-model",
|
||||||
|
Temperature: 0.4,
|
||||||
|
MaxTokens: 32,
|
||||||
|
TopP: defaults.ExecutionTargetDefault().TopP,
|
||||||
|
TimeoutSeconds: 45,
|
||||||
|
ServiceTier: "priority",
|
||||||
|
ReasoningEffort: "high",
|
||||||
|
APIKeyEnv: "BACKEND_KEY",
|
||||||
|
ExtraParams: map[string]any{
|
||||||
|
"profile": "value",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
if inspection.ProfileID != "profile" || inspection.APIKeyRequired ||
|
||||||
|
!reflect.DeepEqual(inspection.EffectiveModelParams, wantTarget) {
|
||||||
|
t.Fatalf("inspection=%#v, want profile=%q target=%#v", inspection, "profile", wantTarget)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunnerInspectProfileDoesNotNeedExecutionCollaboratorsOrCredentials(t *testing.T) {
|
||||||
|
t.Setenv("PROMPTKIT_INSPECTION_TEST_KEY", "")
|
||||||
|
profiles := &inspectionProfileRepository{profile: &domain.ExecutionProfile{
|
||||||
|
ID: "endpoint-only",
|
||||||
|
Endpoint: "https://profile.example/v1",
|
||||||
|
Model: "profile-model",
|
||||||
|
APIKeyEnv: "PROMPTKIT_INSPECTION_TEST_KEY",
|
||||||
|
}}
|
||||||
|
runner := &Runner{profiles: profiles}
|
||||||
|
|
||||||
|
inspection, err := runner.InspectProfile(context.Background(), "endpoint-only")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("inspect endpoint-only profile: %v", err)
|
||||||
|
}
|
||||||
|
if inspection.EffectiveModelParams.BackendID != "" ||
|
||||||
|
inspection.EffectiveModelParams.APIKeyEnv != "PROMPTKIT_INSPECTION_TEST_KEY" ||
|
||||||
|
inspection.APIKeyRequired {
|
||||||
|
t.Fatalf("unexpected endpoint-only inspection: %#v", inspection)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunnerInspectProfileDirectCredentialRequirementClearsBackendEnvironment(t *testing.T) {
|
||||||
|
profiles := &inspectionProfileRepository{profile: &domain.ExecutionProfile{
|
||||||
|
ID: "direct-key",
|
||||||
|
BackendID: "backend",
|
||||||
|
Model: "profile-model",
|
||||||
|
APIKeyRequired: true,
|
||||||
|
}}
|
||||||
|
backends := &inspectionBackendResolver{backend: domain.Backend{
|
||||||
|
ID: "backend",
|
||||||
|
Endpoint: "https://backend.example/v1",
|
||||||
|
APIKeyEnv: "BACKEND_KEY",
|
||||||
|
}}
|
||||||
|
|
||||||
|
inspection, err := (&Runner{profiles: profiles, backends: backends}).InspectProfile(
|
||||||
|
context.Background(),
|
||||||
|
"direct-key",
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("inspect direct-key profile: %v", err)
|
||||||
|
}
|
||||||
|
if !inspection.APIKeyRequired || inspection.EffectiveModelParams.APIKeyEnv != "" {
|
||||||
|
t.Fatalf("credential requirement was not resolved exclusively: %#v", inspection)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunnerInspectProfileClassifiesFailuresWithoutRepositoryWorkAfterCancellation(t *testing.T) {
|
||||||
|
t.Run("blank ID", func(t *testing.T) {
|
||||||
|
profiles := &inspectionProfileRepository{}
|
||||||
|
_, err := (&Runner{profiles: profiles}).InspectProfile(context.Background(), " \t ")
|
||||||
|
if !errors.Is(err, ErrInvalidRequest) || profiles.calls != 0 {
|
||||||
|
t.Fatalf("blank inspection=(%v, calls=%d), want invalid request without lookup", err, profiles.calls)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("canceled context", func(t *testing.T) {
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
cancel()
|
||||||
|
profiles := &inspectionProfileRepository{}
|
||||||
|
_, err := (&Runner{profiles: profiles}).InspectProfile(ctx, "profile")
|
||||||
|
if !errors.Is(err, ErrProfileLoad) || !errors.Is(err, context.Canceled) || profiles.calls != 0 {
|
||||||
|
t.Fatalf("canceled inspection=(%v, calls=%d), want profile load and context identities without lookup", err, profiles.calls)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("missing profile", func(t *testing.T) {
|
||||||
|
profiles := &inspectionProfileRepository{err: profile.ErrProfileNotFound}
|
||||||
|
_, err := (&Runner{profiles: profiles}).InspectProfile(context.Background(), "missing")
|
||||||
|
if !errors.Is(err, ErrProfileLoad) || !errors.Is(err, profile.ErrProfileNotFound) {
|
||||||
|
t.Fatalf("missing profile error=%v, want profile load and not-found identities", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("unknown backend", func(t *testing.T) {
|
||||||
|
backendErr := errors.New("unknown backend")
|
||||||
|
profiles := &inspectionProfileRepository{profile: &domain.ExecutionProfile{
|
||||||
|
ID: "profile", BackendID: "backend", Model: "profile-model",
|
||||||
|
}}
|
||||||
|
backends := &inspectionBackendResolver{err: backendErr}
|
||||||
|
_, err := (&Runner{profiles: profiles, backends: backends}).InspectProfile(context.Background(), "profile")
|
||||||
|
if !errors.Is(err, ErrProfileLoad) || !errors.Is(err, backendErr) {
|
||||||
|
t.Fatalf("unknown backend error=%v, want profile load and backend identities", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("defensive invalid dependencies", func(t *testing.T) {
|
||||||
|
cases := []struct {
|
||||||
|
name string
|
||||||
|
runner *Runner
|
||||||
|
}{
|
||||||
|
{name: "nil repository", runner: &Runner{}},
|
||||||
|
{name: "nil profile", runner: &Runner{profiles: &inspectionProfileRepository{}}},
|
||||||
|
{name: "invalid target", runner: &Runner{profiles: &inspectionProfileRepository{
|
||||||
|
profile: &domain.ExecutionProfile{ID: "profile", Endpoint: "https://profile.example/v1"},
|
||||||
|
}}},
|
||||||
|
}
|
||||||
|
for _, tc := range cases {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
_, err := tc.runner.InspectProfile(context.Background(), "profile")
|
||||||
|
if !errors.Is(err, ErrProfileLoad) {
|
||||||
|
t.Fatalf("inspection error=%v, want profile load", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
73
internal/usecase/prompt_inspection.go
Normal file
73
internal/usecase/prompt_inspection.go
Normal file
@@ -0,0 +1,73 @@
|
|||||||
|
package usecase
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||||
|
)
|
||||||
|
|
||||||
|
type resolvedPromptDefinition struct {
|
||||||
|
definition *domain.PromptDefinition
|
||||||
|
hash string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *Runner) resolvePromptDefinition(
|
||||||
|
ctx context.Context,
|
||||||
|
promptID string,
|
||||||
|
promptVersion string,
|
||||||
|
) (*resolvedPromptDefinition, error) {
|
||||||
|
if strings.TrimSpace(promptID) == "" {
|
||||||
|
return nil, fmt.Errorf("%w: prompt id is required", ErrInvalidRequest)
|
||||||
|
}
|
||||||
|
if r == nil || r.promptDefs == nil {
|
||||||
|
return nil, fmt.Errorf("%w: prompt repository is not configured", ErrPromptLoad)
|
||||||
|
}
|
||||||
|
|
||||||
|
definition, err := r.promptDefs.GetPromptDefinition(ctx, promptID, promptVersion)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("%w: %w", ErrPromptLoad, err)
|
||||||
|
}
|
||||||
|
if definition == nil {
|
||||||
|
return nil, fmt.Errorf("%w: prompt repository returned nil definition", ErrPromptLoad)
|
||||||
|
}
|
||||||
|
|
||||||
|
hash, err := hashPromptDefinition(definition)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("%w: failed to hash prompt definition: %v", ErrPromptLoad, err)
|
||||||
|
}
|
||||||
|
return &resolvedPromptDefinition{definition: definition, hash: hash}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// InspectPrompt resolves one explicit prompt without execution work.
|
||||||
|
func (r *Runner) InspectPrompt(
|
||||||
|
ctx context.Context,
|
||||||
|
promptID string,
|
||||||
|
promptVersion string,
|
||||||
|
) (*domain.PromptInspection, error) {
|
||||||
|
if strings.TrimSpace(promptID) == "" {
|
||||||
|
return nil, fmt.Errorf("%w: prompt id is required", ErrInvalidRequest)
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return nil, fmt.Errorf("%w: %w", ErrPromptLoad, ctx.Err())
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
|
||||||
|
selection, err := r.resolvePromptDefinition(ctx, promptID, promptVersion)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
inputs := make([]domain.PromptInput, len(selection.definition.Inputs))
|
||||||
|
copy(inputs, selection.definition.Inputs)
|
||||||
|
|
||||||
|
return &domain.PromptInspection{
|
||||||
|
PromptID: selection.definition.ID,
|
||||||
|
PromptVersion: selection.definition.Version,
|
||||||
|
PromptHash: selection.hash,
|
||||||
|
DefaultProfileID: selection.definition.DefaultProfile,
|
||||||
|
Inputs: inputs,
|
||||||
|
OutputContract: selection.definition.Validation,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
157
internal/usecase/prompt_inspection_test.go
Normal file
157
internal/usecase/prompt_inspection_test.go
Normal file
@@ -0,0 +1,157 @@
|
|||||||
|
package usecase
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"reflect"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/promptdef"
|
||||||
|
)
|
||||||
|
|
||||||
|
type inspectionPromptRepository struct {
|
||||||
|
definition *domain.PromptDefinition
|
||||||
|
err error
|
||||||
|
calls int
|
||||||
|
id string
|
||||||
|
version string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *inspectionPromptRepository) GetPromptDefinition(
|
||||||
|
_ context.Context,
|
||||||
|
id string,
|
||||||
|
version string,
|
||||||
|
) (*domain.PromptDefinition, error) {
|
||||||
|
r.calls++
|
||||||
|
r.id = id
|
||||||
|
r.version = version
|
||||||
|
if r.err != nil {
|
||||||
|
return nil, r.err
|
||||||
|
}
|
||||||
|
return r.definition, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunnerInspectPromptResolvesOneDefinitionWithoutExecutionCollaborators(t *testing.T) {
|
||||||
|
definition := &domain.PromptDefinition{
|
||||||
|
ID: "normalized.prompt",
|
||||||
|
Version: "1.2.3",
|
||||||
|
DefaultProfile: "not-resolved",
|
||||||
|
Inputs: []domain.PromptInput{
|
||||||
|
{Name: "document", Required: true, ContentType: "text/plain", Description: "Source document."},
|
||||||
|
{Name: "audience", ContentType: "text/plain", Description: "Intended reader."},
|
||||||
|
},
|
||||||
|
Validation: domain.OutputContract{
|
||||||
|
Format: domain.FormatJSON,
|
||||||
|
ValidationMode: domain.ValidationJSONSchema,
|
||||||
|
SchemaPath: "schemas/result.json",
|
||||||
|
RepairAttempts: 2,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
repository := &inspectionPromptRepository{definition: definition}
|
||||||
|
runner := &Runner{promptDefs: repository}
|
||||||
|
|
||||||
|
inspection, err := runner.InspectPrompt(context.Background(), " prompt-id ", " version ")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("inspect prompt: %v", err)
|
||||||
|
}
|
||||||
|
wantHash, err := hashPromptDefinition(definition)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("hash prompt definition: %v", err)
|
||||||
|
}
|
||||||
|
if repository.calls != 1 || repository.id != " prompt-id " || repository.version != " version " {
|
||||||
|
t.Fatalf("prompt lookup=(calls=%d id=%q version=%q), want one unchanged lookup", repository.calls, repository.id, repository.version)
|
||||||
|
}
|
||||||
|
if inspection.PromptID != definition.ID ||
|
||||||
|
inspection.PromptVersion != definition.Version ||
|
||||||
|
inspection.PromptHash != wantHash ||
|
||||||
|
inspection.DefaultProfileID != definition.DefaultProfile ||
|
||||||
|
!reflect.DeepEqual(inspection.Inputs, definition.Inputs) ||
|
||||||
|
inspection.OutputContract != definition.Validation {
|
||||||
|
t.Fatalf("inspection=%#v, want definition metadata", inspection)
|
||||||
|
}
|
||||||
|
|
||||||
|
inspection.Inputs[0].Name = "changed"
|
||||||
|
second, err := runner.InspectPrompt(context.Background(), " prompt-id ", " version ")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("inspect prompt again: %v", err)
|
||||||
|
}
|
||||||
|
if definition.Inputs[0].Name != "document" || second.Inputs[0].Name != "document" {
|
||||||
|
t.Fatalf("inspection input mutation escaped caller result: definition=%#v next=%#v", definition.Inputs, second.Inputs)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunnerInspectPromptClassifiesFailuresWithoutRepositoryWorkAfterCancellation(t *testing.T) {
|
||||||
|
t.Run("blank ID", func(t *testing.T) {
|
||||||
|
repository := &inspectionPromptRepository{}
|
||||||
|
_, err := (&Runner{promptDefs: repository}).InspectPrompt(context.Background(), " \t ", "version")
|
||||||
|
if !errors.Is(err, ErrInvalidRequest) || repository.calls != 0 {
|
||||||
|
t.Fatalf("blank inspection=(%v, calls=%d), want invalid request without lookup", err, repository.calls)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("canceled context", func(t *testing.T) {
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
cancel()
|
||||||
|
repository := &inspectionPromptRepository{}
|
||||||
|
_, err := (&Runner{promptDefs: repository}).InspectPrompt(ctx, "prompt", "version")
|
||||||
|
if !errors.Is(err, ErrPromptLoad) || !errors.Is(err, context.Canceled) || repository.calls != 0 {
|
||||||
|
t.Fatalf("canceled inspection=(%v, calls=%d), want prompt load and context identities without lookup", err, repository.calls)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("missing prompt", func(t *testing.T) {
|
||||||
|
repository := &inspectionPromptRepository{err: promptdef.ErrPromptDefinitionNotFound}
|
||||||
|
_, err := (&Runner{promptDefs: repository}).InspectPrompt(context.Background(), "missing", "version")
|
||||||
|
if !errors.Is(err, ErrPromptLoad) || !errors.Is(err, promptdef.ErrPromptDefinitionNotFound) {
|
||||||
|
t.Fatalf("missing prompt error=%v, want prompt load and not-found identities", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("defensive prompt dependencies", func(t *testing.T) {
|
||||||
|
var nilRunner *Runner
|
||||||
|
cases := []struct {
|
||||||
|
name string
|
||||||
|
runner *Runner
|
||||||
|
}{
|
||||||
|
{name: "nil runner", runner: nilRunner},
|
||||||
|
{name: "nil repository", runner: &Runner{}},
|
||||||
|
{name: "nil definition", runner: &Runner{promptDefs: &inspectionPromptRepository{}}},
|
||||||
|
}
|
||||||
|
for _, tc := range cases {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
_, err := tc.runner.InspectPrompt(context.Background(), "prompt", "version")
|
||||||
|
if !errors.Is(err, ErrPromptLoad) {
|
||||||
|
t.Fatalf("inspection error=%v, want prompt load", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunnerPrepareUsesThePromptInspectionSelectionAndHash(t *testing.T) {
|
||||||
|
definition := promptDef(domain.FormatMarkdown, domain.ValidationBasic, 0)
|
||||||
|
repository := &fakePromptRepo{def: definition}
|
||||||
|
runner := NewRunner(
|
||||||
|
repository,
|
||||||
|
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
||||||
|
nil,
|
||||||
|
&fakeArtifactReader{},
|
||||||
|
&fakeRenderer{rendered: &domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hello"}}}},
|
||||||
|
nil,
|
||||||
|
nil,
|
||||||
|
nil,
|
||||||
|
)
|
||||||
|
|
||||||
|
inspection, err := runner.InspectPrompt(context.Background(), definition.ID, definition.Version)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("inspect prompt: %v", err)
|
||||||
|
}
|
||||||
|
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{PromptID: definition.ID, PromptVersion: definition.Version, ProfileID: "exec"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("prepare prompt: %v", err)
|
||||||
|
}
|
||||||
|
if inspection.PromptHash != prepared.PromptHash {
|
||||||
|
t.Fatalf("inspection hash=%q, preparation hash=%q", inspection.PromptHash, prepared.PromptHash)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -131,29 +131,46 @@ func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunRes
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
if r.admitter != nil {
|
release, err := r.admitRun(ctx, state.effectiveModel.BackendID)
|
||||||
release, admitErr := r.admitter.Admit(ctx, state.effectiveModel.BackendID)
|
if err != nil {
|
||||||
if admitErr != nil {
|
return nil, err
|
||||||
if errors.Is(admitErr, capacity.ErrCapacityExceeded) {
|
|
||||||
return nil, fmt.Errorf(
|
|
||||||
"backend %q admission: %w",
|
|
||||||
state.effectiveModel.BackendID,
|
|
||||||
admitErr,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
return nil, admitErr
|
|
||||||
}
|
|
||||||
defer release()
|
|
||||||
}
|
}
|
||||||
|
defer release()
|
||||||
|
|
||||||
prepared, err := r.completePreparation(ctx, req, state)
|
prepared, err := r.completePreparation(ctx, req, state)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
directAPIKey := state.effectiveModel.APIKey
|
||||||
|
|
||||||
|
return r.executePreparedRun(ctx, prepared, directAPIKey, runID, start, func(
|
||||||
|
ctx context.Context,
|
||||||
|
artifact *domain.Artifact,
|
||||||
|
attemptsUsed int,
|
||||||
|
) (domain.ValidationResult, error) {
|
||||||
|
return r.validateOutput(ctx, artifact, prepared.OutputContract, attemptsUsed)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
type preparedValidationFunc func(
|
||||||
|
context.Context,
|
||||||
|
*domain.Artifact,
|
||||||
|
int,
|
||||||
|
) (domain.ValidationResult, error)
|
||||||
|
|
||||||
|
func (r *Runner) executePreparedRun(
|
||||||
|
ctx context.Context,
|
||||||
|
prepared *domain.PreparedRun,
|
||||||
|
directAPIKey string,
|
||||||
|
runID string,
|
||||||
|
start time.Time,
|
||||||
|
validateArtifact preparedValidationFunc,
|
||||||
|
) (*domain.RunResult, error) {
|
||||||
|
executionTarget := prepared.EffectiveModelParams
|
||||||
|
executionTarget.APIKey = directAPIKey
|
||||||
genResp, err := r.llm.Generate(ctx, domain.GenerateRequest{
|
genResp, err := r.llm.Generate(ctx, domain.GenerateRequest{
|
||||||
Prompt: domain.RenderedPrompt{SessionID: prepared.SessionID, Messages: prepared.Messages},
|
Prompt: domain.RenderedPrompt{SessionID: prepared.SessionID, Messages: prepared.Messages},
|
||||||
Target: prepared.EffectiveModelParams,
|
Target: executionTarget,
|
||||||
TargetPresence: prepared.TargetPresence,
|
TargetPresence: prepared.TargetPresence,
|
||||||
StructuredOutput: prepared.StructuredOutput,
|
StructuredOutput: prepared.StructuredOutput,
|
||||||
})
|
})
|
||||||
@@ -165,7 +182,7 @@ func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunRes
|
|||||||
}
|
}
|
||||||
|
|
||||||
outputArtifact := buildOutputArtifact(genResp.Content, prepared.OutputContract.Format)
|
outputArtifact := buildOutputArtifact(genResp.Content, prepared.OutputContract.Format)
|
||||||
validationResult, err := r.validateOutput(ctx, &outputArtifact, prepared.OutputContract, 0)
|
validationResult, err := validateArtifact(ctx, &outputArtifact, 0)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("%w: %w", ErrValidation, err)
|
return nil, fmt.Errorf("%w: %w", ErrValidation, err)
|
||||||
}
|
}
|
||||||
@@ -179,7 +196,7 @@ func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunRes
|
|||||||
PreviousOutput: genResp.Content,
|
PreviousOutput: genResp.Content,
|
||||||
ValidationErrors: validationResult.Errors,
|
ValidationErrors: validationResult.Errors,
|
||||||
SessionID: prepared.SessionID,
|
SessionID: prepared.SessionID,
|
||||||
Target: prepared.EffectiveModelParams,
|
Target: executionTarget,
|
||||||
StructuredOutput: prepared.StructuredOutput,
|
StructuredOutput: prepared.StructuredOutput,
|
||||||
Attempt: attemptsUsed,
|
Attempt: attemptsUsed,
|
||||||
MaxAttempts: prepared.OutputContract.RepairAttempts,
|
MaxAttempts: prepared.OutputContract.RepairAttempts,
|
||||||
@@ -195,7 +212,7 @@ func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunRes
|
|||||||
genResp = repairResp
|
genResp = repairResp
|
||||||
outputArtifact = buildOutputArtifact(genResp.Content, prepared.OutputContract.Format)
|
outputArtifact = buildOutputArtifact(genResp.Content, prepared.OutputContract.Format)
|
||||||
|
|
||||||
validationResult, err = r.validateOutput(ctx, &outputArtifact, prepared.OutputContract, attemptsUsed)
|
validationResult, err = validateArtifact(ctx, &outputArtifact, attemptsUsed)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("%w: %w", ErrValidation, err)
|
return nil, fmt.Errorf("%w: %w", ErrValidation, err)
|
||||||
}
|
}
|
||||||
@@ -203,6 +220,7 @@ func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunRes
|
|||||||
}
|
}
|
||||||
|
|
||||||
end := time.Now().UTC()
|
end := time.Now().UTC()
|
||||||
|
executionTarget.APIKey = ""
|
||||||
|
|
||||||
return &domain.RunResult{
|
return &domain.RunResult{
|
||||||
RunID: runID,
|
RunID: runID,
|
||||||
@@ -218,7 +236,7 @@ func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunRes
|
|||||||
SelectedBackendID: prepared.SelectedBackendID,
|
SelectedBackendID: prepared.SelectedBackendID,
|
||||||
ModelName: prepared.EffectiveModelParams.Model,
|
ModelName: prepared.EffectiveModelParams.Model,
|
||||||
Endpoint: prepared.EffectiveModelParams.Endpoint,
|
Endpoint: prepared.EffectiveModelParams.Endpoint,
|
||||||
EffectiveModelParams: prepared.EffectiveModelParams,
|
EffectiveModelParams: executionTarget,
|
||||||
InputHashes: prepared.InputHashes,
|
InputHashes: prepared.InputHashes,
|
||||||
Usage: genResp.Usage,
|
Usage: genResp.Usage,
|
||||||
StartTime: start,
|
StartTime: start,
|
||||||
@@ -248,14 +266,12 @@ func (r *Runner) resolvePreparation(
|
|||||||
return nil, fmt.Errorf("%w: session_id: %v", ErrInvalidRequest, err)
|
return nil, fmt.Errorf("%w: session_id: %v", ErrInvalidRequest, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
def, err := r.promptDefs.GetPromptDefinition(ctx, req.PromptID, req.PromptVersion)
|
promptSelection, err := r.resolvePromptDefinition(ctx, req.PromptID, req.PromptVersion)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("%w: %w", ErrPromptLoad, err)
|
return nil, err
|
||||||
}
|
|
||||||
promptDefinitionHash, err := hashPromptDefinition(def)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("%w: failed to hash prompt definition: %v", ErrPromptLoad, err)
|
|
||||||
}
|
}
|
||||||
|
def := promptSelection.definition
|
||||||
|
promptDefinitionHash := promptSelection.hash
|
||||||
|
|
||||||
selectedProfileID := strings.TrimSpace(req.ProfileID)
|
selectedProfileID := strings.TrimSpace(req.ProfileID)
|
||||||
if selectedProfileID == "" {
|
if selectedProfileID == "" {
|
||||||
@@ -265,34 +281,18 @@ func (r *Runner) resolvePreparation(
|
|||||||
return nil, fmt.Errorf("%w: %w: profile id is required either in request or prompt default_profile", ErrInvalidRequest, ErrProfileRequired)
|
return nil, fmt.Errorf("%w: %w: profile id is required either in request or prompt default_profile", ErrInvalidRequest, ErrProfileRequired)
|
||||||
}
|
}
|
||||||
|
|
||||||
execProfile, err := r.profiles.GetProfile(ctx, selectedProfileID)
|
selection, err := r.resolveProfileSelection(ctx, selectedProfileID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("%w: %w", ErrProfileLoad, err)
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
var selectedBackend *domain.Backend
|
effectiveModel, targetPresence, err := resolveExecutionTarget(selection.backend, selection.profile, req.Execution)
|
||||||
if backendID := strings.TrimSpace(execProfile.BackendID); backendID != "" {
|
|
||||||
execProfile.BackendID = backendID
|
|
||||||
if r.backends == nil {
|
|
||||||
return nil, fmt.Errorf("%w: backend %q cannot be resolved", ErrProfileLoad, backendID)
|
|
||||||
}
|
|
||||||
resolvedBackend, resolveErr := r.backends.GetBackend(backendID)
|
|
||||||
if resolveErr != nil {
|
|
||||||
return nil, fmt.Errorf("%w: backend %q: %w", ErrProfileLoad, backendID, resolveErr)
|
|
||||||
}
|
|
||||||
selectedBackend = &resolvedBackend
|
|
||||||
}
|
|
||||||
|
|
||||||
effectiveModel, targetPresence, err := resolveExecutionTarget(selectedBackend, execProfile, req.Execution)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("%w: %w", ErrInvalidRequest, err)
|
return nil, fmt.Errorf("%w: %w", ErrInvalidRequest, err)
|
||||||
}
|
}
|
||||||
effectiveModel.APIKey = req.APIKey
|
effectiveModel.APIKey = req.APIKey
|
||||||
if strings.TrimSpace(effectiveModel.Endpoint) == "" {
|
if err := validateResolvedExecutionTarget(effectiveModel); err != nil {
|
||||||
return nil, fmt.Errorf("%w: execution endpoint is required", ErrInvalidRequest)
|
return nil, fmt.Errorf("%w: %w", ErrInvalidRequest, err)
|
||||||
}
|
|
||||||
if strings.TrimSpace(effectiveModel.Model) == "" {
|
|
||||||
return nil, fmt.Errorf("%w: execution model is required", ErrInvalidRequest)
|
|
||||||
}
|
}
|
||||||
if err := validateAPIKey(effectiveModel.APIKeyEnv, effectiveModel.APIKey, effectiveModel.APIKeyRequired); err != nil {
|
if err := validateAPIKey(effectiveModel.APIKeyEnv, effectiveModel.APIKey, effectiveModel.APIKeyRequired); err != nil {
|
||||||
return nil, fmt.Errorf("%w: %w", ErrInvalidRequest, err)
|
return nil, fmt.Errorf("%w: %w", ErrInvalidRequest, err)
|
||||||
@@ -303,7 +303,7 @@ func (r *Runner) resolvePreparation(
|
|||||||
definition: def,
|
definition: def,
|
||||||
directSessionID: directSessionID,
|
directSessionID: directSessionID,
|
||||||
promptDefinitionHash: promptDefinitionHash,
|
promptDefinitionHash: promptDefinitionHash,
|
||||||
selectedProfileID: selectedProfileID,
|
selectedProfileID: selection.id,
|
||||||
effectiveModel: effectiveModel,
|
effectiveModel: effectiveModel,
|
||||||
targetPresence: targetPresence,
|
targetPresence: targetPresence,
|
||||||
effectiveContract: effectiveContract,
|
effectiveContract: effectiveContract,
|
||||||
@@ -324,7 +324,15 @@ func (r *Runner) completePreparation(
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
return r.completePreparationWithStructuredOutput(ctx, req, state, structuredOutput)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *Runner) completePreparationWithStructuredOutput(
|
||||||
|
ctx context.Context,
|
||||||
|
req domain.RunRequest,
|
||||||
|
state *preparationState,
|
||||||
|
structuredOutput *domain.StructuredOutputSpec,
|
||||||
|
) (*domain.PreparedRun, error) {
|
||||||
resolvedInputs := make(map[string]*domain.Artifact, len(req.Inputs))
|
resolvedInputs := make(map[string]*domain.Artifact, len(req.Inputs))
|
||||||
inputHashes := make(map[string]string, len(req.Inputs))
|
inputHashes := make(map[string]string, len(req.Inputs))
|
||||||
for name, ref := range req.Inputs {
|
for name, ref := range req.Inputs {
|
||||||
@@ -354,13 +362,15 @@ func (r *Runner) completePreparation(
|
|||||||
}
|
}
|
||||||
|
|
||||||
end := time.Now().UTC()
|
end := time.Now().UTC()
|
||||||
|
effectiveModel := state.effectiveModel
|
||||||
|
effectiveModel.APIKey = ""
|
||||||
return &domain.PreparedRun{
|
return &domain.PreparedRun{
|
||||||
PromptID: state.definition.ID,
|
PromptID: state.definition.ID,
|
||||||
PromptVersion: state.definition.Version,
|
PromptVersion: state.definition.Version,
|
||||||
PromptHash: state.promptDefinitionHash,
|
PromptHash: state.promptDefinitionHash,
|
||||||
SelectedProfileID: state.selectedProfileID,
|
SelectedProfileID: state.selectedProfileID,
|
||||||
SelectedBackendID: state.effectiveModel.BackendID,
|
SelectedBackendID: state.effectiveModel.BackendID,
|
||||||
EffectiveModelParams: state.effectiveModel,
|
EffectiveModelParams: effectiveModel,
|
||||||
TargetPresence: state.targetPresence,
|
TargetPresence: state.targetPresence,
|
||||||
OutputContract: state.effectiveContract,
|
OutputContract: state.effectiveContract,
|
||||||
StructuredOutput: structuredOutput,
|
StructuredOutput: structuredOutput,
|
||||||
@@ -374,6 +384,20 @@ func (r *Runner) completePreparation(
|
|||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (r *Runner) admitRun(ctx context.Context, backendID string) (func(), error) {
|
||||||
|
if r.admitter == nil {
|
||||||
|
return func() {}, nil
|
||||||
|
}
|
||||||
|
release, err := r.admitter.Admit(ctx, backendID)
|
||||||
|
if err != nil {
|
||||||
|
if errors.Is(err, capacity.ErrCapacityExceeded) && strings.TrimSpace(backendID) != "" {
|
||||||
|
return nil, &CapacityError{BackendID: backendID}
|
||||||
|
}
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return release, nil
|
||||||
|
}
|
||||||
|
|
||||||
func (r *Runner) resolveStructuredOutput(ctx context.Context, def *domain.PromptDefinition, contract domain.OutputContract) (*domain.StructuredOutputSpec, error) {
|
func (r *Runner) resolveStructuredOutput(ctx context.Context, def *domain.PromptDefinition, contract domain.OutputContract) (*domain.StructuredOutputSpec, error) {
|
||||||
if contract.ValidationMode != domain.ValidationJSONSchema {
|
if contract.ValidationMode != domain.ValidationJSONSchema {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
@@ -389,14 +413,18 @@ func (r *Runner) resolveStructuredOutput(ctx context.Context, def *domain.Prompt
|
|||||||
return nil, fmt.Errorf("%w: failed to load json schema for structured output: %v", ErrValidation, err)
|
return nil, fmt.Errorf("%w: failed to load json schema for structured output: %v", ErrValidation, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
return structuredOutputSpec(def, schemaDoc), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func structuredOutputSpec(def *domain.PromptDefinition, schemaDocument any) *domain.StructuredOutputSpec {
|
||||||
return &domain.StructuredOutputSpec{
|
return &domain.StructuredOutputSpec{
|
||||||
Type: domain.StructuredOutputJSONSchema,
|
Type: domain.StructuredOutputJSONSchema,
|
||||||
JSONSchema: &domain.StructuredOutputJSONSpec{
|
JSONSchema: &domain.StructuredOutputJSONSpec{
|
||||||
Name: deriveStructuredSchemaName(def.ID, def.Version),
|
Name: deriveStructuredSchemaName(def.ID, def.Version),
|
||||||
Strict: true,
|
Strict: true,
|
||||||
Schema: schemaDoc,
|
Schema: schemaDocument,
|
||||||
},
|
},
|
||||||
}, nil
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func deriveStructuredSchemaName(promptID string, promptVersion string) string {
|
func deriveStructuredSchemaName(promptID string, promptVersion string) string {
|
||||||
|
|||||||
@@ -1357,14 +1357,14 @@ func TestRunnerAdmissionUsesResolvedBackendIdentity(t *testing.T) {
|
|||||||
|
|
||||||
func TestRunnerAdmissionFailureSkipsCompletionCollaborators(t *testing.T) {
|
func TestRunnerAdmissionFailureSkipsCompletionCollaborators(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
admissionError error
|
admissionError error
|
||||||
wantBackendContext bool
|
wantCapacityType bool
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
name: "capacity exhausted",
|
name: "capacity exhausted",
|
||||||
admissionError: capacity.ErrCapacityExceeded,
|
admissionError: capacity.ErrCapacityExceeded,
|
||||||
wantBackendContext: true,
|
wantCapacityType: true,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "context canceled",
|
name: "context canceled",
|
||||||
@@ -1414,8 +1414,16 @@ func TestRunnerAdmissionFailureSkipsCompletionCollaborators(t *testing.T) {
|
|||||||
if errors.Is(err, ErrInvalidRequest) || errors.Is(err, ErrLLMGenerate) {
|
if errors.Is(err, ErrInvalidRequest) || errors.Is(err, ErrLLMGenerate) {
|
||||||
t.Fatalf("admission error was recategorized: %v", err)
|
t.Fatalf("admission error was recategorized: %v", err)
|
||||||
}
|
}
|
||||||
if tc.wantBackendContext && !strings.Contains(err.Error(), "custom") {
|
var capacityErr *CapacityError
|
||||||
t.Fatalf("capacity error lacks backend context: %v", err)
|
if tc.wantCapacityType {
|
||||||
|
if !errors.As(err, &capacityErr) {
|
||||||
|
t.Fatalf("capacity error=%v, want internal typed identity", err)
|
||||||
|
}
|
||||||
|
if capacityErr.BackendID != "custom" {
|
||||||
|
t.Fatalf("capacity backend ID=%q, want custom", capacityErr.BackendID)
|
||||||
|
}
|
||||||
|
} else if errors.As(err, &capacityErr) {
|
||||||
|
t.Fatalf("non-capacity admission error exposed typed capacity identity: %v", err)
|
||||||
}
|
}
|
||||||
if !reflect.DeepEqual(admitter.backendIDs, []string{"custom"}) {
|
if !reflect.DeepEqual(admitter.backendIDs, []string{"custom"}) {
|
||||||
t.Fatalf("admitted backend IDs=%#v, want custom", admitter.backendIDs)
|
t.Fatalf("admitted backend IDs=%#v, want custom", admitter.backendIDs)
|
||||||
@@ -1682,36 +1690,6 @@ func TestRunnerRunSelectedProfileBeatsBuiltInDefault(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRunnerRunBuiltInDefaultsUsedWhenProfileOmitsOptionalFields(t *testing.T) {
|
|
||||||
promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}
|
|
||||||
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
|
|
||||||
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model"},
|
|
||||||
}}
|
|
||||||
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
|
|
||||||
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), llmClient, nil, nil)
|
|
||||||
|
|
||||||
res, err := runner.Run(context.Background(), domain.RunRequest{
|
|
||||||
PromptID: "p",
|
|
||||||
ProfileID: "exec",
|
|
||||||
Inputs: singleInputRef(),
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("expected no error, got %v", err)
|
|
||||||
}
|
|
||||||
if res.EffectiveModelParams.Temperature != defaults.ExecutionDefaultTemperature {
|
|
||||||
t.Fatalf("expected default temperature %v, got %v", defaults.ExecutionDefaultTemperature, res.EffectiveModelParams.Temperature)
|
|
||||||
}
|
|
||||||
if res.EffectiveModelParams.TopP != defaults.ExecutionDefaultTopP {
|
|
||||||
t.Fatalf("expected default top_p %v, got %v", defaults.ExecutionDefaultTopP, res.EffectiveModelParams.TopP)
|
|
||||||
}
|
|
||||||
if res.EffectiveModelParams.MaxTokens != defaults.ExecutionDefaultMaxTokens {
|
|
||||||
t.Fatalf("expected default max_tokens %d, got %d", defaults.ExecutionDefaultMaxTokens, res.EffectiveModelParams.MaxTokens)
|
|
||||||
}
|
|
||||||
if res.EffectiveModelParams.TimeoutSeconds != defaults.ExecutionDefaultTimeoutSeconds {
|
|
||||||
t.Fatalf("expected default timeout_seconds %d, got %d", defaults.ExecutionDefaultTimeoutSeconds, res.EffectiveModelParams.TimeoutSeconds)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunnerRunAPIKeyEnvResolvesFromEnvironment(t *testing.T) {
|
func TestRunnerRunAPIKeyEnvResolvesFromEnvironment(t *testing.T) {
|
||||||
t.Setenv("PROMPTKIT_TEST_API_KEY", "secret")
|
t.Setenv("PROMPTKIT_TEST_API_KEY", "secret")
|
||||||
promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}
|
promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}
|
||||||
@@ -1757,7 +1735,7 @@ func TestRunnerRunDirectAPIKeyBypassesMissingEnvAndReachesLLM(t *testing.T) {
|
|||||||
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
|
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
|
||||||
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), llmClient, nil, nil)
|
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), llmClient, nil, nil)
|
||||||
|
|
||||||
_, err := runner.Run(context.Background(), domain.RunRequest{
|
result, err := runner.Run(context.Background(), domain.RunRequest{
|
||||||
PromptID: "p",
|
PromptID: "p",
|
||||||
ProfileID: "exec",
|
ProfileID: "exec",
|
||||||
APIKey: directKey,
|
APIKey: directKey,
|
||||||
@@ -1769,6 +1747,9 @@ func TestRunnerRunDirectAPIKeyBypassesMissingEnvAndReachesLLM(t *testing.T) {
|
|||||||
if llmClient.lastReq.Target.APIKey != directKey {
|
if llmClient.lastReq.Target.APIKey != directKey {
|
||||||
t.Fatalf("expected direct API key to reach LLM request")
|
t.Fatalf("expected direct API key to reach LLM request")
|
||||||
}
|
}
|
||||||
|
if result.EffectiveModelParams.APIKey != "" {
|
||||||
|
t.Fatal("run result retained direct API key")
|
||||||
|
}
|
||||||
if llmClient.lastReq.Target.APIKeyEnv != "PROMPTKIT_MISSING_KEY" {
|
if llmClient.lastReq.Target.APIKeyEnv != "PROMPTKIT_MISSING_KEY" {
|
||||||
t.Fatalf("expected api_key_env name to remain on target, got %q", llmClient.lastReq.Target.APIKeyEnv)
|
t.Fatalf("expected api_key_env name to remain on target, got %q", llmClient.lastReq.Target.APIKeyEnv)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -45,6 +45,101 @@ func (v *FSValidator) Validate(ctx context.Context, artifact *domain.Artifact, c
|
|||||||
return validateArtifact(ctx, artifact, contract, v.validateJSONSchema)
|
return validateArtifact(ctx, artifact, contract, v.validateJSONSchema)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type preparedValidation struct {
|
||||||
|
contract domain.OutputContract
|
||||||
|
schemaDocument any
|
||||||
|
schema *jsonschema.Schema
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *preparedValidation) Validate(ctx context.Context, artifact *domain.Artifact) (domain.ValidationResult, error) {
|
||||||
|
return validateArtifact(ctx, artifact, p.contract, p.validateJSONSchema)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *preparedValidation) SchemaDocument() any {
|
||||||
|
return p.schemaDocument
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *preparedValidation) validateJSONSchema(instance any, _ string) ([]string, error) {
|
||||||
|
if p.schema == nil {
|
||||||
|
return nil, errors.New("prepared JSON schema is unavailable")
|
||||||
|
}
|
||||||
|
if err := p.schema.Validate(instance); err != nil {
|
||||||
|
return []string{fmt.Sprintf("json schema validation failed: %v", err)}, nil
|
||||||
|
}
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (v *StandardValidator) PrepareValidation(ctx context.Context, contract domain.OutputContract) (PreparedValidation, error) {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
prepared := &preparedValidation{contract: contract}
|
||||||
|
if contract.ValidationMode != domain.ValidationJSONSchema {
|
||||||
|
return prepared, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
resolvedSchemaPath, err := v.resolveSchemaPath(contract.SchemaPath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
schemaDocument, err := loadJSONSchemaFile(resolvedSchemaPath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to compile JSON schema %q: %w", resolvedSchemaPath, err)
|
||||||
|
}
|
||||||
|
schemaRoot, err := v.schemaRoot()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
compiler := newSchemaCompiler(standardSchemaLoader{root: schemaRoot})
|
||||||
|
if err := compiler.AddResource(resolvedSchemaPath, schemaDocument); err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to register JSON schema %q: %w", resolvedSchemaPath, err)
|
||||||
|
}
|
||||||
|
schema, err := compiler.Compile(resolvedSchemaPath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to compile JSON schema %q: %w", resolvedSchemaPath, err)
|
||||||
|
}
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
prepared.schemaDocument = schemaDocument
|
||||||
|
prepared.schema = schema
|
||||||
|
return prepared, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (v *FSValidator) PrepareValidation(ctx context.Context, contract domain.OutputContract) (PreparedValidation, error) {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
prepared := &preparedValidation{contract: contract}
|
||||||
|
if contract.ValidationMode != domain.ValidationJSONSchema {
|
||||||
|
return prepared, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
schemaName, schemaDocument, err := v.loadSchemaDocument(contract.SchemaPath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
resourceURL := fsSchemaResourceURL(schemaName)
|
||||||
|
compiler := newSchemaCompiler(fsSchemaLoader{fsys: v.fsys, root: filecatalog.CleanFSRoot(v.root)})
|
||||||
|
if err := compiler.AddResource(resourceURL, schemaDocument); err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to register JSON schema %q: %w", schemaName, err)
|
||||||
|
}
|
||||||
|
schema, err := compiler.Compile(resourceURL)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to compile JSON schema %q: %w", schemaName, err)
|
||||||
|
}
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
prepared.schemaDocument = schemaDocument
|
||||||
|
prepared.schema = schema
|
||||||
|
return prepared, nil
|
||||||
|
}
|
||||||
|
|
||||||
type schemaValidatorFunc func(instance any, schemaPath string) ([]string, error)
|
type schemaValidatorFunc func(instance any, schemaPath string) ([]string, error)
|
||||||
|
|
||||||
func validateArtifact(ctx context.Context, artifact *domain.Artifact, contract domain.OutputContract, validateSchema schemaValidatorFunc) (domain.ValidationResult, error) {
|
func validateArtifact(ctx context.Context, artifact *domain.Artifact, contract domain.OutputContract, validateSchema schemaValidatorFunc) (domain.ValidationResult, error) {
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ import (
|
|||||||
"encoding/json"
|
"encoding/json"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"reflect"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
@@ -120,6 +121,68 @@ func TestStandardValidatorJSONSchemaSuccess(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestStandardValidatorPreparedSchemaSurvivesSourceRemoval(t *testing.T) {
|
||||||
|
tmp := t.TempDir()
|
||||||
|
rootSchema := []byte(`{
|
||||||
|
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||||
|
"title": "original root",
|
||||||
|
"type": "object",
|
||||||
|
"required": ["value"],
|
||||||
|
"properties": {
|
||||||
|
"value": {"$ref": "value.json"}
|
||||||
|
}
|
||||||
|
}`)
|
||||||
|
rootPath := filepath.Join(tmp, "schema.json")
|
||||||
|
referencePath := filepath.Join(tmp, "value.json")
|
||||||
|
if err := os.WriteFile(rootPath, rootSchema, 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(referencePath, []byte(`{
|
||||||
|
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||||
|
"type": "integer",
|
||||||
|
"minimum": 2
|
||||||
|
}`), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
validator := NewStandardValidator(tmp)
|
||||||
|
preparer, ok := validator.(ValidationPreparer)
|
||||||
|
if !ok {
|
||||||
|
t.Fatal("standard validator does not support validation preparation")
|
||||||
|
}
|
||||||
|
prepared, err := preparer.PrepareValidation(context.Background(), domain.OutputContract{
|
||||||
|
ValidationMode: domain.ValidationJSONSchema,
|
||||||
|
SchemaPath: "schema.json",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("prepare validation: %v", err)
|
||||||
|
}
|
||||||
|
assertSchemaDocument(t, prepared.SchemaDocument(), rootSchema)
|
||||||
|
|
||||||
|
if err := os.Remove(rootPath); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.Remove(referencePath); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
valid, err := prepared.Validate(context.Background(), &domain.Artifact{Body: []byte(`{"value":3}`)})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("validate prepared artifact: %v", err)
|
||||||
|
}
|
||||||
|
if valid.Status != domain.ValidationPassed || !valid.IsValid {
|
||||||
|
t.Fatalf("expected passed/valid, got status=%q valid=%v errors=%v", valid.Status, valid.IsValid, valid.Errors)
|
||||||
|
}
|
||||||
|
|
||||||
|
invalid, err := prepared.Validate(context.Background(), &domain.Artifact{Body: []byte(`{"value":"changed"}`)})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("validate prepared artifact: %v", err)
|
||||||
|
}
|
||||||
|
if invalid.Status != domain.ValidationFailed || invalid.IsValid {
|
||||||
|
t.Fatalf("expected failed/invalid, got status=%q valid=%v", invalid.Status, invalid.IsValid)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestStandardValidatorJSONSchemaNestedSchemaPathSuccess(t *testing.T) {
|
func TestStandardValidatorJSONSchemaNestedSchemaPathSuccess(t *testing.T) {
|
||||||
tmp := t.TempDir()
|
tmp := t.TempDir()
|
||||||
nestedDir := filepath.Join(tmp, "dnd")
|
nestedDir := filepath.Join(tmp, "dnd")
|
||||||
@@ -295,6 +358,64 @@ func TestFSValidatorJSONSchemaSuccess(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestFSValidatorPreparedSchemaSurvivesSourceMutation(t *testing.T) {
|
||||||
|
rootSchema := []byte(`{
|
||||||
|
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||||
|
"title": "original root",
|
||||||
|
"type": "object",
|
||||||
|
"required": ["value"],
|
||||||
|
"properties": {
|
||||||
|
"value": {"$ref": "value.json"}
|
||||||
|
}
|
||||||
|
}`)
|
||||||
|
fsys := fstest.MapFS{
|
||||||
|
"schema.json": &fstest.MapFile{Data: rootSchema},
|
||||||
|
"value.json": &fstest.MapFile{Data: []byte(`{
|
||||||
|
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||||
|
"type": "integer",
|
||||||
|
"minimum": 2
|
||||||
|
}`)},
|
||||||
|
}
|
||||||
|
validator := NewFSValidator(fsys, ".")
|
||||||
|
preparer, ok := validator.(ValidationPreparer)
|
||||||
|
if !ok {
|
||||||
|
t.Fatal("filesystem validator does not support validation preparation")
|
||||||
|
}
|
||||||
|
prepared, err := preparer.PrepareValidation(context.Background(), domain.OutputContract{
|
||||||
|
ValidationMode: domain.ValidationJSONSchema,
|
||||||
|
SchemaPath: "schema.json",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("prepare validation: %v", err)
|
||||||
|
}
|
||||||
|
assertSchemaDocument(t, prepared.SchemaDocument(), rootSchema)
|
||||||
|
|
||||||
|
fsys["schema.json"] = &fstest.MapFile{Data: []byte(`{
|
||||||
|
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||||
|
"type": "string"
|
||||||
|
}`)}
|
||||||
|
fsys["value.json"] = &fstest.MapFile{Data: []byte(`{
|
||||||
|
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||||
|
"type": "string"
|
||||||
|
}`)}
|
||||||
|
|
||||||
|
valid, err := prepared.Validate(context.Background(), &domain.Artifact{Body: []byte(`{"value":3}`)})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("validate prepared artifact: %v", err)
|
||||||
|
}
|
||||||
|
if valid.Status != domain.ValidationPassed || !valid.IsValid {
|
||||||
|
t.Fatalf("expected passed/valid, got status=%q valid=%v errors=%v", valid.Status, valid.IsValid, valid.Errors)
|
||||||
|
}
|
||||||
|
|
||||||
|
invalid, err := prepared.Validate(context.Background(), &domain.Artifact{Body: []byte(`{"value":"changed"}`)})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("validate prepared artifact: %v", err)
|
||||||
|
}
|
||||||
|
if invalid.Status != domain.ValidationFailed || invalid.IsValid {
|
||||||
|
t.Fatalf("expected failed/invalid, got status=%q valid=%v", invalid.Status, invalid.IsValid)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestFSValidatorJSONSchemaRegistrationError(t *testing.T) {
|
func TestFSValidatorJSONSchemaRegistrationError(t *testing.T) {
|
||||||
v := NewFSValidator(fstest.MapFS{
|
v := NewFSValidator(fstest.MapFS{
|
||||||
"schemas/%zz.json": &fstest.MapFile{Data: []byte(`{"type":"object"}`)},
|
"schemas/%zz.json": &fstest.MapFile{Data: []byte(`{"type":"object"}`)},
|
||||||
@@ -548,3 +669,15 @@ func TestJSONSchemaDialectIsDraft2020(t *testing.T) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func assertSchemaDocument(t *testing.T, got any, expectedJSON []byte) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
var expected any
|
||||||
|
if err := json.Unmarshal(expectedJSON, &expected); err != nil {
|
||||||
|
t.Fatalf("decode expected schema document: %v", err)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(got, expected) {
|
||||||
|
t.Fatalf("schema document mismatch:\n got: %#v\nwant: %#v", got, expected)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ package validate
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -10,6 +11,20 @@ type Validator interface {
|
|||||||
Validate(ctx context.Context, artifact *domain.Artifact, contract domain.OutputContract) (domain.ValidationResult, error)
|
Validate(ctx context.Context, artifact *domain.Artifact, contract domain.OutputContract) (domain.ValidationResult, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// PreparedValidation validates artifacts against one frozen output contract.
|
||||||
|
type PreparedValidation interface {
|
||||||
|
Validate(ctx context.Context, artifact *domain.Artifact) (domain.ValidationResult, error)
|
||||||
|
// SchemaDocument returns the root JSON Schema document used for provider
|
||||||
|
// structured output, or nil for non-schema modes. Returned internal
|
||||||
|
// immutable state must not be mutated.
|
||||||
|
SchemaDocument() any
|
||||||
|
}
|
||||||
|
|
||||||
|
// ValidationPreparer freezes validation resources for one output contract.
|
||||||
|
type ValidationPreparer interface {
|
||||||
|
PrepareValidation(ctx context.Context, contract domain.OutputContract) (PreparedValidation, error)
|
||||||
|
}
|
||||||
|
|
||||||
// SchemaDocumentLoader loads JSON schema documents using validator path semantics.
|
// SchemaDocumentLoader loads JSON schema documents using validator path semantics.
|
||||||
type SchemaDocumentLoader interface {
|
type SchemaDocumentLoader interface {
|
||||||
LoadSchemaDocument(ctx context.Context, schemaPath string) (any, error)
|
LoadSchemaDocument(ctx context.Context, schemaPath string) (any, error)
|
||||||
|
|||||||
55
prepared_execution.go
Normal file
55
prepared_execution.go
Normal file
@@ -0,0 +1,55 @@
|
|||||||
|
package promptkit
|
||||||
|
|
||||||
|
import "gitea.maximumdirect.net/eric/promptkit/internal/usecase"
|
||||||
|
|
||||||
|
const preparedExecutionString = "promptkit.PreparedExecution{opaque}"
|
||||||
|
|
||||||
|
// PreparedExecution is an opaque, in-process handle for one completely
|
||||||
|
// prepared execution. A handle is bound to the [Engine] that created it and
|
||||||
|
// permits one [Engine.RunPrepared] invocation.
|
||||||
|
//
|
||||||
|
// PreparedExecution contains no supported serializable state and cannot be
|
||||||
|
// used as a restartable job. Copying the value preserves the same shared
|
||||||
|
// lifecycle; it does not create another execution attempt.
|
||||||
|
type PreparedExecution struct {
|
||||||
|
internal *usecase.PreparedExecution
|
||||||
|
}
|
||||||
|
|
||||||
|
// Details returns a fresh caller-owned, credential-redacted copy of the
|
||||||
|
// prepared request details. Mutating the result cannot affect execution or a
|
||||||
|
// later Details call. Details remains available after execution or discard.
|
||||||
|
//
|
||||||
|
// A nil receiver or zero-value PreparedExecution returns a zero [PreparedRun].
|
||||||
|
func (p *PreparedExecution) Details() PreparedRun {
|
||||||
|
if p == nil || p.internal == nil {
|
||||||
|
return PreparedRun{}
|
||||||
|
}
|
||||||
|
details := fromDomainPreparedRun(p.internal.Details())
|
||||||
|
if details == nil {
|
||||||
|
return PreparedRun{}
|
||||||
|
}
|
||||||
|
return *details
|
||||||
|
}
|
||||||
|
|
||||||
|
// Discard invalidates an unclaimed handle and drops Promptkit's references to
|
||||||
|
// its execution-only state. Discard is nil-safe and idempotent. It does not
|
||||||
|
// cancel an execution that has already claimed the handle; use the
|
||||||
|
// [Engine.RunPrepared] context for cancellation.
|
||||||
|
func (p *PreparedExecution) Discard() {
|
||||||
|
if p == nil || p.internal == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
p.internal.Discard()
|
||||||
|
}
|
||||||
|
|
||||||
|
// String returns a constant representation that exposes no retained request,
|
||||||
|
// rendered content, or credential data.
|
||||||
|
func (p *PreparedExecution) String() string {
|
||||||
|
return preparedExecutionString
|
||||||
|
}
|
||||||
|
|
||||||
|
// GoString returns a constant Go-syntax representation that exposes no
|
||||||
|
// retained request, rendered content, or credential data.
|
||||||
|
func (p *PreparedExecution) GoString() string {
|
||||||
|
return preparedExecutionString
|
||||||
|
}
|
||||||
773
prepared_execution_contract_test.go
Normal file
773
prepared_execution_contract_test.go
Normal file
@@ -0,0 +1,773 @@
|
|||||||
|
package promptkit_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
"testing/fstest"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestPreparedExecutionFreezesSourcesAndReturnsIndependentDetails(t *testing.T) {
|
||||||
|
promptSource := preparedPromptSource("original")
|
||||||
|
profileSource := preparedProfileSource("original-model")
|
||||||
|
schemaSource := preparedSchemaSource()
|
||||||
|
reader := &mutablePreparedArtifactReader{
|
||||||
|
body: "original artifact",
|
||||||
|
hash: "original-input-hash",
|
||||||
|
}
|
||||||
|
client := &preparedRecordingClient{
|
||||||
|
response: &promptkit.GenerateResponse{Content: `{"value":3}`},
|
||||||
|
}
|
||||||
|
engine, err := promptkit.NewEngine(
|
||||||
|
promptkit.Config{},
|
||||||
|
promptkit.WithPromptFS(promptSource, "."),
|
||||||
|
promptkit.WithProfileFS(profileSource, "."),
|
||||||
|
promptkit.WithSchemaFS(schemaSource, "."),
|
||||||
|
promptkit.WithArtifactReader(reader),
|
||||||
|
promptkit.WithLLMClient(client),
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("construct engine: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
temperature := 0.25
|
||||||
|
extraParams := map[string]any{
|
||||||
|
"nested": map[string]any{"source": "original"},
|
||||||
|
}
|
||||||
|
request := promptkit.RunRequest{
|
||||||
|
PromptID: "prepared",
|
||||||
|
Inputs: map[string]promptkit.ArtifactRef{
|
||||||
|
"input": promptkit.Inline("original request input"),
|
||||||
|
},
|
||||||
|
Vars: map[string]string{"label": "original variable"},
|
||||||
|
Execution: &promptkit.ExecutionTargetOverride{
|
||||||
|
Temperature: &temperature,
|
||||||
|
ExtraParams: extraParams,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
preparationContext, cancelPreparation := context.WithCancel(context.Background())
|
||||||
|
prepared, err := engine.PrepareExecution(preparationContext, request)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("prepare execution: %v", err)
|
||||||
|
}
|
||||||
|
cancelPreparation()
|
||||||
|
|
||||||
|
request.PromptID = "changed"
|
||||||
|
request.Inputs["input"] = promptkit.Inline("changed request input")
|
||||||
|
request.Vars["label"] = "changed variable"
|
||||||
|
temperature = 1.5
|
||||||
|
extraParams["nested"].(map[string]any)["source"] = "changed"
|
||||||
|
promptSource["prompt.yaml"] = &fstest.MapFile{Data: []byte(`id: changed`)}
|
||||||
|
profileSource["profile.yaml"] = &fstest.MapFile{Data: []byte(`id: changed`)}
|
||||||
|
schemaSource["schema.json"] = &fstest.MapFile{Data: []byte(`{
|
||||||
|
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||||
|
"title": "changed root",
|
||||||
|
"type": "string"
|
||||||
|
}`)}
|
||||||
|
schemaSource["value.json"] = &fstest.MapFile{Data: []byte(`{
|
||||||
|
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||||
|
"type": "string"
|
||||||
|
}`)}
|
||||||
|
reader.set("changed artifact", "changed-input-hash")
|
||||||
|
|
||||||
|
first := prepared.Details()
|
||||||
|
first.Messages[0].Content = "changed details"
|
||||||
|
first.InputHashes["input"] = "changed-details-hash"
|
||||||
|
first.EffectiveModelParams.ExtraParams["nested"].(map[string]any)["source"] = "changed details"
|
||||||
|
first.StructuredOutput.JSONSchema.Schema.(map[string]any)["title"] = "changed details"
|
||||||
|
|
||||||
|
second := prepared.Details()
|
||||||
|
if second.Messages[0].Content != "Input=original artifact Label=original variable" {
|
||||||
|
t.Fatalf("details message changed: %q", second.Messages[0].Content)
|
||||||
|
}
|
||||||
|
if second.InputHashes["input"] != "original-input-hash" {
|
||||||
|
t.Fatalf("details input hash changed: %q", second.InputHashes["input"])
|
||||||
|
}
|
||||||
|
if second.EffectiveModelParams.Model != "original-model" ||
|
||||||
|
second.EffectiveModelParams.Temperature != 0.25 ||
|
||||||
|
second.EffectiveModelParams.ExtraParams["nested"].(map[string]any)["source"] != "original" {
|
||||||
|
t.Fatalf("details target changed: %+v", second.EffectiveModelParams)
|
||||||
|
}
|
||||||
|
schema := second.StructuredOutput.JSONSchema.Schema.(map[string]any)
|
||||||
|
if schema["title"] != "original root" {
|
||||||
|
t.Fatalf("details schema changed: %#v", schema)
|
||||||
|
}
|
||||||
|
|
||||||
|
result, err := engine.RunPrepared(context.Background(), prepared)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("run prepared after preparation-context cancellation: %v", err)
|
||||||
|
}
|
||||||
|
if result.Validation.Status != promptkit.ValidationPassed || !result.Validation.IsValid {
|
||||||
|
t.Fatalf("frozen schema did not validate original output: %+v", result.Validation)
|
||||||
|
}
|
||||||
|
if reader.callCount() != 1 {
|
||||||
|
t.Fatalf("execution reopened artifact source: calls=%d", reader.callCount())
|
||||||
|
}
|
||||||
|
|
||||||
|
requests := client.snapshot()
|
||||||
|
if len(requests) != 1 {
|
||||||
|
t.Fatalf("generation calls=%d, want 1", len(requests))
|
||||||
|
}
|
||||||
|
generated := requests[0]
|
||||||
|
if generated.Prompt.Messages[0].Content != second.Messages[0].Content ||
|
||||||
|
generated.Target.Model != second.EffectiveModelParams.Model ||
|
||||||
|
!reflect.DeepEqual(generated.Target.ExtraParams, second.EffectiveModelParams.ExtraParams) ||
|
||||||
|
!reflect.DeepEqual(generated.StructuredOutput, second.StructuredOutput) {
|
||||||
|
t.Fatalf("generation did not use frozen details:\nrequest=%+v\ndetails=%+v", generated, second)
|
||||||
|
}
|
||||||
|
if result.PromptID != second.PromptID ||
|
||||||
|
result.PromptVersion != second.PromptVersion ||
|
||||||
|
result.PromptHash != second.PromptHash ||
|
||||||
|
result.SessionID != second.SessionID ||
|
||||||
|
result.RenderedPromptHash != second.RenderedPromptHash ||
|
||||||
|
result.SelectedProfileID != second.SelectedProfileID ||
|
||||||
|
result.SelectedBackendID != second.SelectedBackendID ||
|
||||||
|
!reflect.DeepEqual(result.EffectiveModelParams, second.EffectiveModelParams) ||
|
||||||
|
!reflect.DeepEqual(result.InputHashes, second.InputHashes) {
|
||||||
|
t.Fatalf("result provenance does not match details:\nresult=%+v\ndetails=%+v", result, second)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPreparedExecutionLifecycleAndEngineBinding(t *testing.T) {
|
||||||
|
ownerClient := &preparedRecordingClient{
|
||||||
|
response: &promptkit.GenerateResponse{Content: "ok"},
|
||||||
|
}
|
||||||
|
owner := newPreparedContractEngine(t, ownerClient, "owner content")
|
||||||
|
foreign := newPreparedContractEngine(t, &preparedRecordingClient{
|
||||||
|
response: &promptkit.GenerateResponse{Content: "unexpected"},
|
||||||
|
}, "foreign content")
|
||||||
|
|
||||||
|
prepared, err := owner.PrepareExecution(context.Background(), promptkit.RunRequest{PromptID: "prepared"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("prepare execution: %v", err)
|
||||||
|
}
|
||||||
|
copied := *prepared
|
||||||
|
|
||||||
|
var nilEngine *promptkit.Engine
|
||||||
|
if result, err := nilEngine.RunPrepared(context.Background(), prepared); result != nil ||
|
||||||
|
!errors.Is(err, promptkit.ErrInvalidConfig) {
|
||||||
|
t.Fatalf("nil engine result=(%+v, %v), want ErrInvalidConfig", result, err)
|
||||||
|
}
|
||||||
|
if result, err := foreign.RunPrepared(context.Background(), prepared); result != nil ||
|
||||||
|
!errors.Is(err, promptkit.ErrInvalidRequest) {
|
||||||
|
t.Fatalf("foreign engine result=(%+v, %v), want ErrInvalidRequest", result, err)
|
||||||
|
}
|
||||||
|
if result, err := owner.RunPrepared(context.Background(), nil); result != nil ||
|
||||||
|
!errors.Is(err, promptkit.ErrInvalidRequest) {
|
||||||
|
t.Fatalf("nil handle result=(%+v, %v), want ErrInvalidRequest", result, err)
|
||||||
|
}
|
||||||
|
if result, err := owner.RunPrepared(context.Background(), &promptkit.PreparedExecution{}); result != nil ||
|
||||||
|
!errors.Is(err, promptkit.ErrInvalidRequest) {
|
||||||
|
t.Fatalf("zero handle result=(%+v, %v), want ErrInvalidRequest", result, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
result, err := owner.RunPrepared(context.Background(), &copied)
|
||||||
|
if err != nil || result == nil {
|
||||||
|
t.Fatalf("owner run prepared=(%+v, %v), want success", result, err)
|
||||||
|
}
|
||||||
|
for name, handle := range map[string]*promptkit.PreparedExecution{
|
||||||
|
"original": prepared,
|
||||||
|
"copy": &copied,
|
||||||
|
} {
|
||||||
|
if result, err := owner.RunPrepared(context.Background(), handle); result != nil ||
|
||||||
|
!errors.Is(err, promptkit.ErrInvalidRequest) {
|
||||||
|
t.Fatalf("%s reused handle result=(%+v, %v), want ErrInvalidRequest", name, result, err)
|
||||||
|
}
|
||||||
|
if handle.Details().PromptID != "prepared" {
|
||||||
|
t.Fatalf("%s details unavailable after execution", name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(ownerClient.snapshot()) != 1 {
|
||||||
|
t.Fatalf("owner generation calls=%d, want 1", len(ownerClient.snapshot()))
|
||||||
|
}
|
||||||
|
|
||||||
|
collaboratorFailure := errors.New("prepared collaborator failure")
|
||||||
|
failingClient := &preparedRecordingClient{err: collaboratorFailure}
|
||||||
|
failingEngine := newPreparedContractEngine(t, failingClient, "failure content")
|
||||||
|
failing, err := failingEngine.PrepareExecution(context.Background(), promptkit.RunRequest{PromptID: "prepared"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("prepare failing execution: %v", err)
|
||||||
|
}
|
||||||
|
if result, err := failingEngine.RunPrepared(context.Background(), failing); result != nil ||
|
||||||
|
!errors.Is(err, promptkit.ErrLLMGenerate) ||
|
||||||
|
!errors.Is(err, collaboratorFailure) {
|
||||||
|
t.Fatalf("generation failure result=(%+v, %v), want public and collaborator identities", result, err)
|
||||||
|
}
|
||||||
|
if result, err := failingEngine.RunPrepared(context.Background(), failing); result != nil ||
|
||||||
|
!errors.Is(err, promptkit.ErrInvalidRequest) {
|
||||||
|
t.Fatalf("failed execution was reusable: result=(%+v, %v)", result, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
cancellationRelease := make(chan struct{})
|
||||||
|
cancellationStarted := make(chan struct{}, 1)
|
||||||
|
cancelingEngine := newPreparedContractEngine(t, &preparedRecordingClient{
|
||||||
|
response: &promptkit.GenerateResponse{Content: "unexpected"},
|
||||||
|
started: cancellationStarted,
|
||||||
|
release: cancellationRelease,
|
||||||
|
}, "cancellation content")
|
||||||
|
canceling, err := cancelingEngine.PrepareExecution(
|
||||||
|
context.Background(),
|
||||||
|
promptkit.RunRequest{PromptID: "prepared"},
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("prepare canceled execution: %v", err)
|
||||||
|
}
|
||||||
|
executionContext, cancelExecution := context.WithCancel(context.Background())
|
||||||
|
type canceledOutcome struct {
|
||||||
|
result *promptkit.RunResult
|
||||||
|
err error
|
||||||
|
}
|
||||||
|
canceledResult := make(chan canceledOutcome, 1)
|
||||||
|
go func() {
|
||||||
|
result, runErr := cancelingEngine.RunPrepared(executionContext, canceling)
|
||||||
|
canceledResult <- canceledOutcome{result: result, err: runErr}
|
||||||
|
}()
|
||||||
|
select {
|
||||||
|
case <-cancellationStarted:
|
||||||
|
case <-time.After(5 * time.Second):
|
||||||
|
t.Fatal("timed out waiting for cancelable generation")
|
||||||
|
}
|
||||||
|
cancelExecution()
|
||||||
|
select {
|
||||||
|
case outcome := <-canceledResult:
|
||||||
|
if outcome.result != nil ||
|
||||||
|
!errors.Is(outcome.err, promptkit.ErrLLMGenerate) ||
|
||||||
|
!errors.Is(outcome.err, context.Canceled) {
|
||||||
|
t.Fatalf(
|
||||||
|
"canceled execution=(%+v, %v), want generation and context identities",
|
||||||
|
outcome.result,
|
||||||
|
outcome.err,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
case <-time.After(5 * time.Second):
|
||||||
|
t.Fatal("timed out waiting for canceled execution")
|
||||||
|
}
|
||||||
|
if result, err := cancelingEngine.RunPrepared(context.Background(), canceling); result != nil ||
|
||||||
|
!errors.Is(err, promptkit.ErrInvalidRequest) {
|
||||||
|
t.Fatalf("canceled execution was reusable: result=(%+v, %v)", result, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPreparedExecutionConcurrentClaimAllowsOneGeneration(t *testing.T) {
|
||||||
|
release := make(chan struct{})
|
||||||
|
client := &preparedRecordingClient{
|
||||||
|
response: &promptkit.GenerateResponse{Content: "ok"},
|
||||||
|
started: make(chan struct{}, 1),
|
||||||
|
release: release,
|
||||||
|
}
|
||||||
|
engine := newPreparedContractEngine(t, client, "concurrent content")
|
||||||
|
prepared, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{PromptID: "prepared"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("prepare execution: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
type outcome struct {
|
||||||
|
result *promptkit.RunResult
|
||||||
|
err error
|
||||||
|
}
|
||||||
|
outcomes := make(chan outcome, 2)
|
||||||
|
for i := 0; i < 2; i++ {
|
||||||
|
go func() {
|
||||||
|
result, runErr := engine.RunPrepared(context.Background(), prepared)
|
||||||
|
outcomes <- outcome{result: result, err: runErr}
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
|
||||||
|
select {
|
||||||
|
case <-client.started:
|
||||||
|
case <-time.After(5 * time.Second):
|
||||||
|
t.Fatal("timed out waiting for generation")
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case loser := <-outcomes:
|
||||||
|
if loser.result != nil || !errors.Is(loser.err, promptkit.ErrInvalidRequest) {
|
||||||
|
t.Fatalf("concurrent loser=(%+v, %v), want ErrInvalidRequest", loser.result, loser.err)
|
||||||
|
}
|
||||||
|
case <-time.After(5 * time.Second):
|
||||||
|
t.Fatal("timed out waiting for rejected concurrent claim")
|
||||||
|
}
|
||||||
|
|
||||||
|
close(release)
|
||||||
|
select {
|
||||||
|
case winner := <-outcomes:
|
||||||
|
if winner.err != nil || winner.result == nil {
|
||||||
|
t.Fatalf("concurrent winner=(%+v, %v), want success", winner.result, winner.err)
|
||||||
|
}
|
||||||
|
case <-time.After(5 * time.Second):
|
||||||
|
t.Fatal("timed out waiting for successful concurrent claim")
|
||||||
|
}
|
||||||
|
if len(client.snapshot()) != 1 {
|
||||||
|
t.Fatalf("generation calls=%d, want 1", len(client.snapshot()))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPreparedExecutionRunAndDiscardRaceHasOneWinner(t *testing.T) {
|
||||||
|
const attempts = 32
|
||||||
|
|
||||||
|
for i := 0; i < attempts; i++ {
|
||||||
|
client := &preparedRecordingClient{
|
||||||
|
response: &promptkit.GenerateResponse{Content: "ok"},
|
||||||
|
}
|
||||||
|
engine := newPreparedContractEngine(t, client, "race content")
|
||||||
|
prepared, err := engine.PrepareExecution(
|
||||||
|
context.Background(),
|
||||||
|
promptkit.RunRequest{PromptID: "prepared"},
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("attempt %d prepare execution: %v", i, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
start := make(chan struct{})
|
||||||
|
type outcome struct {
|
||||||
|
result *promptkit.RunResult
|
||||||
|
err error
|
||||||
|
}
|
||||||
|
runOutcome := make(chan outcome, 1)
|
||||||
|
discardDone := make(chan struct{})
|
||||||
|
|
||||||
|
go func() {
|
||||||
|
<-start
|
||||||
|
result, runErr := engine.RunPrepared(context.Background(), prepared)
|
||||||
|
runOutcome <- outcome{result: result, err: runErr}
|
||||||
|
}()
|
||||||
|
go func() {
|
||||||
|
<-start
|
||||||
|
prepared.Discard()
|
||||||
|
close(discardDone)
|
||||||
|
}()
|
||||||
|
|
||||||
|
close(start)
|
||||||
|
runResult := <-runOutcome
|
||||||
|
<-discardDone
|
||||||
|
calls := len(client.snapshot())
|
||||||
|
switch {
|
||||||
|
case runResult.err == nil:
|
||||||
|
if runResult.result == nil || calls != 1 {
|
||||||
|
t.Fatalf(
|
||||||
|
"attempt %d run won with outcome=(%+v, %v), generation calls=%d",
|
||||||
|
i,
|
||||||
|
runResult.result,
|
||||||
|
runResult.err,
|
||||||
|
calls,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
case errors.Is(runResult.err, promptkit.ErrInvalidRequest):
|
||||||
|
if runResult.result != nil || calls != 0 {
|
||||||
|
t.Fatalf(
|
||||||
|
"attempt %d discard won with outcome=(%+v, %v), generation calls=%d",
|
||||||
|
i,
|
||||||
|
runResult.result,
|
||||||
|
runResult.err,
|
||||||
|
calls,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
t.Fatalf("attempt %d unexpected run outcome=(%+v, %v)", i, runResult.result, runResult.err)
|
||||||
|
}
|
||||||
|
if prepared.Details().PromptID != "prepared" {
|
||||||
|
t.Fatalf("attempt %d details unavailable after race", i)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPreparedExecutionDiscardAndFormattingDoNotExposePrivateState(t *testing.T) {
|
||||||
|
const (
|
||||||
|
directCredential = "pk-test-direct-credential-41f7"
|
||||||
|
renderedContent = "rendered-content-sentinel-98d2"
|
||||||
|
)
|
||||||
|
client := &preparedRecordingClient{
|
||||||
|
response: &promptkit.GenerateResponse{Content: "generated output"},
|
||||||
|
}
|
||||||
|
engine, err := promptkit.NewEngine(
|
||||||
|
promptkit.Config{},
|
||||||
|
promptkit.WithPromptFS(contractPromptFS("prepared", "profile", renderedContent), "."),
|
||||||
|
promptkit.WithProfiles(promptkit.Profile{
|
||||||
|
ID: "profile",
|
||||||
|
Endpoint: "http://example.test/v1",
|
||||||
|
Model: "model",
|
||||||
|
APIKeyRequired: true,
|
||||||
|
}),
|
||||||
|
promptkit.WithLLMClient(client),
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("construct engine: %v", err)
|
||||||
|
}
|
||||||
|
prepared, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{
|
||||||
|
PromptID: "prepared",
|
||||||
|
APIKey: directCredential,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("prepare execution: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
formattedValues := []string{
|
||||||
|
fmt.Sprint(prepared),
|
||||||
|
fmt.Sprintf("%+v", prepared),
|
||||||
|
fmt.Sprintf("%#v", prepared),
|
||||||
|
}
|
||||||
|
for _, formatted := range formattedValues {
|
||||||
|
if formatted != "promptkit.PreparedExecution{opaque}" {
|
||||||
|
t.Fatalf("unexpected opaque formatting: %q", formatted)
|
||||||
|
}
|
||||||
|
assertPreparedPrivateValuesAbsent(t, formatted, directCredential, renderedContent)
|
||||||
|
}
|
||||||
|
payload, err := json.Marshal(prepared)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("marshal opaque handle: %v", err)
|
||||||
|
}
|
||||||
|
assertPreparedPrivateValuesAbsent(t, string(payload), directCredential, renderedContent)
|
||||||
|
|
||||||
|
detailsBefore := prepared.Details()
|
||||||
|
detailsJSON, err := json.Marshal(detailsBefore)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("marshal prepared details: %v", err)
|
||||||
|
}
|
||||||
|
assertPreparedPrivateValuesAbsent(t, string(detailsJSON), directCredential)
|
||||||
|
|
||||||
|
prepared.Discard()
|
||||||
|
prepared.Discard()
|
||||||
|
result, lifecycleErr := engine.RunPrepared(context.Background(), prepared)
|
||||||
|
if result != nil || !errors.Is(lifecycleErr, promptkit.ErrInvalidRequest) {
|
||||||
|
t.Fatalf("discarded execution result=(%+v, %v), want ErrInvalidRequest", result, lifecycleErr)
|
||||||
|
}
|
||||||
|
assertPreparedPrivateValuesAbsent(t, lifecycleErr.Error(), directCredential, renderedContent)
|
||||||
|
if !reflect.DeepEqual(prepared.Details(), detailsBefore) {
|
||||||
|
t.Fatal("details changed after discard")
|
||||||
|
}
|
||||||
|
|
||||||
|
executed, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{
|
||||||
|
PromptID: "prepared",
|
||||||
|
APIKey: directCredential,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("prepare execution for request inspection: %v", err)
|
||||||
|
}
|
||||||
|
executionResult, err := engine.RunPrepared(context.Background(), executed)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("run execution for request inspection: %v", err)
|
||||||
|
}
|
||||||
|
requests := client.snapshot()
|
||||||
|
if len(requests) != 1 || requests[0].APIKey != directCredential {
|
||||||
|
t.Fatalf("direct credential did not reach only the client credential field: %#v", requests)
|
||||||
|
}
|
||||||
|
requestJSON, err := json.Marshal(requests[0])
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("marshal captured generate request: %v", err)
|
||||||
|
}
|
||||||
|
for _, value := range []string{
|
||||||
|
fmt.Sprint(requests[0]),
|
||||||
|
fmt.Sprintf("%+v", requests[0]),
|
||||||
|
fmt.Sprintf("%#v", requests[0]),
|
||||||
|
string(requestJSON),
|
||||||
|
fmt.Sprint(executionResult),
|
||||||
|
} {
|
||||||
|
assertPreparedPrivateValuesAbsent(t, value, directCredential)
|
||||||
|
}
|
||||||
|
resultJSON, err := json.Marshal(executionResult)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("marshal execution result: %v", err)
|
||||||
|
}
|
||||||
|
assertPreparedPrivateValuesAbsent(t, string(resultJSON), directCredential)
|
||||||
|
|
||||||
|
var nilHandle *promptkit.PreparedExecution
|
||||||
|
nilHandle.Discard()
|
||||||
|
if !reflect.DeepEqual(nilHandle.Details(), promptkit.PreparedRun{}) {
|
||||||
|
t.Fatalf("nil handle details=%+v, want zero value", nilHandle.Details())
|
||||||
|
}
|
||||||
|
zeroHandle := &promptkit.PreparedExecution{}
|
||||||
|
zeroHandle.Discard()
|
||||||
|
if !reflect.DeepEqual(zeroHandle.Details(), promptkit.PreparedRun{}) {
|
||||||
|
t.Fatalf("zero handle details=%+v, want zero value", zeroHandle.Details())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPreparedExecutionCredentialCapacityAndTimingBoundaries(t *testing.T) {
|
||||||
|
t.Run("credential is rechecked before generation", func(t *testing.T) {
|
||||||
|
const (
|
||||||
|
environmentName = "PROMPTKIT_PREPARED_CONTRACT_KEY"
|
||||||
|
environmentKey = "environment-credential-sentinel"
|
||||||
|
)
|
||||||
|
t.Setenv(environmentName, environmentKey)
|
||||||
|
client := &preparedRecordingClient{
|
||||||
|
response: &promptkit.GenerateResponse{Content: "unexpected"},
|
||||||
|
}
|
||||||
|
engine, err := promptkit.NewEngine(
|
||||||
|
promptkit.Config{},
|
||||||
|
promptkit.WithPromptFS(contractPromptFS("prepared", "profile", "content"), "."),
|
||||||
|
promptkit.WithProfileFS(preparedCredentialProfileSource(environmentName), "."),
|
||||||
|
promptkit.WithLLMClient(client),
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("construct credential engine: %v", err)
|
||||||
|
}
|
||||||
|
prepared, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{PromptID: "prepared"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("prepare credential execution: %v", err)
|
||||||
|
}
|
||||||
|
if err := os.Unsetenv(environmentName); err != nil {
|
||||||
|
t.Fatalf("unset credential environment: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
result, err := engine.RunPrepared(context.Background(), prepared)
|
||||||
|
if result != nil ||
|
||||||
|
!errors.Is(err, promptkit.ErrInvalidRequest) ||
|
||||||
|
!errors.Is(err, promptkit.ErrAPIKeyEnvMissing) {
|
||||||
|
t.Fatalf("credential execution=(%+v, %v), want credential identities", result, err)
|
||||||
|
}
|
||||||
|
if len(client.snapshot()) != 0 {
|
||||||
|
t.Fatalf("credential failure reached generation: %d calls", len(client.snapshot()))
|
||||||
|
}
|
||||||
|
assertPreparedPrivateValuesAbsent(t, err.Error(), environmentKey)
|
||||||
|
if result, err := engine.RunPrepared(context.Background(), prepared); result != nil ||
|
||||||
|
!errors.Is(err, promptkit.ErrInvalidRequest) {
|
||||||
|
t.Fatalf("credential failure did not consume handle: result=(%+v, %v)", result, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("preparation does not admit and execution timing starts after retention", func(t *testing.T) {
|
||||||
|
release := make(chan struct{})
|
||||||
|
client := newCapacityGateClient(release, 4)
|
||||||
|
engine := newBackendCapacityEngine(t, client, 1, capacityInt(0), nil)
|
||||||
|
|
||||||
|
activeRun := make(chan capacityRunResult, 1)
|
||||||
|
go runCapacityRequest(
|
||||||
|
engine,
|
||||||
|
context.Background(),
|
||||||
|
promptkit.RunRequest{PromptID: "prompt"},
|
||||||
|
activeRun,
|
||||||
|
)
|
||||||
|
awaitCapacityRequest(t, client.started)
|
||||||
|
|
||||||
|
prepared, err := engine.PrepareExecution(
|
||||||
|
context.Background(),
|
||||||
|
promptkit.RunRequest{PromptID: "prompt"},
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("prepare while capacity is full: %v", err)
|
||||||
|
}
|
||||||
|
if _, _, calls := client.snapshot(); calls != 1 {
|
||||||
|
t.Fatalf("preparation invoked generation: calls=%d", calls)
|
||||||
|
}
|
||||||
|
if result, err := engine.RunPrepared(context.Background(), prepared); result != nil ||
|
||||||
|
!errors.Is(err, promptkit.ErrCapacityExceeded) {
|
||||||
|
t.Fatalf("capacity execution=(%+v, %v), want ErrCapacityExceeded", result, err)
|
||||||
|
} else {
|
||||||
|
var capacityErr *promptkit.CapacityError
|
||||||
|
if !errors.As(err, &capacityErr) || capacityErr == nil || capacityErr.BackendID != "limited" {
|
||||||
|
t.Fatalf("capacity execution=%v, want limited CapacityError", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if result, err := engine.RunPrepared(context.Background(), prepared); result != nil ||
|
||||||
|
!errors.Is(err, promptkit.ErrInvalidRequest) {
|
||||||
|
t.Fatalf("capacity rejection did not consume handle: result=(%+v, %v)", result, err)
|
||||||
|
}
|
||||||
|
if prepared.Details().PromptID != "prompt" {
|
||||||
|
t.Fatal("details unavailable after capacity rejection")
|
||||||
|
}
|
||||||
|
|
||||||
|
close(release)
|
||||||
|
activeOutcome := awaitCapacityRun(t, activeRun)
|
||||||
|
if activeOutcome.err != nil || activeOutcome.result == nil {
|
||||||
|
t.Fatalf("active run outcome=(%+v, %v), want success", activeOutcome.result, activeOutcome.err)
|
||||||
|
}
|
||||||
|
|
||||||
|
timed, err := engine.PrepareExecution(
|
||||||
|
context.Background(),
|
||||||
|
promptkit.RunRequest{PromptID: "prompt"},
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("prepare timed execution: %v", err)
|
||||||
|
}
|
||||||
|
details := timed.Details()
|
||||||
|
time.Sleep(25 * time.Millisecond)
|
||||||
|
executionFloor := time.Now().UTC()
|
||||||
|
result, err := engine.RunPrepared(context.Background(), timed)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("run timed execution: %v", err)
|
||||||
|
}
|
||||||
|
if result.StartTime.Before(executionFloor) ||
|
||||||
|
!result.StartTime.After(details.EndTime) ||
|
||||||
|
result.EndTime.Before(result.StartTime) ||
|
||||||
|
result.Duration != result.EndTime.Sub(result.StartTime) {
|
||||||
|
t.Fatalf(
|
||||||
|
"execution timing includes preparation or retention: details_end=%s floor=%s result=%+v",
|
||||||
|
details.EndTime,
|
||||||
|
executionFloor,
|
||||||
|
result,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
if _, _, calls := client.snapshot(); calls != 2 {
|
||||||
|
t.Fatalf("generation calls=%d, want active and timed executions only", calls)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
type mutablePreparedArtifactReader struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
body string
|
||||||
|
hash string
|
||||||
|
calls int
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *mutablePreparedArtifactReader) Read(
|
||||||
|
_ context.Context,
|
||||||
|
_ promptkit.ArtifactRef,
|
||||||
|
) (*promptkit.Artifact, error) {
|
||||||
|
r.mu.Lock()
|
||||||
|
defer r.mu.Unlock()
|
||||||
|
r.calls++
|
||||||
|
return &promptkit.Artifact{
|
||||||
|
Body: []byte(r.body),
|
||||||
|
Hash: r.hash,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *mutablePreparedArtifactReader) set(body, hash string) {
|
||||||
|
r.mu.Lock()
|
||||||
|
defer r.mu.Unlock()
|
||||||
|
r.body = body
|
||||||
|
r.hash = hash
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *mutablePreparedArtifactReader) callCount() int {
|
||||||
|
r.mu.Lock()
|
||||||
|
defer r.mu.Unlock()
|
||||||
|
return r.calls
|
||||||
|
}
|
||||||
|
|
||||||
|
type preparedRecordingClient struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
response *promptkit.GenerateResponse
|
||||||
|
err error
|
||||||
|
requests []promptkit.GenerateRequest
|
||||||
|
started chan struct{}
|
||||||
|
release <-chan struct{}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *preparedRecordingClient) Generate(
|
||||||
|
ctx context.Context,
|
||||||
|
request promptkit.GenerateRequest,
|
||||||
|
) (*promptkit.GenerateResponse, error) {
|
||||||
|
c.mu.Lock()
|
||||||
|
c.requests = append(c.requests, request)
|
||||||
|
c.mu.Unlock()
|
||||||
|
|
||||||
|
if c.started != nil {
|
||||||
|
c.started <- struct{}{}
|
||||||
|
}
|
||||||
|
if c.release != nil {
|
||||||
|
select {
|
||||||
|
case <-c.release:
|
||||||
|
case <-ctx.Done():
|
||||||
|
return nil, ctx.Err()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if c.err != nil {
|
||||||
|
return nil, c.err
|
||||||
|
}
|
||||||
|
return c.response, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *preparedRecordingClient) snapshot() []promptkit.GenerateRequest {
|
||||||
|
c.mu.Lock()
|
||||||
|
defer c.mu.Unlock()
|
||||||
|
return append([]promptkit.GenerateRequest(nil), c.requests...)
|
||||||
|
}
|
||||||
|
|
||||||
|
func newPreparedContractEngine(
|
||||||
|
t *testing.T,
|
||||||
|
client promptkit.LLMClient,
|
||||||
|
message string,
|
||||||
|
) *promptkit.Engine {
|
||||||
|
t.Helper()
|
||||||
|
engine, err := promptkit.NewEngine(
|
||||||
|
promptkit.Config{},
|
||||||
|
promptkit.WithPromptFS(contractPromptFS("prepared", "profile", message), "."),
|
||||||
|
promptkit.WithProfiles(promptkit.Profile{
|
||||||
|
ID: "profile",
|
||||||
|
Endpoint: "http://example.test/v1",
|
||||||
|
Model: "model",
|
||||||
|
}),
|
||||||
|
promptkit.WithLLMClient(client),
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("construct prepared execution engine: %v", err)
|
||||||
|
}
|
||||||
|
return engine
|
||||||
|
}
|
||||||
|
|
||||||
|
func preparedPromptSource(label string) fstest.MapFS {
|
||||||
|
return fstest.MapFS{
|
||||||
|
"prompt.yaml": &fstest.MapFile{Data: []byte(`id: prepared
|
||||||
|
version: "1"
|
||||||
|
default_profile: profile
|
||||||
|
inputs:
|
||||||
|
- name: input
|
||||||
|
required: true
|
||||||
|
messages:
|
||||||
|
- role: user
|
||||||
|
content: 'Input={{input "input"}} Label={{.label}}'
|
||||||
|
description: ` + label + `
|
||||||
|
output:
|
||||||
|
format: json
|
||||||
|
validation_mode: json_schema
|
||||||
|
schema_path: schema.json
|
||||||
|
`)},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func preparedProfileSource(model string) fstest.MapFS {
|
||||||
|
return fstest.MapFS{
|
||||||
|
"profile.yaml": &fstest.MapFile{Data: []byte(`id: profile
|
||||||
|
endpoint: http://example.test/v1
|
||||||
|
model: ` + model + `
|
||||||
|
`)},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func preparedCredentialProfileSource(environmentName string) fstest.MapFS {
|
||||||
|
return fstest.MapFS{
|
||||||
|
"profile.yaml": &fstest.MapFile{Data: []byte(`id: profile
|
||||||
|
endpoint: http://example.test/v1
|
||||||
|
model: model
|
||||||
|
api_key_env: ` + environmentName + `
|
||||||
|
`)},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func preparedSchemaSource() fstest.MapFS {
|
||||||
|
return fstest.MapFS{
|
||||||
|
"schema.json": &fstest.MapFile{Data: []byte(`{
|
||||||
|
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||||
|
"title": "original root",
|
||||||
|
"type": "object",
|
||||||
|
"required": ["value"],
|
||||||
|
"properties": {
|
||||||
|
"value": {"$ref": "value.json"}
|
||||||
|
}
|
||||||
|
}`)},
|
||||||
|
"value.json": &fstest.MapFile{Data: []byte(`{
|
||||||
|
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||||
|
"type": "integer",
|
||||||
|
"minimum": 2
|
||||||
|
}`)},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertPreparedPrivateValuesAbsent(t *testing.T, value string, privateValues ...string) {
|
||||||
|
t.Helper()
|
||||||
|
for _, privateValue := range privateValues {
|
||||||
|
if strings.Contains(value, privateValue) {
|
||||||
|
t.Fatalf("value exposed private data %q: %s", privateValue, value)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -5,6 +5,8 @@ import (
|
|||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"io/fs"
|
||||||
|
"reflect"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
@@ -15,6 +17,380 @@ import (
|
|||||||
"gitea.maximumdirect.net/eric/promptkit"
|
"gitea.maximumdirect.net/eric/promptkit"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
type inspectionCountingFS struct {
|
||||||
|
opens atomic.Int64
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *inspectionCountingFS) Open(string) (fs.File, error) {
|
||||||
|
f.opens.Add(1)
|
||||||
|
return nil, fs.ErrNotExist
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestInspectProfileResolvesCredentialStatesWithoutPromptOrGeneration(t *testing.T) {
|
||||||
|
const environmentName = "PROMPTKIT_INSPECTION_ABSENT_KEY"
|
||||||
|
t.Setenv(environmentName, "")
|
||||||
|
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "unexpected"}}
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{},
|
||||||
|
promptkit.WithPromptFS(fstest.MapFS{}, "."),
|
||||||
|
promptkit.WithProfileFS(fstest.MapFS{
|
||||||
|
"environment.yaml": &fstest.MapFile{Data: []byte(`id: environment
|
||||||
|
endpoint: http://environment.example/v1
|
||||||
|
model: environment-model
|
||||||
|
api_key_env: PROMPTKIT_INSPECTION_ABSENT_KEY
|
||||||
|
`)},
|
||||||
|
}, "."),
|
||||||
|
promptkit.WithProfiles(
|
||||||
|
promptkit.Profile{ID: "direct", Endpoint: "http://direct.example/v1", Model: "direct-model", APIKeyRequired: true},
|
||||||
|
promptkit.Profile{ID: "none", Endpoint: "http://none.example/v1", Model: "none-model"},
|
||||||
|
),
|
||||||
|
promptkit.WithLLMClient(client),
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("construct inspection engine: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range []struct {
|
||||||
|
profileID string
|
||||||
|
wantEnv string
|
||||||
|
wantDirectKey bool
|
||||||
|
wantEndpoint string
|
||||||
|
}{
|
||||||
|
{profileID: " environment ", wantEnv: environmentName, wantEndpoint: "http://environment.example/v1"},
|
||||||
|
{profileID: "direct", wantDirectKey: true, wantEndpoint: "http://direct.example/v1"},
|
||||||
|
{profileID: "none", wantEndpoint: "http://none.example/v1"},
|
||||||
|
} {
|
||||||
|
t.Run(tc.profileID, func(t *testing.T) {
|
||||||
|
inspection, err := engine.InspectProfile(context.Background(), tc.profileID)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("inspect profile: %v", err)
|
||||||
|
}
|
||||||
|
if inspection.ProfileID != strings.TrimSpace(tc.profileID) ||
|
||||||
|
inspection.EffectiveModelParams.Endpoint != tc.wantEndpoint ||
|
||||||
|
inspection.EffectiveModelParams.BackendID != "" ||
|
||||||
|
inspection.EffectiveModelParams.APIKeyEnv != tc.wantEnv ||
|
||||||
|
inspection.APIKeyRequired != tc.wantDirectKey {
|
||||||
|
t.Fatalf("unexpected inspection: %#v", inspection)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
if len(client.requests) != 0 {
|
||||||
|
t.Fatalf("inspection invoked the model client %d times", len(client.requests))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestInspectProfilePreservesPublicErrorIdentities(t *testing.T) {
|
||||||
|
newEngine := func(t *testing.T, options ...promptkit.Option) *promptkit.Engine {
|
||||||
|
t.Helper()
|
||||||
|
engine, err := promptkit.NewEngine(
|
||||||
|
promptkit.Config{},
|
||||||
|
append([]promptkit.Option{promptkit.WithPromptFS(fstest.MapFS{}, ".")}, options...)...,
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("construct inspection engine: %v", err)
|
||||||
|
}
|
||||||
|
return engine
|
||||||
|
}
|
||||||
|
|
||||||
|
var nilEngine *promptkit.Engine
|
||||||
|
if result, err := nilEngine.InspectProfile(context.Background(), "profile"); result != nil ||
|
||||||
|
!errors.Is(err, promptkit.ErrInvalidConfig) {
|
||||||
|
t.Fatalf("nil engine result=(%#v, %v), want ErrInvalidConfig", result, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
valid := newEngine(t, promptkit.WithProfiles(promptkit.Profile{
|
||||||
|
ID: "profile", Endpoint: "http://profile.example/v1", Model: "model",
|
||||||
|
}))
|
||||||
|
if result, err := valid.InspectProfile(context.Background(), " \t "); result != nil ||
|
||||||
|
!errors.Is(err, promptkit.ErrInvalidRequest) {
|
||||||
|
t.Fatalf("blank profile result=(%#v, %v), want ErrInvalidRequest", result, err)
|
||||||
|
}
|
||||||
|
if result, err := valid.InspectProfile(context.Background(), "missing"); result != nil ||
|
||||||
|
!errors.Is(err, promptkit.ErrProfileNotFound) || errors.Is(err, promptkit.ErrProfileLoad) {
|
||||||
|
t.Fatalf("missing profile result=(%#v, %v), want only ErrProfileNotFound", result, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
malformed := newEngine(t, promptkit.WithProfileFS(fstest.MapFS{
|
||||||
|
"broken.yaml": &fstest.MapFile{Data: []byte("id: broken\nendpoint: http://broken.example/v1\nmodel: model\nunknown: value\n")},
|
||||||
|
}, "."))
|
||||||
|
if result, err := malformed.InspectProfile(context.Background(), "broken"); result != nil ||
|
||||||
|
!errors.Is(err, promptkit.ErrProfileLoad) {
|
||||||
|
t.Fatalf("malformed profile result=(%#v, %v), want ErrProfileLoad", result, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
unknownBackend := newEngine(t, promptkit.WithProfiles(promptkit.Profile{
|
||||||
|
ID: "unknown-backend", BackendID: "unknown", Model: "model",
|
||||||
|
}))
|
||||||
|
if result, err := unknownBackend.InspectProfile(context.Background(), "unknown-backend"); result != nil ||
|
||||||
|
!errors.Is(err, promptkit.ErrProfileLoad) {
|
||||||
|
t.Fatalf("unknown backend result=(%#v, %v), want ErrProfileLoad", result, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
countingFS := &inspectionCountingFS{}
|
||||||
|
canceled := newEngine(t, promptkit.WithProfileFS(countingFS, "."))
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
cancel()
|
||||||
|
if result, err := canceled.InspectProfile(ctx, "profile"); result != nil ||
|
||||||
|
!errors.Is(err, promptkit.ErrProfileLoad) || !errors.Is(err, context.Canceled) || countingFS.opens.Load() != 0 {
|
||||||
|
t.Fatalf("canceled inspection result=(%#v, %v), opens=%d", result, err, countingFS.opens.Load())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestInspectProfileReturnsIndependentTargetMatchingPreparation(t *testing.T) {
|
||||||
|
extraParams := map[string]any{
|
||||||
|
"nested": map[string]any{"value": "original"},
|
||||||
|
}
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{},
|
||||||
|
promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."),
|
||||||
|
promptkit.WithProfiles(promptkit.Profile{
|
||||||
|
ID: "profile", Endpoint: "http://profile.example/v1", Model: "model", ExtraParams: extraParams,
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("construct inspection engine: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
first, err := engine.InspectProfile(context.Background(), "profile")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("first inspection: %v", err)
|
||||||
|
}
|
||||||
|
first.EffectiveModelParams.ExtraParams["nested"].(map[string]any)["value"] = "changed"
|
||||||
|
first.EffectiveModelParams.ExtraParams["later"] = true
|
||||||
|
|
||||||
|
second, err := engine.InspectProfile(context.Background(), "profile")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("second inspection: %v", err)
|
||||||
|
}
|
||||||
|
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("prepare after inspection mutation: %v", err)
|
||||||
|
}
|
||||||
|
for _, target := range []promptkit.ExecutionTarget{second.EffectiveModelParams, prepared.EffectiveModelParams} {
|
||||||
|
if target.ExtraParams["nested"].(map[string]any)["value"] != "original" || target.ExtraParams["later"] != nil {
|
||||||
|
t.Fatalf("inspection mutation reached engine-owned target: %#v", target.ExtraParams)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(second.EffectiveModelParams, prepared.EffectiveModelParams) {
|
||||||
|
t.Fatalf("inspection target=%#v, preparation target=%#v", second.EffectiveModelParams, prepared.EffectiveModelParams)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestInspectPromptReturnsDeclaredMetadataWithoutExecutionWork(t *testing.T) {
|
||||||
|
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "unexpected"}}
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{},
|
||||||
|
promptkit.WithPromptFS(fstest.MapFS{
|
||||||
|
"report-v1.yaml": &fstest.MapFile{Data: []byte(`id: report
|
||||||
|
version: "1.0.0"
|
||||||
|
messages:
|
||||||
|
- role: user
|
||||||
|
content: old report
|
||||||
|
output:
|
||||||
|
format: text
|
||||||
|
validation_mode: none
|
||||||
|
`)},
|
||||||
|
"report-v2.yaml": &fstest.MapFile{Data: []byte(`id: report
|
||||||
|
version: "2.0.0"
|
||||||
|
default_profile: missing-profile
|
||||||
|
inputs:
|
||||||
|
- name: location
|
||||||
|
required: true
|
||||||
|
content_type: text/plain
|
||||||
|
description: Forecast location.
|
||||||
|
- name: units
|
||||||
|
content_type: text/plain
|
||||||
|
description: Unit preference.
|
||||||
|
messages:
|
||||||
|
- role: user
|
||||||
|
content_file: messages/report.md
|
||||||
|
output:
|
||||||
|
format: json
|
||||||
|
validation_mode: json_schema
|
||||||
|
schema_path: schemas/report.json
|
||||||
|
`)},
|
||||||
|
"messages/report.md": &fstest.MapFile{Data: []byte("rendered report body is not returned")},
|
||||||
|
}, "."),
|
||||||
|
promptkit.WithLLMClient(client),
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("construct prompt-inspection engine: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
inspection, err := engine.InspectPrompt(context.Background(), "report", "2.0.0")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("inspect prompt: %v", err)
|
||||||
|
}
|
||||||
|
if inspection.PromptID != "report" ||
|
||||||
|
inspection.PromptVersion != "2.0.0" ||
|
||||||
|
inspection.PromptHash == "" ||
|
||||||
|
inspection.DefaultProfileID != "missing-profile" ||
|
||||||
|
inspection.OutputContract != (promptkit.OutputContract{
|
||||||
|
Format: promptkit.FormatJSON,
|
||||||
|
ValidationMode: promptkit.ValidationJSONSchema,
|
||||||
|
SchemaPath: "schemas/report.json",
|
||||||
|
}) {
|
||||||
|
t.Fatalf("unexpected inspection metadata: %#v", inspection)
|
||||||
|
}
|
||||||
|
wantInputs := []promptkit.PromptInputDefinition{
|
||||||
|
{Name: "location", Required: true, ContentType: "text/plain", Description: "Forecast location."},
|
||||||
|
{Name: "units", ContentType: "text/plain", Description: "Unit preference."},
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(inspection.Inputs, wantInputs) {
|
||||||
|
t.Fatalf("inspection inputs=%#v, want %#v", inspection.Inputs, wantInputs)
|
||||||
|
}
|
||||||
|
if len(client.requests) != 0 {
|
||||||
|
t.Fatalf("inspection invoked the model client %d times", len(client.requests))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestInspectPromptPreservesPublicErrorIdentities(t *testing.T) {
|
||||||
|
newEngine := func(t *testing.T, source fstest.MapFS) *promptkit.Engine {
|
||||||
|
t.Helper()
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{}, promptkit.WithPromptFS(source, "."))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("construct prompt-inspection engine: %v", err)
|
||||||
|
}
|
||||||
|
return engine
|
||||||
|
}
|
||||||
|
validSource := fstest.MapFS{
|
||||||
|
"prompt.yaml": &fstest.MapFile{Data: []byte(`id: prompt
|
||||||
|
version: "1"
|
||||||
|
messages:
|
||||||
|
- role: user
|
||||||
|
content: body
|
||||||
|
output:
|
||||||
|
format: text
|
||||||
|
validation_mode: none
|
||||||
|
`)},
|
||||||
|
}
|
||||||
|
|
||||||
|
var nilEngine *promptkit.Engine
|
||||||
|
if result, err := nilEngine.InspectPrompt(context.Background(), "prompt", "1"); result != nil ||
|
||||||
|
!errors.Is(err, promptkit.ErrInvalidConfig) {
|
||||||
|
t.Fatalf("nil engine result=(%#v, %v), want ErrInvalidConfig", result, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
valid := newEngine(t, validSource)
|
||||||
|
if result, err := valid.InspectPrompt(context.Background(), " \t ", "1"); result != nil ||
|
||||||
|
!errors.Is(err, promptkit.ErrInvalidRequest) {
|
||||||
|
t.Fatalf("blank prompt result=(%#v, %v), want ErrInvalidRequest", result, err)
|
||||||
|
}
|
||||||
|
if result, err := valid.InspectPrompt(context.Background(), "missing", "1"); result != nil ||
|
||||||
|
!errors.Is(err, promptkit.ErrPromptNotFound) || errors.Is(err, promptkit.ErrPromptLoad) {
|
||||||
|
t.Fatalf("missing prompt result=(%#v, %v), want only ErrPromptNotFound", result, err)
|
||||||
|
}
|
||||||
|
if result, err := valid.InspectPrompt(context.Background(), "prompt", "missing"); result != nil ||
|
||||||
|
!errors.Is(err, promptkit.ErrPromptNotFound) || errors.Is(err, promptkit.ErrPromptLoad) {
|
||||||
|
t.Fatalf("missing version result=(%#v, %v), want only ErrPromptNotFound", result, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
ambiguous := newEngine(t, fstest.MapFS{
|
||||||
|
"one.yaml": &fstest.MapFile{Data: []byte(`id: prompt
|
||||||
|
version: "1"
|
||||||
|
messages:
|
||||||
|
- role: user
|
||||||
|
content: first
|
||||||
|
output:
|
||||||
|
format: text
|
||||||
|
validation_mode: none
|
||||||
|
`)},
|
||||||
|
"two.yaml": &fstest.MapFile{Data: []byte(`id: prompt
|
||||||
|
version: "2"
|
||||||
|
messages:
|
||||||
|
- role: user
|
||||||
|
content: second
|
||||||
|
output:
|
||||||
|
format: text
|
||||||
|
validation_mode: none
|
||||||
|
`)},
|
||||||
|
})
|
||||||
|
if result, err := ambiguous.InspectPrompt(context.Background(), "prompt", ""); result != nil ||
|
||||||
|
!errors.Is(err, promptkit.ErrPromptLoad) {
|
||||||
|
t.Fatalf("ambiguous prompt result=(%#v, %v), want ErrPromptLoad", result, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
for name, source := range map[string]fstest.MapFS{
|
||||||
|
"malformed definition": {
|
||||||
|
"broken.yaml": &fstest.MapFile{Data: []byte("id: broken\nversion: \"1\"\nunknown: value\n")},
|
||||||
|
},
|
||||||
|
"missing content file": {
|
||||||
|
"broken.yaml": &fstest.MapFile{Data: []byte(`id: broken
|
||||||
|
version: "1"
|
||||||
|
messages:
|
||||||
|
- role: user
|
||||||
|
content_file: missing.md
|
||||||
|
output:
|
||||||
|
format: text
|
||||||
|
validation_mode: none
|
||||||
|
`)},
|
||||||
|
},
|
||||||
|
} {
|
||||||
|
t.Run(name, func(t *testing.T) {
|
||||||
|
if result, err := newEngine(t, source).InspectPrompt(context.Background(), "broken", "1"); result != nil ||
|
||||||
|
!errors.Is(err, promptkit.ErrPromptLoad) {
|
||||||
|
t.Fatalf("broken prompt result=(%#v, %v), want ErrPromptLoad", result, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
countingFS := &inspectionCountingFS{}
|
||||||
|
canceled, err := promptkit.NewEngine(promptkit.Config{}, promptkit.WithPromptFS(countingFS, "."))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("construct canceled prompt-inspection engine: %v", err)
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
cancel()
|
||||||
|
if result, err := canceled.InspectPrompt(ctx, "prompt", "1"); result != nil ||
|
||||||
|
!errors.Is(err, promptkit.ErrPromptLoad) || !errors.Is(err, context.Canceled) || countingFS.opens.Load() != 0 {
|
||||||
|
t.Fatalf("canceled inspection result=(%#v, %v), opens=%d", result, err, countingFS.opens.Load())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestInspectPromptReturnsIndependentMetadataMatchingPreparation(t *testing.T) {
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{},
|
||||||
|
promptkit.WithPromptFS(fstest.MapFS{
|
||||||
|
"prompt.yaml": &fstest.MapFile{Data: []byte(`id: prompt
|
||||||
|
version: "1"
|
||||||
|
default_profile: profile
|
||||||
|
inputs:
|
||||||
|
- name: subject
|
||||||
|
content_type: text/plain
|
||||||
|
description: Summary subject.
|
||||||
|
messages:
|
||||||
|
- role: user
|
||||||
|
content: summarize
|
||||||
|
output:
|
||||||
|
format: markdown
|
||||||
|
validation_mode: basic
|
||||||
|
`)},
|
||||||
|
}, "."),
|
||||||
|
promptkit.WithProfiles(promptkit.Profile{
|
||||||
|
ID: "profile", Endpoint: "http://profile.example/v1", Model: "model",
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("construct prompt-inspection engine: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
first, err := engine.InspectPrompt(context.Background(), "prompt", "1")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("first inspection: %v", err)
|
||||||
|
}
|
||||||
|
first.Inputs[0].Name = "changed"
|
||||||
|
first.OutputContract.SchemaPath = "changed.json"
|
||||||
|
|
||||||
|
second, err := engine.InspectPrompt(context.Background(), "prompt", "1")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("second inspection: %v", err)
|
||||||
|
}
|
||||||
|
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt", PromptVersion: "1"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("prepare after inspection mutation: %v", err)
|
||||||
|
}
|
||||||
|
if second.Inputs[0].Name != "subject" || second.OutputContract.SchemaPath != "" ||
|
||||||
|
prepared.OutputContract.SchemaPath != "" || second.PromptHash != prepared.PromptHash {
|
||||||
|
t.Fatalf("inspection mutation reached engine-owned prompt metadata: inspection=%#v prepared=%#v", second, prepared)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestPreparedRunJSONOmitsZeroTimingValues(t *testing.T) {
|
func TestPreparedRunJSONOmitsZeroTimingValues(t *testing.T) {
|
||||||
payload, err := json.Marshal(promptkit.PreparedRun{})
|
payload, err := json.Marshal(promptkit.PreparedRun{})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -138,6 +514,49 @@ func TestUnknownProfileBackendHasProfileLoadIdentity(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestLocalBackendConstructsAndRegistersConventionalBackend(t *testing.T) {
|
||||||
|
const (
|
||||||
|
localBackendID = "local"
|
||||||
|
limit = 2
|
||||||
|
)
|
||||||
|
if promptkit.BackendLocal != localBackendID {
|
||||||
|
t.Fatalf("BackendLocal=%q, want %q", promptkit.BackendLocal, localBackendID)
|
||||||
|
}
|
||||||
|
|
||||||
|
endpoint := "http://local.example/v1"
|
||||||
|
backend := promptkit.LocalBackend(endpoint, limit)
|
||||||
|
want := promptkit.Backend{
|
||||||
|
ID: localBackendID,
|
||||||
|
Endpoint: endpoint,
|
||||||
|
ConcurrencyLimit: limit,
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(backend, want) {
|
||||||
|
t.Fatalf("LocalBackend()=%+v, want %+v", backend, want)
|
||||||
|
}
|
||||||
|
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{},
|
||||||
|
promptkit.WithPromptFS(contractPromptFS("prompt", "local-profile", "message"), "."),
|
||||||
|
promptkit.WithBackend(backend),
|
||||||
|
promptkit.WithProfiles(promptkit.Profile{
|
||||||
|
ID: "local-profile",
|
||||||
|
BackendID: localBackendID,
|
||||||
|
Model: "model",
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("construct engine with local backend: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("prepare with local backend: %v", err)
|
||||||
|
}
|
||||||
|
if prepared.SelectedBackendID != localBackendID ||
|
||||||
|
prepared.EffectiveModelParams.Endpoint != endpoint {
|
||||||
|
t.Fatalf("unexpected local backend preparation: %+v", prepared)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestCustomBackendFlowsThroughProfilesOverridesAndInjectedClient(t *testing.T) {
|
func TestCustomBackendFlowsThroughProfilesOverridesAndInjectedClient(t *testing.T) {
|
||||||
t.Setenv("CUSTOM_LLM_KEY", "test-key")
|
t.Setenv("CUSTOM_LLM_KEY", "test-key")
|
||||||
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}}
|
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}}
|
||||||
@@ -554,6 +973,24 @@ func TestRepeatedOptionsUseLastValueInEachCategory(t *testing.T) {
|
|||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
|
t.Run("fallback profile source", func(t *testing.T) {
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{},
|
||||||
|
promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."),
|
||||||
|
promptkit.WithFallbackProfileFS(contractProfileFS("profile", "first-model"), "."),
|
||||||
|
promptkit.WithFallbackProfileFS(contractProfileFS("profile", "second-model"), "."),
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("construct engine: %v", err)
|
||||||
|
}
|
||||||
|
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("prepare from last fallback profile source: %v", err)
|
||||||
|
}
|
||||||
|
if prepared.EffectiveModelParams.Model != "second-model" {
|
||||||
|
t.Fatalf("expected last fallback profile source, got %q", prepared.EffectiveModelParams.Model)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
t.Run("in-memory profiles", func(t *testing.T) {
|
t.Run("in-memory profiles", func(t *testing.T) {
|
||||||
first := profile
|
first := profile
|
||||||
first.Model = "first-model"
|
first.Model = "first-model"
|
||||||
@@ -638,6 +1075,212 @@ func TestRepeatedOptionsUseLastValueInEachCategory(t *testing.T) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestFallbackProfileSourcePrecedence(t *testing.T) {
|
||||||
|
const profileID = "application-profile"
|
||||||
|
|
||||||
|
prepareModel := func(t *testing.T, engine *promptkit.Engine, promptID string) string {
|
||||||
|
t.Helper()
|
||||||
|
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: promptID})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("prepare: %v", err)
|
||||||
|
}
|
||||||
|
return prepared.EffectiveModelParams.Model
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Run("in-memory profiles override ordinary and fallback profiles", func(t *testing.T) {
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{},
|
||||||
|
promptkit.WithPromptFS(contractPromptFS("prompt", profileID, "message"), "."),
|
||||||
|
promptkit.WithFallbackProfileFS(contractProfileFS(profileID, "fallback-model"), "."),
|
||||||
|
promptkit.WithProfileFS(contractProfileFS(profileID, "ordinary-model"), "."),
|
||||||
|
promptkit.WithProfiles(promptkit.Profile{ID: profileID, Endpoint: "http://example.test/v1", Model: "memory-model"}),
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("construct engine: %v", err)
|
||||||
|
}
|
||||||
|
if model := prepareModel(t, engine, "prompt"); model != "memory-model" {
|
||||||
|
t.Fatalf("expected in-memory profile, got %q", model)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("ordinary filesystem source overrides fallback profile", func(t *testing.T) {
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{},
|
||||||
|
promptkit.WithPromptFS(contractPromptFS("prompt", profileID, "message"), "."),
|
||||||
|
promptkit.WithFallbackProfileFS(contractProfileFS(profileID, "fallback-model"), "."),
|
||||||
|
promptkit.WithProfileFS(contractProfileFS(profileID, "ordinary-model"), "."),
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("construct engine: %v", err)
|
||||||
|
}
|
||||||
|
if model := prepareModel(t, engine, "prompt"); model != "ordinary-model" {
|
||||||
|
t.Fatalf("expected ordinary profile, got %q", model)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("configured directory overrides fallback profile", func(t *testing.T) {
|
||||||
|
profileDir := t.TempDir()
|
||||||
|
writePublicProfileFile(t, profileDir, profileID, "http://example.test/v1", "directory-model")
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{ProfileDir: profileDir},
|
||||||
|
promptkit.WithPromptFS(contractPromptFS("prompt", profileID, "message"), "."),
|
||||||
|
promptkit.WithFallbackProfileFS(contractProfileFS(profileID, "fallback-model"), "."),
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("construct engine: %v", err)
|
||||||
|
}
|
||||||
|
if model := prepareModel(t, engine, "prompt"); model != "directory-model" {
|
||||||
|
t.Fatalf("expected configured directory profile, got %q", model)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("fallback profile overrides built-in profile", func(t *testing.T) {
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{},
|
||||||
|
promptkit.WithPromptFS(contractPromptFS("prompt", "mistral-small-3", "message"), "."),
|
||||||
|
promptkit.WithFallbackProfileFS(contractProfileFS("mistral-small-3", "fallback-model"), "."),
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("construct engine: %v", err)
|
||||||
|
}
|
||||||
|
if model := prepareModel(t, engine, "prompt"); model != "fallback-model" {
|
||||||
|
t.Fatalf("expected fallback profile, got %q", model)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("missing fallback profile uses built-in profile", func(t *testing.T) {
|
||||||
|
t.Setenv("OPENROUTER_API_KEY", "test-key")
|
||||||
|
baseline, err := promptkit.NewEngine(promptkit.Config{},
|
||||||
|
promptkit.WithPromptFS(contractPromptFS("prompt", "mistral-small-3", "message"), "."),
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("construct baseline engine: %v", err)
|
||||||
|
}
|
||||||
|
want := prepareModel(t, baseline, "prompt")
|
||||||
|
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{},
|
||||||
|
promptkit.WithPromptFS(contractPromptFS("prompt", "mistral-small-3", "message"), "."),
|
||||||
|
promptkit.WithFallbackProfileFS(contractProfileFS(profileID, "fallback-model"), "."),
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("construct engine: %v", err)
|
||||||
|
}
|
||||||
|
if model := prepareModel(t, engine, "prompt"); model != want {
|
||||||
|
t.Fatalf("expected built-in profile model %q, got %q", want, model)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFallbackProfileSourcePreservesLazyLoadingAndErrors(t *testing.T) {
|
||||||
|
const profileID = "application-profile"
|
||||||
|
|
||||||
|
t.Run("construction defers malformed fallback profiles", func(t *testing.T) {
|
||||||
|
_, err := promptkit.NewEngine(promptkit.Config{},
|
||||||
|
promptkit.WithPromptFS(fstest.MapFS{}, "."),
|
||||||
|
promptkit.WithFallbackProfileFS(fstest.MapFS{
|
||||||
|
"broken.yaml": &fstest.MapFile{Data: []byte("id: broken\nunknown: value\n")},
|
||||||
|
}, "."),
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("construct engine with malformed fallback profile: %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("unrelated malformed fallback profile does not block matching definition", func(t *testing.T) {
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{},
|
||||||
|
promptkit.WithPromptFS(contractPromptFS("prompt", profileID, "message"), "."),
|
||||||
|
promptkit.WithFallbackProfileFS(fstest.MapFS{
|
||||||
|
"broken.yaml": &fstest.MapFile{Data: []byte("id: unrelated\nunknown: value\n")},
|
||||||
|
"valid.yaml": &fstest.MapFile{Data: []byte("id: application-profile\nendpoint: http://example.test/v1\nmodel: fallback-model\n")},
|
||||||
|
}, "."),
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("construct engine: %v", err)
|
||||||
|
}
|
||||||
|
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("prepare from valid fallback profile: %v", err)
|
||||||
|
}
|
||||||
|
if prepared.EffectiveModelParams.Model != "fallback-model" {
|
||||||
|
t.Fatalf("unexpected fallback profile model: %q", prepared.EffectiveModelParams.Model)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("matching malformed fallback profile does not reach built-in profile", func(t *testing.T) {
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{},
|
||||||
|
promptkit.WithPromptFS(contractPromptFS("prompt", "mistral-small-3", "message"), "."),
|
||||||
|
promptkit.WithFallbackProfileFS(fstest.MapFS{
|
||||||
|
"mistral-small-3.yaml": &fstest.MapFile{Data: []byte("id: mistral-small-3\nendpoint: http://example.test/v1\nmodel: fallback-model\nunknown: value\n")},
|
||||||
|
}, "."),
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("construct engine: %v", err)
|
||||||
|
}
|
||||||
|
if _, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"}); !errors.Is(err, promptkit.ErrProfileLoad) {
|
||||||
|
t.Fatalf("expected ErrProfileLoad, got %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("matching malformed ordinary profile does not reach fallback profile", func(t *testing.T) {
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{},
|
||||||
|
promptkit.WithPromptFS(contractPromptFS("prompt", profileID, "message"), "."),
|
||||||
|
promptkit.WithProfileFS(fstest.MapFS{
|
||||||
|
"application-profile.yaml": &fstest.MapFile{Data: []byte("id: application-profile\nendpoint: http://example.test/v1\nmodel: ordinary-model\nunknown: value\n")},
|
||||||
|
}, "."),
|
||||||
|
promptkit.WithFallbackProfileFS(contractProfileFS(profileID, "fallback-model"), "."),
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("construct engine: %v", err)
|
||||||
|
}
|
||||||
|
if _, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"}); !errors.Is(err, promptkit.ErrProfileLoad) {
|
||||||
|
t.Fatalf("expected ErrProfileLoad, got %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFallbackProfileSourceWorksAcrossWorkflows(t *testing.T) {
|
||||||
|
const profileID = "application-profile"
|
||||||
|
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}}
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{},
|
||||||
|
promptkit.WithPromptFS(contractPromptFS("prompt", profileID, "message"), "."),
|
||||||
|
promptkit.WithFallbackProfileFS(contractProfileFS(profileID, "fallback-model"), "."),
|
||||||
|
promptkit.WithLLMClient(client),
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("construct engine: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
inspection, err := engine.InspectProfile(context.Background(), profileID)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("inspect fallback profile: %v", err)
|
||||||
|
}
|
||||||
|
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("prepare fallback profile: %v", err)
|
||||||
|
}
|
||||||
|
preparedExecution, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("prepare execution with fallback profile: %v", err)
|
||||||
|
}
|
||||||
|
preparedDetails := preparedExecution.Details()
|
||||||
|
preparedResult, err := engine.RunPrepared(context.Background(), preparedExecution)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("run prepared fallback profile: %v", err)
|
||||||
|
}
|
||||||
|
runResult, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("run fallback profile: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
for name, model := range map[string]string{
|
||||||
|
"inspection": inspection.EffectiveModelParams.Model,
|
||||||
|
"preparation": prepared.EffectiveModelParams.Model,
|
||||||
|
"prepared execution": preparedDetails.EffectiveModelParams.Model,
|
||||||
|
"prepared result": preparedResult.EffectiveModelParams.Model,
|
||||||
|
"run result": runResult.EffectiveModelParams.Model,
|
||||||
|
} {
|
||||||
|
if model != "fallback-model" {
|
||||||
|
t.Fatalf("%s model=%q, want fallback-model", name, model)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestEngineSupportsConcurrentPrepareAndRun(t *testing.T) {
|
func TestEngineSupportsConcurrentPrepareAndRun(t *testing.T) {
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{},
|
engine, err := promptkit.NewEngine(promptkit.Config{},
|
||||||
promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."),
|
promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."),
|
||||||
|
|||||||
186
types.go
186
types.go
@@ -51,7 +51,8 @@ const (
|
|||||||
// ValidationPassed means the generated output satisfied its contract.
|
// ValidationPassed means the generated output satisfied its contract.
|
||||||
ValidationPassed ValidationStatus = "passed"
|
ValidationPassed ValidationStatus = "passed"
|
||||||
// ValidationFailed means validation completed and rejected the generated
|
// ValidationFailed means validation completed and rejected the generated
|
||||||
// output. Engine.Run returns this status in a result, not as an error.
|
// output. Engine.Run and Engine.RunPrepared return this status in a result,
|
||||||
|
// not as an error.
|
||||||
ValidationFailed ValidationStatus = "failed"
|
ValidationFailed ValidationStatus = "failed"
|
||||||
// ValidationSkipped means ValidationNone selected no content check.
|
// ValidationSkipped means ValidationNone selected no content check.
|
||||||
ValidationSkipped ValidationStatus = "skipped"
|
ValidationSkipped ValidationStatus = "skipped"
|
||||||
@@ -78,9 +79,10 @@ const (
|
|||||||
// RunRequest selects one prompt execution. It has no stable JSON
|
// RunRequest selects one prompt execution. It has no stable JSON
|
||||||
// representation.
|
// representation.
|
||||||
//
|
//
|
||||||
// Prepare and Run copy the request's maps, pointers, and nested
|
// Prepare, PrepareExecution, and Run copy the request's maps, pointers, and
|
||||||
// JSON-compatible values before using them. The caller may mutate the request
|
// nested JSON-compatible values before using them. The caller may mutate the
|
||||||
// after either method returns.
|
// request after any method returns. A successful PrepareExecution retains its
|
||||||
|
// own private execution snapshot for RunPrepared.
|
||||||
type RunRequest struct {
|
type RunRequest struct {
|
||||||
// PromptID is the required non-empty prompt identifier.
|
// PromptID is the required non-empty prompt identifier.
|
||||||
PromptID string
|
PromptID string
|
||||||
@@ -98,12 +100,14 @@ type RunRequest struct {
|
|||||||
// opaque consumer metadata, not a credential, and may be exposed in
|
// opaque consumer metadata, not a credential, and may be exposed in
|
||||||
// prepared values, results, collaborator requests, provider requests, and
|
// prepared values, results, collaborator requests, provider requests, and
|
||||||
// provider observability. Callers should use stable, non-sensitive
|
// provider observability. Callers should use stable, non-sensitive
|
||||||
// identifiers. An overlong direct value makes Prepare or Run return an
|
// identifiers. An overlong direct value makes Prepare, PrepareExecution, or
|
||||||
// error matching ErrInvalidRequest.
|
// Run return an error matching ErrInvalidRequest.
|
||||||
SessionID string
|
SessionID string
|
||||||
// APIKey is a request-scoped direct credential. It takes precedence over
|
// APIKey is a request-scoped direct credential. It takes precedence over
|
||||||
// APIKeyEnv, is passed to the selected LLMClient, and is never included in
|
// APIKeyEnv, is passed to the selected LLMClient, and is never included in
|
||||||
// prepared values, results, hashes, JSON, String, or GoString output.
|
// prepared values, results, hashes, JSON, String, or GoString output. A
|
||||||
|
// successful PrepareExecution retains it only in the opaque handle until
|
||||||
|
// RunPrepared claims the handle or Discard invalidates it.
|
||||||
APIKey string `json:"-"`
|
APIKey string `json:"-"`
|
||||||
// Inputs maps prompt input names to references. A nil or empty map is valid
|
// Inputs maps prompt input names to references. A nil or empty map is valid
|
||||||
// only when the selected prompt and its templates require no inputs.
|
// only when the selected prompt and its templates require no inputs.
|
||||||
@@ -112,16 +116,18 @@ type RunRequest struct {
|
|||||||
// empty maps are equivalent.
|
// empty maps are equivalent.
|
||||||
Vars map[string]string
|
Vars map[string]string
|
||||||
// Execution optionally overrides individual execution settings. Nil uses
|
// Execution optionally overrides individual execution settings. Nil uses
|
||||||
// the selected profile over its backend, when any, and framework defaults.
|
// the selected profile over its backend, when any, and the framework
|
||||||
|
// baseline.
|
||||||
Execution *ExecutionTargetOverride
|
Execution *ExecutionTargetOverride
|
||||||
// Validation optionally replaces the prompt's complete output contract. It
|
// Validation optionally replaces the prompt's complete output contract. It
|
||||||
// does not merge individual fields. Nil uses the prompt contract.
|
// does not merge individual fields. Nil uses the prompt contract.
|
||||||
Validation *OutputContract
|
Validation *OutputContract
|
||||||
}
|
}
|
||||||
|
|
||||||
// PreparedRun contains prepared prompt execution state. It does not include
|
// PreparedRun contains prepared prompt execution state returned by
|
||||||
// resolved API key values, model output, validation results, or internal target
|
// [Engine.Prepare] or [PreparedExecution.Details]. It does not include resolved
|
||||||
// presence metadata. PreparedRun has a stable JSON representation.
|
// API key values, model output, validation results, or internal target presence
|
||||||
|
// metadata. PreparedRun has a stable JSON representation.
|
||||||
//
|
//
|
||||||
// All maps, slices, pointers, and schema values are caller-owned copies. JSON
|
// All maps, slices, pointers, and schema values are caller-owned copies. JSON
|
||||||
// timestamps use RFC 3339 and zero timing values are omitted. Hash formats are
|
// timestamps use RFC 3339 and zero timing values are omitted. Hash formats are
|
||||||
@@ -139,9 +145,10 @@ type PreparedRun struct {
|
|||||||
// SelectedBackendID equals EffectiveModelParams.BackendID. It is empty for
|
// SelectedBackendID equals EffectiveModelParams.BackendID. It is empty for
|
||||||
// an endpoint-only profile.
|
// an endpoint-only profile.
|
||||||
SelectedBackendID string `json:"selected_backend_id,omitempty"`
|
SelectedBackendID string `json:"selected_backend_id,omitempty"`
|
||||||
// EffectiveModelParams contains framework defaults overlaid by the selected
|
// EffectiveModelParams contains settings resolved from the framework timeout
|
||||||
// backend, profile, and then request overrides. It excludes resolved API-key
|
// baseline, selected backend, profile, and then request overrides. Unset
|
||||||
// values.
|
// optional provider controls remain zero rather than reporting a provider
|
||||||
|
// default. It excludes resolved API-key values.
|
||||||
EffectiveModelParams ExecutionTarget `json:"effective_model_params"`
|
EffectiveModelParams ExecutionTarget `json:"effective_model_params"`
|
||||||
// OutputContract is the complete effective output contract.
|
// OutputContract is the complete effective output contract.
|
||||||
OutputContract OutputContract `json:"output_contract"`
|
OutputContract OutputContract `json:"output_contract"`
|
||||||
@@ -154,7 +161,8 @@ type PreparedRun struct {
|
|||||||
SessionID string `json:"session_id,omitempty"`
|
SessionID string `json:"session_id,omitempty"`
|
||||||
// RenderedPromptHash is an opaque equality value for SessionID and Messages.
|
// RenderedPromptHash is an opaque equality value for SessionID and Messages.
|
||||||
RenderedPromptHash string `json:"rendered_prompt_hash"`
|
RenderedPromptHash string `json:"rendered_prompt_hash"`
|
||||||
// Messages are the rendered messages that Run passes to the LLM client.
|
// Messages are the rendered messages that Run or RunPrepared passes to the
|
||||||
|
// LLM client.
|
||||||
Messages []RenderedMessage `json:"messages"`
|
Messages []RenderedMessage `json:"messages"`
|
||||||
// StartTime is the UTC time at which preparation began.
|
// StartTime is the UTC time at which preparation began.
|
||||||
StartTime time.Time `json:"start_time,omitempty"`
|
StartTime time.Time `json:"start_time,omitempty"`
|
||||||
@@ -214,12 +222,15 @@ type RunResult struct {
|
|||||||
InputHashes map[string]string `json:"input_hashes,omitempty"`
|
InputHashes map[string]string `json:"input_hashes,omitempty"`
|
||||||
// Usage is the token accounting reported by the LLM client.
|
// Usage is the token accounting reported by the LLM client.
|
||||||
Usage TokenUsage `json:"usage"`
|
Usage TokenUsage `json:"usage"`
|
||||||
// StartTime is the UTC time immediately before preparation begins.
|
// StartTime is the UTC time immediately before ordinary Run preparation or
|
||||||
|
// after RunPrepared claims its handle.
|
||||||
StartTime time.Time `json:"start_time,omitempty"`
|
StartTime time.Time `json:"start_time,omitempty"`
|
||||||
// EndTime is the UTC time after generation and validation complete.
|
// EndTime is the UTC time after generation and validation complete.
|
||||||
EndTime time.Time `json:"end_time,omitempty"`
|
EndTime time.Time `json:"end_time,omitempty"`
|
||||||
// Duration covers preparation, generation, and validation. JSON represents
|
// Duration covers preparation, generation, and validation for Run. For
|
||||||
// it as integer milliseconds in duration_ms and omits a zero value.
|
// RunPrepared it covers only the execution attempt after claim and excludes
|
||||||
|
// preparation and consumer-held delay. JSON represents it as integer
|
||||||
|
// milliseconds in duration_ms and omits a zero value.
|
||||||
Duration time.Duration `json:"-"`
|
Duration time.Duration `json:"-"`
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -259,10 +270,11 @@ type Artifact struct {
|
|||||||
// ArtifactReader resolves a prompt input reference into its content.
|
// ArtifactReader resolves a prompt input reference into its content.
|
||||||
//
|
//
|
||||||
// Read may be called concurrently. It must honor ctx cancellation to make
|
// Read may be called concurrently. It must honor ctx cancellation to make
|
||||||
// Prepare and Run responsive to cancellation. The engine passes a copied ref
|
// Prepare, PrepareExecution, and Run responsive to cancellation. The engine
|
||||||
// and immediately copies the returned Artifact.Body; it does not retain either
|
// passes a copied ref and immediately copies the returned Artifact.Body; it
|
||||||
// value. Readers supply artifact metadata, and the engine assigns an input-map
|
// does not retain either value. Readers supply artifact metadata, and the
|
||||||
// name only when the returned artifact name is empty.
|
// engine assigns an input-map name only when the returned artifact name is
|
||||||
|
// empty.
|
||||||
//
|
//
|
||||||
// An injected reader owns any application-specific path containment,
|
// An injected reader owns any application-specific path containment,
|
||||||
// authorization, content-size, and content-type policy. It must protect
|
// authorization, content-size, and content-type policy. It must protect
|
||||||
@@ -288,13 +300,17 @@ type ExecutionTarget struct {
|
|||||||
Endpoint string `json:"endpoint"`
|
Endpoint string `json:"endpoint"`
|
||||||
// Model is the provider model identifier.
|
// Model is the provider model identifier.
|
||||||
Model string `json:"model"`
|
Model string `json:"model"`
|
||||||
// Temperature is the effective sampling temperature from 0 through 2.
|
// Temperature is the resolved sampling temperature from 0 through 2. Zero
|
||||||
|
// leaves the field unspecified to compatible providers unless the
|
||||||
|
// corresponding ExecutionTargetPresence bit is true.
|
||||||
Temperature float64 `json:"temperature"`
|
Temperature float64 `json:"temperature"`
|
||||||
// MaxTokens is the non-negative effective output-token limit. Zero leaves
|
// MaxTokens is the non-negative resolved output-token limit. Zero leaves
|
||||||
// the limit unspecified to compatible providers unless it was an explicit
|
// the limit unspecified to compatible providers unless the corresponding
|
||||||
// request override.
|
// ExecutionTargetPresence bit is true.
|
||||||
MaxTokens int `json:"max_tokens"`
|
MaxTokens int `json:"max_tokens"`
|
||||||
// TopP is the effective nucleus-sampling value from 0 through 1.
|
// TopP is the resolved nucleus-sampling value from 0 through 1. Zero leaves
|
||||||
|
// the field unspecified to compatible providers unless the corresponding
|
||||||
|
// ExecutionTargetPresence bit is true.
|
||||||
TopP float64 `json:"top_p"`
|
TopP float64 `json:"top_p"`
|
||||||
// TimeoutSeconds is the non-negative per-generation deadline. Zero disables
|
// TimeoutSeconds is the non-negative per-generation deadline. Zero disables
|
||||||
// this deadline without disabling caller cancellation or the transport cap.
|
// this deadline without disabling caller cancellation or the transport cap.
|
||||||
@@ -310,6 +326,63 @@ type ExecutionTarget struct {
|
|||||||
ExtraParams map[string]any `json:"extra_params"`
|
ExtraParams map[string]any `json:"extra_params"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ProfileInspection is the caller-owned result of [Engine.InspectProfile].
|
||||||
|
// It has no stable JSON representation.
|
||||||
|
//
|
||||||
|
// EffectiveModelParams contains a copied effective target. APIKeyRequired is
|
||||||
|
// separate from that target to preserve ExecutionTarget's general execution
|
||||||
|
// and stable JSON contracts.
|
||||||
|
type ProfileInspection struct {
|
||||||
|
// ProfileID is the trimmed, exact profile ID inspected by the engine.
|
||||||
|
ProfileID string
|
||||||
|
// EffectiveModelParams contains settings resolved from the framework timeout
|
||||||
|
// baseline, selected backend, and then profile, without a request override.
|
||||||
|
// Unset optional provider controls remain zero rather than reporting a
|
||||||
|
// provider default. APIKeyEnv is an environment-variable name, never its
|
||||||
|
// credential value.
|
||||||
|
EffectiveModelParams ExecutionTarget
|
||||||
|
// APIKeyRequired reports that a later execution must supply a direct API
|
||||||
|
// key or an explicit request environment override. It is mutually exclusive
|
||||||
|
// with a nonblank EffectiveModelParams.APIKeyEnv.
|
||||||
|
APIKeyRequired bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// PromptInputDefinition describes one declared prompt input.
|
||||||
|
// It has no stable JSON representation.
|
||||||
|
type PromptInputDefinition struct {
|
||||||
|
// Name is the normalized prompt input name.
|
||||||
|
Name string
|
||||||
|
// Required is the prompt definition's declared required flag. When true,
|
||||||
|
// preparation fails if the input is omitted. A false value does not account
|
||||||
|
// for input references in message or session-ID templates.
|
||||||
|
Required bool
|
||||||
|
// ContentType is the declared input media-type metadata.
|
||||||
|
ContentType string
|
||||||
|
// Description is the declared human-readable input description.
|
||||||
|
Description string
|
||||||
|
}
|
||||||
|
|
||||||
|
// PromptInspection is the caller-owned result of [Engine.InspectPrompt].
|
||||||
|
// It has no stable JSON representation.
|
||||||
|
//
|
||||||
|
// Inputs contains copied declared input metadata in definition order.
|
||||||
|
// OutputContract is the normalized contract declared by the prompt definition,
|
||||||
|
// rather than a request-level effective override. PromptHash is opaque.
|
||||||
|
type PromptInspection struct {
|
||||||
|
// PromptID is the normalized ID of the selected prompt definition.
|
||||||
|
PromptID string
|
||||||
|
// PromptVersion is the normalized version of the selected prompt definition.
|
||||||
|
PromptVersion string
|
||||||
|
// PromptHash is the opaque equality value for the selected definition.
|
||||||
|
PromptHash string
|
||||||
|
// DefaultProfileID is declared metadata and is not resolved by inspection.
|
||||||
|
DefaultProfileID string
|
||||||
|
// Inputs contains caller-owned declared input metadata in definition order.
|
||||||
|
Inputs []PromptInputDefinition
|
||||||
|
// OutputContract is the normalized contract declared by the definition.
|
||||||
|
OutputContract OutputContract
|
||||||
|
}
|
||||||
|
|
||||||
// ExecutionTargetOverride represents per-request runtime setting overrides and
|
// ExecutionTargetOverride represents per-request runtime setting overrides and
|
||||||
// has no stable JSON representation.
|
// has no stable JSON representation.
|
||||||
//
|
//
|
||||||
@@ -317,19 +390,26 @@ type ExecutionTarget struct {
|
|||||||
// fields replace profile values and preserve explicit zero or empty values. A
|
// fields replace profile values and preserve explicit zero or empty values. A
|
||||||
// non-empty ExtraParams map replaces the complete profile or backend map
|
// non-empty ExtraParams map replaces the complete profile or backend map
|
||||||
// rather than merging keys. Empty string fields, nil pointers, and a nil or
|
// rather than merging keys. Empty string fields, nil pointers, and a nil or
|
||||||
// empty ExtraParams map inherit the selected profile over its backend, when
|
// empty ExtraParams map inherit lower-precedence values. An optional provider
|
||||||
// any, and framework defaults.
|
// control that remains zero is unspecified; TimeoutSeconds retains its
|
||||||
|
// framework deadline when no higher-precedence value is present.
|
||||||
type ExecutionTargetOverride struct {
|
type ExecutionTargetOverride struct {
|
||||||
// Endpoint replaces the profile or backend endpoint when non-empty without
|
// Endpoint replaces the profile or backend endpoint when non-empty without
|
||||||
// changing the effective BackendID.
|
// changing the effective BackendID.
|
||||||
Endpoint string
|
Endpoint string
|
||||||
// Model replaces the profile model when non-empty.
|
// Model replaces the profile model when non-empty.
|
||||||
Model string
|
Model string
|
||||||
// Temperature, when non-nil, must point to a value from 0 through 2.
|
// Temperature, when non-nil, must point to a value from 0 through 2. A
|
||||||
|
// pointed-to zero is explicitly present; nil inherits a lower-precedence
|
||||||
|
// value and otherwise leaves the provider control unspecified.
|
||||||
Temperature *float64
|
Temperature *float64
|
||||||
// MaxTokens, when non-nil, must point to a non-negative value.
|
// MaxTokens, when non-nil, must point to a non-negative value. A pointed-to
|
||||||
|
// zero is explicitly present; nil inherits a lower-precedence value and
|
||||||
|
// otherwise leaves the provider control unspecified.
|
||||||
MaxTokens *int
|
MaxTokens *int
|
||||||
// TopP, when non-nil, must point to a value from 0 through 1.
|
// TopP, when non-nil, must point to a value from 0 through 1. A pointed-to
|
||||||
|
// zero is explicitly present; nil inherits a lower-precedence value and
|
||||||
|
// otherwise leaves the provider control unspecified.
|
||||||
TopP *float64
|
TopP *float64
|
||||||
// TimeoutSeconds, when non-nil, must point to a non-negative value. A
|
// TimeoutSeconds, when non-nil, must point to a non-negative value. A
|
||||||
// pointed-to zero disables the per-generation deadline.
|
// pointed-to zero disables the per-generation deadline.
|
||||||
@@ -360,9 +440,12 @@ type ExecutionTargetOverride struct {
|
|||||||
// use profile YAML api_key_env with file and FS profile sources. Profile has no
|
// use profile YAML api_key_env with file and FS profile sources. Profile has no
|
||||||
// stable JSON representation.
|
// stable JSON representation.
|
||||||
//
|
//
|
||||||
// WithProfiles validates and copies Profile values during NewEngine. Numeric
|
// WithProfiles validates and copies Profile values during NewEngine. Zero
|
||||||
// zero, blank strings, and an empty ExtraParams map inherit framework defaults;
|
// Temperature, MaxTokens, and TopP values and blank ServiceTier and
|
||||||
// use ExecutionTargetOverride pointer fields to request explicit numeric zero.
|
// ReasoningEffort values leave those provider controls unspecified. A zero
|
||||||
|
// TimeoutSeconds retains the framework deadline, while an empty ExtraParams map
|
||||||
|
// inherits backend request defaults. Use ExecutionTargetOverride pointer fields
|
||||||
|
// to request an explicit numeric zero.
|
||||||
type Profile struct {
|
type Profile struct {
|
||||||
// ID is the required non-blank profile identifier. WithProfiles trims it.
|
// ID is the required non-blank profile identifier. WithProfiles trims it.
|
||||||
ID string
|
ID string
|
||||||
@@ -376,19 +459,19 @@ type Profile struct {
|
|||||||
Endpoint string
|
Endpoint string
|
||||||
// Model is the required non-blank provider model identifier.
|
// Model is the required non-blank provider model identifier.
|
||||||
Model string
|
Model string
|
||||||
// Temperature is from 0 through 2. Zero inherits the framework default.
|
// Temperature is from 0 through 2. Zero leaves the provider control
|
||||||
|
// unspecified.
|
||||||
Temperature float64
|
Temperature float64
|
||||||
// MaxTokens is non-negative. Zero inherits the framework default.
|
// MaxTokens is non-negative. Zero leaves the provider control unspecified.
|
||||||
MaxTokens int
|
MaxTokens int
|
||||||
// TopP is from 0 through 1. Zero inherits the framework default rather than
|
// TopP is from 0 through 1. Zero leaves the provider control unspecified
|
||||||
// selecting an explicit zero.
|
// rather than selecting an explicit zero.
|
||||||
TopP float64
|
TopP float64
|
||||||
// TimeoutSeconds is non-negative. Zero inherits the framework default.
|
// TimeoutSeconds is non-negative. Zero retains the framework deadline.
|
||||||
TimeoutSeconds int
|
TimeoutSeconds int
|
||||||
// ServiceTier is optional; a blank value inherits the framework default.
|
// ServiceTier is optional; a blank value leaves it unspecified.
|
||||||
ServiceTier string
|
ServiceTier string
|
||||||
// ReasoningEffort is optional; a blank value inherits the framework
|
// ReasoningEffort is optional; a blank value leaves it unspecified.
|
||||||
// default.
|
|
||||||
ReasoningEffort string
|
ReasoningEffort string
|
||||||
// APIKeyRequired clears a backend's inherited API-key environment name and
|
// APIKeyRequired clears a backend's inherited API-key environment name and
|
||||||
// requires a non-blank RunRequest.APIKey unless the request explicitly
|
// requires a non-blank RunRequest.APIKey unless the request explicitly
|
||||||
@@ -559,25 +642,26 @@ type StructuredOutputJSONSpec struct {
|
|||||||
Schema any `json:"schema"`
|
Schema any `json:"schema"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// LLMClient executes rendered prompts for [Engine.Run].
|
// LLMClient executes rendered prompts for [Engine.Run] and
|
||||||
|
// [Engine.RunPrepared].
|
||||||
//
|
//
|
||||||
// Generate is scheduled according to the resolved backend's capacity policy.
|
// Generate is scheduled according to the resolved backend's capacity policy.
|
||||||
// It may still be called concurrently for different backend pools or unlimited
|
// It may still be called concurrently for different backend pools or unlimited
|
||||||
// backends. Cancellation while waiting for capacity can prevent Generate from
|
// backends. Cancellation while waiting for capacity can prevent Generate from
|
||||||
// being called. Once invoked, it must honor context cancellation to make Run
|
// being called. Once invoked, it must honor context cancellation to make Run
|
||||||
// responsive to cancellation. The request and all nested maps, slices, and
|
// and RunPrepared responsive to cancellation. The request and all nested maps,
|
||||||
// pointers are client-owned copies and may be mutated or retained without
|
// slices, and pointers are client-owned copies and may be mutated or retained
|
||||||
// affecting engine state.
|
// without affecting engine state.
|
||||||
//
|
//
|
||||||
// Generate receives rendered messages and may receive a direct API key. A
|
// Generate receives rendered messages and may receive a direct API key. A
|
||||||
// client must protect those values and any raw output in its logging, storage,
|
// client must protect those values and any raw output in its logging, storage,
|
||||||
// and retained copies. It is responsible for the cancellation behavior of any
|
// and retained copies. It is responsible for the cancellation behavior of any
|
||||||
// work it starts and for synchronizing access to retained or shared data.
|
// work it starts and for synchronizing access to retained or shared data.
|
||||||
//
|
//
|
||||||
// A returned error makes Run return ErrLLMGenerate while preserving the client
|
// A returned error makes Run or RunPrepared return ErrLLMGenerate while
|
||||||
// error through errors.Is. A nil response with a nil error also produces
|
// preserving the client error through errors.Is. A nil response with a nil
|
||||||
// ErrLLMGenerate. Promptkit copies the non-nil response before returning from
|
// error also produces ErrLLMGenerate. Promptkit copies the non-nil response
|
||||||
// Run.
|
// before returning from either method.
|
||||||
type LLMClient interface {
|
type LLMClient interface {
|
||||||
Generate(context.Context, GenerateRequest) (*GenerateResponse, error)
|
Generate(context.Context, GenerateRequest) (*GenerateResponse, error)
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user